Compare commits

..

1 Commits

Author SHA1 Message Date
Stuart Lynn
f72d2785d8 kde_contours inital stub 2016-08-10 15:39:58 -04:00
15 changed files with 113 additions and 487 deletions

View File

@@ -1,57 +0,0 @@
language: c
env:
global:
- PAGER=cat
before_install:
- ./check-up-to-date-with-master.sh
- sudo apt-get -y install python-pip
- sudo apt-get -y install python-software-properties
- sudo add-apt-repository -y ppa:cartodb/sci
- sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
- sudo add-apt-repository -y ppa:cartodb/gis
- sudo add-apt-repository -y ppa:cartodb/gis-testing
- sudo apt-get update
- sudo apt-get -y install python-joblib=0.8.3-1-cdb1
- sudo apt-get -y install python-numpy=1:1.6.1-6ubuntu1
# Install pysal
- sudo pip install -I pysal==1.11.2
- sudo apt-get -y install python-scipy=0.14.0-2-cdb6
- sudo apt-get -y --no-install-recommends install python-sklearn-lib=0.14.1-3-cdb2
- sudo apt-get -y --no-install-recommends install python-sklearn=0.14.1-3-cdb2
- sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2
# Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis)
- sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2
# Install postgres db and build deps
- sudo /etc/init.d/postgresql stop # stop travis default instance
- sudo apt-get -y remove --purge postgresql-9.1
- sudo apt-get -y remove --purge postgresql-9.2
- sudo apt-get -y remove --purge postgresql-9.3
- sudo apt-get -y remove --purge postgresql-9.4
- sudo apt-get -y remove --purge postgis
- sudo apt-get -y autoremove
- sudo apt-get -y install postgresql-9.5=9.5.2-2ubuntu1
- sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-2ubuntu1
- sudo apt-get -y install postgresql-plpython-9.5=9.5.2-2ubuntu1
- sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
- sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
# configure it to accept local connections from postgres
- echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \
| sudo tee /etc/postgresql/9.5/main/pg_hba.conf
- sudo /etc/init.d/postgresql restart 9.5
install:
- sudo make install
script:
- make test || { cat src/pg/test/regression.diffs; false; }
- ./check-compatibility.sh

View File

@@ -1,4 +1,4 @@
# crankshaft [![Build Status](https://travis-ci.org/CartoDB/crankshaft.svg?branch=develop)](https://travis-ci.org/CartoDB/crankshaft)
# crankshaft
CartoDB Spatial Analysis extension for PostgreSQL.

View File

@@ -1,108 +0,0 @@
#!/bin/bash
export PGUSER=postgres
DBNAME=crankshaft_compatcheck
function die {
echo $1
exit -1
}
# Create fresh DB
psql -c "CREATE DATABASE $DBNAME;" || die "Could not create DB"
# Hook for cleanup
function cleanup {
psql -c "DROP DATABASE IF EXISTS crankshaft_compatcheck;"
}
trap cleanup EXIT
# Deploy previous release
(cd src/py && sudo make deploy RUN_OPTIONS="--no-deps") || die "Could not deploy python extension"
(cd src/pg && sudo make deploy) || die " Could not deploy last release"
psql -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'crankshaft';"
# Install in the fresh DB
psql $DBNAME <<'EOF'
-- Install dependencies
CREATE EXTENSION plpythonu;
CREATE EXTENSION postgis VERSION '2.2.2';
-- Create role publicuser if it does not exist
DO
$$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'publicuser') THEN
CREATE ROLE publicuser LOGIN;
END IF;
END
$$ LANGUAGE plpgsql;
-- Install the default version
CREATE EXTENSION crankshaft;
\dx
EOF
# Save public function signatures
psql $DBNAME <<'EOF'
CREATE TABLE release_function_signatures AS
SELECT
p.proname as name,
pg_catalog.pg_get_function_result(p.oid) as result_type,
pg_catalog.pg_get_function_arguments(p.oid) as arguments,
CASE
WHEN p.proisagg THEN 'agg'
WHEN p.proiswindow THEN 'window'
WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
ELSE 'normal'
END as type
FROM pg_catalog.pg_proc p
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
WHERE
n.nspname = 'cdb_crankshaft'
AND p.proname LIKE 'cdb_%'
ORDER BY 1, 2, 4;
EOF
# Deploy current dev branch
make clean-dev || die "Could not clean dev files"
sudo make install || die "Could not deploy current dev branch"
# Check it can be upgraded
psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version"
# Check against saved public function signatures
psql $DBNAME <<'EOF'
CREATE TABLE dev_function_signatures AS
SELECT
p.proname as name,
pg_catalog.pg_get_function_result(p.oid) as result_type,
pg_catalog.pg_get_function_arguments(p.oid) as arguments,
CASE
WHEN p.proisagg THEN 'agg'
WHEN p.proiswindow THEN 'window'
WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
ELSE 'normal'
END as type
FROM pg_catalog.pg_proc p
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
WHERE
n.nspname = 'cdb_crankshaft'
AND p.proname LIKE 'cdb_%'
ORDER BY 1, 2, 4;
EOF
echo "Functions in development not in latest release (ok):"
psql $DBNAME -c "SELECT * FROM dev_function_signatures EXCEPT SELECT * FROM release_function_signatures;"
echo "Functions in latest release not in development (compat issue):"
psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;"
# Fail if there's a signature mismatch / missing functions
psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;" | fgrep '(0 rows)' \
|| die "Function signatures changed"

View File

@@ -1,16 +0,0 @@
#!/bin/bash
# Add remote-master
git remote add -t master remote-master https://github.com/CartoDB/crankshaft.git
# Fetch master reference
git fetch --depth=1 remote-master master
# Compare HEAD with master
# NOTE: travis by default uses --depth=50 so we are actually checking that the tip
# of the branch is no more than 50 commits away from master as well.
git rev-list HEAD | grep $(git rev-parse remote-master/master) ||
{ echo "Your branch is not up to date with latest release";
echo "Please update it by running the following:";
echo " git fetch && git merge origin/develop";
false; }

View File

@@ -1,78 +0,0 @@
## Gravity Model
Gravity Models are derived from Newton's Law of Gravity and are used to predict the interaction between a group of populated areas (sources) and a specific target among a group of potential targets, in terms of an attraction factor (weight)
**CDB_Gravity** is based on the model defined in *Huff's Law of Shopper attraction (1963)*
### CDB_Gravity(t_id bigint[], t_geom geometry[], t_weight numeric[], s_id bigint[], s_geom geometry[], s_pop numeric[], target bigint, radius integer, minval numeric DEFAULT -10e307)
#### Arguments
| Name | Type | Description |
|------|------|-------------|
| t_id | bigint[] | Array of targets ID |
| t_geom | geometry[] | Array of targets' geometries |
| t_weight | numeric[] | Array of targets's weights |
| s_id | bigint[] | Array of sources ID |
| s_geom | geometry[] | Array of sources' geometries |
| s_pop | numeric[] | Array of sources's population |
| target | bigint | ID of the target under study |
| radius | integer | Radius in meters around the target under study that will be taken into account|
| minval (optional) | numeric | Lowest accepted value of weight, defaults to numeric min_value |
### CDB_Gravity( target_query text, weight_column text, source_query text, pop_column text, target bigint, radius integer, minval numeric DEFAULT -10e307)
#### Arguments
| Name | Type | Description |
|------|------|-------------|
| target_query | text | Query that defines targets |
| weight_column | text | Column name of weights |
| source_query | text | Query that defines sources |
| pop_column | text | Column name of population |
| target | bigint | cartodb_id of the target under study |
| radius | integer | Radius in meters around the target under study that will be taken into account|
| minval (optional) | numeric | Lowest accepted value of weight, defaults to numeric min_value |
### Returns
| Column Name | Type | Description |
|-------------|------|-------------|
| the_geom | geometry | Geometries of the sources within the radius |
| source_id | bigint | ID of the source |
| target_id | bigint | Target ID from input |
| dist | numeric | Distance in meters source to target (if not points, distance between centroids) |
| h | numeric | Probability of patronage |
| hpop | numeric | Patronaging population |
#### Example Usage
```sql
with t as (
SELECT
array_agg(cartodb_id::bigint) as id,
array_agg(the_geom) as g,
array_agg(coalesce(gla,0)::numeric) as w
FROM
abel.centros_comerciales_de_madrid
WHERE not no_cc
),
s as (
SELECT
array_agg(cartodb_id::bigint) as id,
array_agg(center) as g,
array_agg(coalesce(t1_1, 0)::numeric) as p
FROM
sscc_madrid
)
select
g.the_geom,
trunc(g.h,2) as h,
round(g.hpop) as hpop,
trunc(g.dist/1000,2) as dist_km
FROM t, s, CDB_Gravity1(t.id, t.g, t.w, s.id, s.g, s.p, newmall_ID, 100000, 5000) g
```

1
src/pg/.gitignore vendored
View File

@@ -4,4 +4,3 @@ results/
crankshaft--dev.sql
crankshaft--dev--current.sql
crankshaft--current--dev.sql
crankshaft--*--dev.sql

View File

@@ -10,15 +10,14 @@ include ../../Makefile.global
# * test runs the tests for the currently generated Development
# extension.
DATA = \
$(EXTENSION)--dev.sql \
$(EXTENSION)--current--dev.sql \
$(EXTENSION)--dev--current.sql \
$(EXTENSION)--$(RELEASE_VERSION)--dev.sql
DATA = $(EXTENSION)--dev.sql \
$(EXTENSION)--current--dev.sql \
$(EXTENSION)--dev--current.sql
SOURCES_DATA_DIR = sql
SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g'
$(DATA): $(SOURCES_DATA)

View File

@@ -10,22 +10,10 @@ CREATE OR REPLACE FUNCTION
END
$$ LANGUAGE plpgsql;
-- Create aggregate if it did not exist
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_proc p
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
WHERE n.nspname = 'cdb_crankshaft'
AND p.proname = 'cdb_pyagg'
AND p.proisagg)
THEN
CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) (
SFUNC = CDB_PyAggS,
STYPE = Numeric[],
INITCOND = "{}"
);
END IF;
END
$$ LANGUAGE plpgsql;
CREATE AGGREGATE CDB_PyAgg(NUMERIC[])(
SFUNC = CDB_PyAggS,
STYPE = Numeric[],
INITCOND = "{}"
);

View File

@@ -1,115 +0,0 @@
CREATE OR REPLACE FUNCTION CDB_Gravity(
IN target_query text,
IN weight_column text,
IN source_query text,
IN pop_column text,
IN target bigint,
IN radius integer,
IN minval numeric DEFAULT -10e307
)
RETURNS TABLE(
the_geom geometry,
source_id bigint,
target_id bigint,
dist numeric,
h numeric,
hpop numeric) AS $$
DECLARE
t_id bigint[];
t_geom geometry[];
t_weight numeric[];
s_id bigint[];
s_geom geometry[];
s_pop numeric[];
BEGIN
EXECUTE 'WITH foo as('+target_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || weight_column || ') FROM foo' INTO t_id, t_geom, t_weight;
EXECUTE 'WITH foo as('+source_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || pop_column || ') FROM foo' INTO s_id, s_geom, s_pop;
RETURN QUERY
SELECT g.* FROM t, s, CDB_Gravity(t_id, t_geom, t_weight, s_id, s_geom, s_pop, target, radius, minval) g;
END;
$$ language plpgsql;
CREATE OR REPLACE FUNCTION CDB_Gravity(
IN t_id bigint[],
IN t_geom geometry[],
IN t_weight numeric[],
IN s_id bigint[],
IN s_geom geometry[],
IN s_pop numeric[],
IN target bigint,
IN radius integer,
IN minval numeric DEFAULT -10e307
)
RETURNS TABLE(
the_geom geometry,
source_id bigint,
target_id bigint,
dist numeric,
h numeric,
hpop numeric) AS $$
DECLARE
t_type text;
s_type text;
t_center geometry[];
s_center geometry[];
BEGIN
t_type := GeometryType(t_geom[1]);
s_type := GeometryType(s_geom[1]);
IF t_type = 'POINT' THEN
t_center := t_geom;
ELSE
WITH tmp as (SELECT unnest(t_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO t_center FROM tmp;
END IF;
IF s_type = 'POINT' THEN
s_center := s_geom;
ELSE
WITH tmp as (SELECT unnest(s_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO s_center FROM tmp;
END IF;
RETURN QUERY
with target0 as(
SELECT unnest(t_center) as tc, unnest(t_weight) as tw, unnest(t_id) as td
),
source0 as(
SELECT unnest(s_center) as sc, unnest(s_id) as sd, unnest (s_geom) as sg, unnest(s_pop) as sp
),
prev0 as(
SELECT
source0.sg,
source0.sd as sourc_id,
coalesce(source0.sp,0) as sp,
target.td as targ_id,
coalesce(target.tw,0) as tw,
GREATEST(1.0,ST_Distance(geography(target.tc), geography(source0.sc)))::numeric as distance
FROM source0
CROSS JOIN LATERAL
(
SELECT
*
FROM target0
WHERE tw > minval
AND ST_DWithin(geography(source0.sc), geography(tc), radius)
) AS target
),
deno as(
SELECT
sourc_id,
sum(tw/distance) as h_deno
FROM
prev0
GROUP BY sourc_id
)
SELECT
p.sg as the_geom,
p.sourc_id as source_id,
p.targ_id as target_id,
case when p.distance > 1 then p.distance else 0.0 end as dist,
100*(p.tw/p.distance)/d.h_deno as h,
p.sp*(p.tw/p.distance)/d.h_deno as hpop
FROM
prev0 p,
deno d
WHERE
p.targ_id = target AND
p.sourc_id = d.sourc_id;
END;
$$ language plpgsql;

View File

@@ -41,23 +41,9 @@ BEGIN
END
$$ LANGUAGE plpgsql;
-- Create aggregate if it did not exist
DO $$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_proc p
LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
WHERE n.nspname = 'cdb_crankshaft'
AND p.proname = 'cdb_weightedmean'
AND p.proisagg)
THEN
CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) (
SFUNC = CDB_WeightedMeanS,
FINALFUNC = CDB_WeightedMeanF,
STYPE = Numeric[],
INITCOND = "{0.0,0.0,0.0}"
);
END IF;
END
$$ LANGUAGE plpgsql;
CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)(
SFUNC = CDB_WeightedMeanS,
FINALFUNC = CDB_WeightedMeanF,
STYPE = Numeric[],
INITCOND = "{0.0,0.0,0.0}"
);

View File

@@ -0,0 +1,93 @@
CREATE OR REPLACE FUNCTION CDB_KDEContours(geoms geometry[], point_vals numeric[], levels numeric[], bandwidth double precision, gridx integer DEFAULT 100, gridy integer DEFAULT 100)
RETURNS TABLE(geom geometry, val float) AS
$BODY$
DECLARE
extent box2d;
rast raster;
xcorner double precision;
ycorner double precision;
resolutionx double precision;
resolutiony double precision;
width double precision;
height double precision;
ynew double precision;
xnew double precision;
kde_value double precision;
kde_matrix double precision[];
distance double precision[];
point_value integer[];
length integer;
kde_term double precision;
query character varying;
constant double precision;
BEGIN
SELECT ST_EXTENT(ST_Collect(geoms)) INTO extent;
rast := st_makeemptyraster(gridx,
gridy,
ST_XMIN(extent),
ST_YMIN(extent),
(ST_XMAX(extent) - ST_XMIN(extent))/gridx,
(ST_YMAX(extent) - ST_YMIN(extent))/gridy,
0,
0,
4326);
rast := ST_AddBand(rast, 1, '32BF', 0, 0);
SELECT ST_UpperLeftX(rast) INTO xcorner;
SELECT ST_UpperLeftY(rast) INTO ycorner;
SELECT ST_ScaleX(rast) INTO resolutionx;
SELECT ST_ScaleY(rast) INTO resolutiony;
SELECT ST_Width(rast) into width;
SELECT ST_Height(rast) into height;
xcorner=xcorner + resolutionx/2;
ycorner=ycorner + resolutiony/2;
constant = 1; -- 3/(pi()*power(bandwidth, 2))*1000000;
FOR j in 0..height-1 LOOP
ynew=ycorner+j*resolutiony;
FOR i in 0..width-1 LOOP
xnew=xcorner+i*resolutionx;
SELECT
array_agg(
st_distance(
CDB_LATLNG(ynew,xnew),
a.geom
)
), array_agg(value)
INTO distance, point_value
FROM (select unnest(geoms) as geom , unnest(point_vals) as value ) as a
WHERE ST_DWITHIN(a.geom, CDB_LATLNG(ynew,xnew), bandwidth) ;
SELECT array_length(point_value, 1 ) into length;
kde_value=0;
IF length IS NOT NULL THEN
FOR k in 1..length LOOP
kde_term = point_value[k]*constant*power(1-power(distance[k]/bandwidth, 2), 2);
kde_value := kde_value+kde_term;
END LOOP;
END IF;
IF kde_value < levels[1] THEN
kde_matrix[i] := 1;
ELSE
kde_matrix[i] := (select bin - 1 from (
select GENERATE_SERIES(1, array_upper(levels,1)) as bin, unnest(levels) as level
) As x where kde_value < level limit 1);
END IF;
END LOOP;
SELECT ST_SetValues(rast, 1, 1, j+1, kde_matrix) into rast;
END LOOP;
RETURN QUERY
select x.geom, levels[x.val::int]::float from (
select (ST_DumpAsPolygons(rast)).*
) as x;
RETURN ;
END;
$BODY$
LANGUAGE plpgsql;

View File

@@ -1,18 +1,5 @@
-- Install dependencies
CREATE EXTENSION plpythonu;
CREATE EXTENSION postgis VERSION '2.2.2';
-- Create role publicuser if it does not exist
DO
$$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'publicuser') THEN
CREATE ROLE publicuser LOGIN;
END IF;
END
$$ LANGUAGE plpgsql;
CREATE EXTENSION postgis;
-- Install the extension
CREATE EXTENSION crankshaft VERSION 'dev';

View File

@@ -1,14 +0,0 @@
SET client_min_messages TO WARNING;
\set ECHO none
the_geom | h | hpop | dist
--------------------------------------------+-------------------------+--------------------------+----------------
01010000001361C3D32B650140DD24068195B34440 | 1.51078258369747945249 | 12.08626066957983561994 | 4964.714459152
01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954183620807430 | 688.08116679285345652007 | 99.955141922
0101000000A167B3EA733501401D5A643BDFAF4440 | 63.70532894711274639196 | 382.23197368267647835174 | 2488.330566505
010100000062A1D634EF380140BE9F1A2FDDB44440 | 35.35415870080995954879 | 176.77079350404979774397 | 4359.370460594
010100000052B81E85EB510140355EBA490CB24440 | 33.12290506987740864904 | 132.49162027950963459615 | 3703.664449828
0101000000C286A757CA320140736891ED7CAF4440 | 65.45251754279248087849 | 196.35755262837744263547 | 2512.092358644
01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792471345639225 | 125.67855584942691278449 | 2926.25725244
0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726651871279586 | 53.54905726651871279586 | 3744.515577777
(8 rows)

View File

@@ -1,20 +1,6 @@
-- Install dependencies
CREATE EXTENSION plpythonu;
CREATE EXTENSION postgis VERSION '2.2.2';
-- Create role publicuser if it does not exist
DO
$$
BEGIN
IF NOT EXISTS (
SELECT *
FROM pg_catalog.pg_user
WHERE usename = 'publicuser') THEN
CREATE ROLE publicuser LOGIN;
END IF;
END
$$ LANGUAGE plpgsql;
CREATE EXTENSION postgis;
-- Install the extension
CREATE EXTENSION crankshaft VERSION 'dev';

View File

@@ -1,24 +0,0 @@
SET client_min_messages TO WARNING;
\set ECHO none
WITH t AS (
SELECT
ARRAY[1,2,3] AS id,
ARRAY[7.0,8.0,3.0] AS w,
ARRAY[ST_GeomFromText('POINT(2.1744 41.4036)'),ST_GeomFromText('POINT(2.1228 41.3809)'),ST_GeomFromText('POINT(2.1511 41.3742)')] AS g
),
s AS (
SELECT
ARRAY[10,20,30,40,50,60,70,80] AS id,
ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS p,
ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g
)
SELECT
g.the_geom,
g.h,
g.hpop,
g.dist
FROM
t,
s,
cdb_crankshaft.CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g;