cleaned

Compatibility checks - take 2

.travis.yml

@@ -0,0 +1,57 @@
+language: c
+
+env:
+  global:
+    - PAGER=cat
+
+before_install:
+  - ./check-up-to-date-with-master.sh
+  - sudo apt-get -y install python-pip
+
+  - sudo apt-get -y install python-software-properties
+  - sudo add-apt-repository -y ppa:cartodb/sci
+  - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
+  - sudo add-apt-repository -y ppa:cartodb/gis
+  - sudo add-apt-repository -y ppa:cartodb/gis-testing
+  - sudo apt-get update
+
+  - sudo apt-get -y install python-joblib=0.8.3-1-cdb1
+  - sudo apt-get -y install python-numpy=1:1.6.1-6ubuntu1
+
+  # Install pysal
+  - sudo pip install -I pysal==1.11.2
+
+  - sudo apt-get -y install python-scipy=0.14.0-2-cdb6
+  - sudo apt-get -y --no-install-recommends install python-sklearn-lib=0.14.1-3-cdb2
+  - sudo apt-get -y --no-install-recommends install python-sklearn=0.14.1-3-cdb2
+  - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2
+
+  # Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis)
+  - sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2
+
+  # Install postgres db and build deps
+  - sudo /etc/init.d/postgresql stop # stop travis default instance
+  - sudo apt-get -y remove --purge postgresql-9.1
+  - sudo apt-get -y remove --purge postgresql-9.2
+  - sudo apt-get -y remove --purge postgresql-9.3
+  - sudo apt-get -y remove --purge postgresql-9.4
+  - sudo apt-get -y remove --purge postgis
+  - sudo apt-get -y autoremove
+
+  - sudo apt-get -y install postgresql-9.5=9.5.2-2ubuntu1
+  - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-2ubuntu1
+  - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-2ubuntu1
+  - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
+  - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
+
+  # configure it to accept local connections from postgres
+  - echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" \
+    | sudo tee /etc/postgresql/9.5/main/pg_hba.conf
+  - sudo /etc/init.d/postgresql restart 9.5
+
+install:
+  - sudo make install
+
+script:
+  - make test || { cat src/pg/test/regression.diffs; false; }
+  - ./check-compatibility.sh

README.md

@@ -1,4 +1,4 @@
-# crankshaft
+# crankshaft [![Build Status](https://travis-ci.org/CartoDB/crankshaft.svg?branch=develop)](https://travis-ci.org/CartoDB/crankshaft)
 
 CartoDB Spatial Analysis extension for PostgreSQL.
 

check-compatibility.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+export PGUSER=postgres
+
+DBNAME=crankshaft_compatcheck
+
+function die {
+    echo $1
+    exit -1
+}
+
+# Create fresh DB
+psql -c "CREATE DATABASE $DBNAME;" || die "Could not create DB"
+
+# Hook for cleanup
+function cleanup {
+    psql -c "DROP DATABASE IF EXISTS crankshaft_compatcheck;"
+}
+trap cleanup EXIT
+
+# Deploy previous release
+(cd src/py && sudo make deploy RUN_OPTIONS="--no-deps") || die "Could not deploy python extension"
+(cd src/pg && sudo make deploy) || die " Could not deploy last release"
+psql -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'crankshaft';"
+
+# Install in the fresh DB
+psql $DBNAME <<'EOF'
+-- Install dependencies
+CREATE EXTENSION plpythonu;
+CREATE EXTENSION postgis VERSION '2.2.2';
+
+-- Create role publicuser if it does not exist
+DO
+$$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+$$ LANGUAGE plpgsql;
+
+-- Install the default version
+CREATE EXTENSION crankshaft;
+\dx
+EOF
+
+# Save public function signatures
+psql $DBNAME <<'EOF'
+CREATE TABLE release_function_signatures AS
+  SELECT
+    p.proname as name,
+    pg_catalog.pg_get_function_result(p.oid) as result_type,
+    pg_catalog.pg_get_function_arguments(p.oid) as arguments,
+   CASE
+    WHEN p.proisagg THEN 'agg'
+    WHEN p.proiswindow THEN 'window'
+    WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
+    ELSE 'normal'
+   END as type
+  FROM pg_catalog.pg_proc p
+       LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+  WHERE
+       n.nspname = 'cdb_crankshaft'
+       AND p.proname LIKE 'cdb_%'
+  ORDER BY 1, 2, 4;
+EOF
+
+# Deploy current dev branch
+make clean-dev || die "Could not clean dev files"
+sudo make install || die "Could not deploy current dev branch"
+
+# Check it can be upgraded
+psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version"
+
+# Check against saved public function signatures
+psql $DBNAME <<'EOF'
+CREATE TABLE dev_function_signatures AS
+  SELECT
+    p.proname as name,
+    pg_catalog.pg_get_function_result(p.oid) as result_type,
+    pg_catalog.pg_get_function_arguments(p.oid) as arguments,
+   CASE
+    WHEN p.proisagg THEN 'agg'
+    WHEN p.proiswindow THEN 'window'
+    WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger'
+    ELSE 'normal'
+   END as type
+  FROM pg_catalog.pg_proc p
+       LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+  WHERE
+       n.nspname = 'cdb_crankshaft'
+       AND p.proname LIKE 'cdb_%'
+  ORDER BY 1, 2, 4;
+EOF
+
+echo "Functions in development not in latest release (ok):"
+psql $DBNAME -c "SELECT * FROM dev_function_signatures EXCEPT SELECT * FROM release_function_signatures;"
+
+echo "Functions in latest release not in development (compat issue):"
+psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;"
+
+# Fail if there's a signature mismatch / missing functions
+psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;" | fgrep '(0 rows)' \
+    || die "Function signatures changed"

check-up-to-date-with-master.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# Add remote-master
+git remote add -t master remote-master https://github.com/CartoDB/crankshaft.git
+
+# Fetch master reference
+git fetch --depth=1 remote-master master
+
+# Compare HEAD with master
+# NOTE: travis by default uses --depth=50 so we are actually checking that the tip
+# of the branch is no more than 50 commits away from master as well.
+git rev-list HEAD | grep $(git rev-parse remote-master/master) ||
+    { echo "Your branch is not up to date with latest release";
+      echo "Please update it by running the following:";
+      echo "    git fetch && git merge origin/develop";
+      false; }

doc/07_gravity.md

@@ -0,0 +1,78 @@
+## Gravity Model
+
+Gravity Models are derived from Newton's Law of Gravity and are used to predict the interaction between a group of populated areas (sources) and a specific target among a group of potential targets, in terms of an attraction factor (weight)
+
+**CDB_Gravity** is based on the model defined in *Huff's Law of Shopper attraction (1963)*
+
+### CDB_Gravity(t_id bigint[], t_geom geometry[], t_weight numeric[], s_id bigint[], s_geom geometry[], s_pop numeric[], target bigint, radius integer, minval numeric DEFAULT -10e307)
+
+#### Arguments
+
+| Name | Type | Description |
+|------|------|-------------|
+| t_id     | bigint[]    | Array of targets ID |
+| t_geom   | geometry[]  | Array of targets' geometries |
+| t_weight | numeric[]   | Array of targets's weights |
+| s_id     | bigint[]    | Array of sources ID |
+| s_geom   | geometry[]  | Array of sources' geometries |
+| s_pop    | numeric[]   | Array of sources's population |
+| target   | bigint      | ID of the target under study |
+| radius   | integer     | Radius in meters around the target under study that will be taken into account|
+| minval (optional)   | numeric     | Lowest accepted value of weight, defaults to numeric min_value |
+
+### CDB_Gravity( target_query text, weight_column text, source_query text, pop_column text, target bigint, radius integer, minval numeric DEFAULT -10e307)
+
+#### Arguments
+
+| Name | Type | Description |
+|------|------|-------------|
+| target_query     | text    | Query that defines targets |
+| weight_column   | text  | Column name of weights |
+| source_query     | text    | Query that defines sources |
+| pop_column   | text  | Column name of population |
+| target   | bigint      | cartodb_id of the target under study |
+| radius   | integer     | Radius in meters around the target under study that will be taken into account|
+| minval (optional)   | numeric     | Lowest accepted value of weight, defaults to numeric min_value |
+
+
+### Returns
+
+| Column Name | Type | Description |
+|-------------|------|-------------|
+| the_geom  | geometry | Geometries of the sources within the radius |
+| source_id | bigint  | ID of the source |
+| target_id | bigint  | Target ID from input |
+| dist      | numeric | Distance in meters source to target (if not points, distance between centroids) |
+| h         | numeric | Probability of patronage |
+| hpop      | numeric | Patronaging population |
+
+
+#### Example Usage
+
+```sql
+with t as (
+SELECT
+    array_agg(cartodb_id::bigint) as id,
+    array_agg(the_geom) as g,
+    array_agg(coalesce(gla,0)::numeric) as w
+FROM
+    abel.centros_comerciales_de_madrid
+WHERE not no_cc
+),
+s as (
+SELECT
+    array_agg(cartodb_id::bigint) as id,
+    array_agg(center) as g,
+    array_agg(coalesce(t1_1, 0)::numeric) as p
+FROM
+    sscc_madrid
+)
+select
+    g.the_geom,
+    trunc(g.h,2) as h,
+    round(g.hpop) as hpop,
+    trunc(g.dist/1000,2) as dist_km
+FROM t, s, CDB_Gravity1(t.id, t.g, t.w, s.id, s.g, s.p, newmall_ID, 100000, 5000) g
+```
+
+

src/pg/.gitignore

@@ -4,3 +4,4 @@ results/
 crankshaft--dev.sql
 crankshaft--dev--current.sql
 crankshaft--current--dev.sql
+crankshaft--*--dev.sql

src/pg/Makefile

@@ -10,14 +10,15 @@ include ../../Makefile.global
 # * test runs the tests for the currently generated Development
 #   extension.
 
-DATA         = $(EXTENSION)--dev.sql \
-	             $(EXTENSION)--current--dev.sql \
-	             $(EXTENSION)--dev--current.sql
+DATA = \
+    $(EXTENSION)--dev.sql \
+    $(EXTENSION)--current--dev.sql \
+    $(EXTENSION)--dev--current.sql \
+    $(EXTENSION)--$(RELEASE_VERSION)--dev.sql
 
 SOURCES_DATA_DIR = sql
 SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql)
 
-
 REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g'
 
 $(DATA): $(SOURCES_DATA)

src/pg/sql/04_py_agg.sql

@@ -10,10 +10,22 @@ CREATE OR REPLACE FUNCTION
     END
     $$ LANGUAGE plpgsql;
 
-
-CREATE AGGREGATE CDB_PyAgg(NUMERIC[])(
-    SFUNC = CDB_PyAggS,
-    STYPE = Numeric[],
-    INITCOND = "{}" 
-);
-
+-- Create aggregate if it did not exist
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT *
+        FROM pg_catalog.pg_proc p
+            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+        WHERE n.nspname = 'cdb_crankshaft'
+            AND p.proname = 'cdb_pyagg'
+            AND p.proisagg)
+    THEN
+        CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) (
+            SFUNC = CDB_PyAggS,
+            STYPE = Numeric[],
+            INITCOND = "{}"
+        );
+    END IF;
+END
+$$ LANGUAGE plpgsql;

src/pg/sql/07_gravity.sql

@@ -0,0 +1,115 @@
+CREATE OR REPLACE FUNCTION CDB_Gravity(
+    IN target_query text,
+    IN weight_column text,
+    IN source_query text,
+    IN pop_column text,
+    IN target bigint,
+    IN radius integer,
+    IN minval numeric DEFAULT -10e307
+    )
+RETURNS TABLE(
+    the_geom geometry,
+    source_id bigint,
+    target_id bigint,
+    dist numeric,
+    h numeric,
+    hpop numeric)  AS $$
+DECLARE
+    t_id bigint[];
+    t_geom geometry[];
+    t_weight numeric[];
+    s_id bigint[];
+    s_geom geometry[];
+    s_pop numeric[];
+BEGIN
+    EXECUTE 'WITH foo as('+target_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || weight_column || ') FROM foo' INTO t_id, t_geom, t_weight;
+    EXECUTE 'WITH foo as('+source_query+') SELECT array_agg(cartodb_id), array_agg(the_geom), array_agg(' || pop_column || ') FROM foo' INTO s_id, s_geom, s_pop;
+    RETURN QUERY
+    SELECT g.* FROM t, s, CDB_Gravity(t_id, t_geom, t_weight, s_id, s_geom, s_pop, target, radius, minval) g;
+END;
+$$ language plpgsql;
+
+CREATE OR REPLACE FUNCTION CDB_Gravity(
+    IN t_id bigint[],
+    IN t_geom geometry[],
+    IN t_weight numeric[],
+    IN s_id bigint[],
+    IN s_geom geometry[],
+    IN s_pop numeric[],
+    IN target bigint,
+    IN radius integer,
+    IN minval numeric DEFAULT -10e307
+    )
+RETURNS TABLE(
+    the_geom geometry,
+    source_id bigint,
+    target_id bigint,
+    dist numeric,
+    h numeric,
+    hpop numeric)  AS $$
+DECLARE
+    t_type text;
+    s_type text;
+    t_center geometry[];
+    s_center geometry[];
+BEGIN
+    t_type := GeometryType(t_geom[1]);
+    s_type := GeometryType(s_geom[1]);
+    IF t_type = 'POINT' THEN
+        t_center := t_geom;
+    ELSE
+        WITH tmp as (SELECT unnest(t_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO t_center FROM tmp;
+    END IF;
+    IF s_type = 'POINT' THEN
+        s_center := s_geom;
+    ELSE
+        WITH tmp as (SELECT unnest(s_geom) as g) SELECT array_agg(ST_Centroid(g)) INTO s_center FROM tmp;
+    END IF;
+    RETURN QUERY
+        with target0 as(
+            SELECT unnest(t_center) as tc, unnest(t_weight) as tw, unnest(t_id) as td
+        ),
+        source0 as(
+            SELECT unnest(s_center) as sc, unnest(s_id) as sd, unnest (s_geom) as sg, unnest(s_pop) as sp
+        ),
+        prev0 as(
+            SELECT
+                source0.sg,
+                source0.sd as sourc_id,
+                coalesce(source0.sp,0) as sp,
+                target.td as targ_id,
+                coalesce(target.tw,0) as tw,
+                GREATEST(1.0,ST_Distance(geography(target.tc), geography(source0.sc)))::numeric as distance
+            FROM source0
+            CROSS JOIN LATERAL
+                (
+                SELECT
+                    *
+                FROM target0
+                    WHERE tw > minval
+                    AND ST_DWithin(geography(source0.sc), geography(tc), radius)
+                ) AS target
+        ),
+        deno as(
+            SELECT
+                sourc_id,
+                sum(tw/distance) as h_deno
+            FROM
+                prev0
+            GROUP BY sourc_id
+        )
+        SELECT
+            p.sg as the_geom,
+            p.sourc_id as source_id,
+            p.targ_id as target_id,
+            case when p.distance > 1 then p.distance else 0.0 end as dist,
+            100*(p.tw/p.distance)/d.h_deno as h,
+            p.sp*(p.tw/p.distance)/d.h_deno as hpop
+        FROM
+            prev0 p,
+            deno d
+        WHERE
+            p.targ_id = target AND
+            p.sourc_id = d.sourc_id;
+END;
+$$ language plpgsql;

src/pg/sql/11_kmeans.sql

@@ -41,9 +41,23 @@ BEGIN
 END
 $$ LANGUAGE plpgsql;
 
-CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)(
-    SFUNC = CDB_WeightedMeanS,
-    FINALFUNC = CDB_WeightedMeanF,
-    STYPE = Numeric[],
-    INITCOND = "{0.0,0.0,0.0}" 
-);
+-- Create aggregate if it did not exist
+DO $$
+BEGIN
+    IF NOT EXISTS (
+        SELECT *
+        FROM pg_catalog.pg_proc p
+            LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace
+        WHERE n.nspname = 'cdb_crankshaft'
+            AND p.proname = 'cdb_weightedmean'
+            AND p.proisagg)
+    THEN
+        CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) (
+            SFUNC = CDB_WeightedMeanS,
+            FINALFUNC = CDB_WeightedMeanF,
+            STYPE = Numeric[],
+            INITCOND = "{0.0,0.0,0.0}" 
+        );
+    END IF;
+END
+$$ LANGUAGE plpgsql;

src/pg/test/expected/01_install_test.out

@@ -1,5 +1,18 @@
 -- Install dependencies
 CREATE EXTENSION plpythonu;
-CREATE EXTENSION postgis;
+CREATE EXTENSION postgis VERSION '2.2.2';
+-- Create role publicuser if it does not exist
+DO
+$$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+$$ LANGUAGE plpgsql;
 -- Install the extension
 CREATE EXTENSION crankshaft VERSION 'dev';

src/pg/test/expected/07_gravity_test.out

@@ -0,0 +1,14 @@
+SET client_min_messages TO WARNING;
+\set ECHO none
+                  the_geom                  |            h            |           hpop           |      dist      
+--------------------------------------------+-------------------------+--------------------------+----------------
+ 01010000001361C3D32B650140DD24068195B34440 |  1.51078258369747945249 |  12.08626066957983561994 | 4964.714459152
+ 01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954183620807430 | 688.08116679285345652007 |   99.955141922
+ 0101000000A167B3EA733501401D5A643BDFAF4440 | 63.70532894711274639196 | 382.23197368267647835174 | 2488.330566505
+ 010100000062A1D634EF380140BE9F1A2FDDB44440 | 35.35415870080995954879 | 176.77079350404979774397 | 4359.370460594
+ 010100000052B81E85EB510140355EBA490CB24440 | 33.12290506987740864904 | 132.49162027950963459615 | 3703.664449828
+ 0101000000C286A757CA320140736891ED7CAF4440 | 65.45251754279248087849 | 196.35755262837744263547 | 2512.092358644
+ 01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792471345639225 | 125.67855584942691278449 |  2926.25725244
+ 0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726651871279586 |  53.54905726651871279586 | 3744.515577777
+(8 rows)
+

src/pg/test/sql/01_install_test.sql

@@ -1,6 +1,20 @@
 -- Install dependencies
 CREATE EXTENSION plpythonu;
-CREATE EXTENSION postgis;
+CREATE EXTENSION postgis VERSION '2.2.2';
+
+-- Create role publicuser if it does not exist
+DO
+$$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+$$ LANGUAGE plpgsql;
 
 -- Install the extension
 CREATE EXTENSION crankshaft VERSION 'dev';

src/pg/test/sql/07_gravity_test.sql

@@ -0,0 +1,24 @@
+SET client_min_messages TO WARNING;
+\set ECHO none
+
+WITH t AS (
+    SELECT
+    ARRAY[1,2,3] AS id,
+    ARRAY[7.0,8.0,3.0] AS w,
+    ARRAY[ST_GeomFromText('POINT(2.1744 41.4036)'),ST_GeomFromText('POINT(2.1228 41.3809)'),ST_GeomFromText('POINT(2.1511 41.3742)')] AS g
+),
+s AS (
+    SELECT
+    ARRAY[10,20,30,40,50,60,70,80] AS id,
+    ARRAY[800, 700, 600, 500, 400, 300, 200, 100] AS p,
+    ARRAY[ST_GeomFromText('POINT(2.1744 41.403)'),ST_GeomFromText('POINT(2.1228 41.380)'),ST_GeomFromText('POINT(2.1511 41.374)'),ST_GeomFromText('POINT(2.1528 41.413)'),ST_GeomFromText('POINT(2.165 41.391)'),ST_GeomFromText('POINT(2.1498 41.371)'),ST_GeomFromText('POINT(2.1533 41.368)'),ST_GeomFromText('POINT(2.131386 41.41399)')] AS g
+)
+SELECT
+    g.the_geom,
+    g.h,
+    g.hpop,
+    g.dist
+FROM
+    t,
+    s,
+    cdb_crankshaft.CDB_Gravity(t.id, t.g, t.w, s.id, s.g, s.p, 2, 100000, 3) g;