From cdd2d9e722fe2e93d458bb647216b7871aeea0b6 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 Mar 2016 19:35:02 +0100 Subject: [PATCH 01/81] Directory reorganization and sketch of new versioning procedure --- Makefile | 4 +- README.md | 84 +++++++++- {pg/doc => doc}/02_moran.md | 0 {pg/doc => doc}/03_overlap_sum.md | 0 pg/.gitignore | 3 - pg/Makefile | 33 ---- pg/crankshaft--0.0.1.sql | 148 ------------------ pg/test/0.0.1/expected/01_install_test.out | 6 - python/Makefile | 11 -- src/pg/.gitignore | 6 + src/pg/Makefile | 41 +++++ {pg => src/pg}/README.md | 0 {pg => src/pg}/crankshaft.control | 0 {pg/sql/0.0.1 => src/pg/sql}/00_header.sql | 0 .../0.0.1 => src/pg/sql}/01_random_seeds.sql | 0 {pg/sql/0.0.1 => src/pg/sql}/02_moran.sql | 0 .../0.0.1 => src/pg/sql}/03_overlap_sum.sql | 0 .../0.0.1 => src/pg/sql}/04_dot_density.sql | 0 .../0.0.1 => src/pg/sql}/90_permissions.sql | 0 .../pg/test/expected}/01_install_test.out | 2 +- .../pg/test}/expected/02_moran_test.out | 0 .../pg/test}/expected/03_overlap_sum_test.out | 0 .../pg/test}/expected/04_dot_density_test.out | 0 {pg => src/pg}/test/fixtures/polyg_values.sql | 0 {pg => src/pg}/test/fixtures/ppoints.sql | 0 {pg => src/pg}/test/fixtures/ppoints2.sql | 0 .../pg/test}/sql/01_install_test.sql | 2 +- .../pg/test}/sql/02_moran_test.sql | 0 .../pg/test}/sql/03_overlap_sum_test.sql | 0 .../pg/test}/sql/04_dot_density_test.sql | 0 .../pg/test}/sql/90_permissions.sql | 0 {python => src/py}/.gitignore | 1 + src/py/Makefile | 9 ++ {python => src/py}/README.md | 0 .../py}/crankshaft/crankshaft/__init__.py | 0 .../crankshaft/clustering/__init__.py | 0 .../crankshaft/crankshaft/clustering/moran.py | 0 .../py}/crankshaft/crankshaft/random_seeds.py | 0 {python => src/py}/crankshaft/setup.py | 0 .../py}/crankshaft/test/fixtures/moran.json | 0 .../crankshaft/test/fixtures/neighbors.json | 0 {python => src/py}/crankshaft/test/helper.py | 0 .../py}/crankshaft/test/mock_plpy.py | 0 .../crankshaft/test/test_clustering_moran.py | 0 44 files changed, 142 insertions(+), 208 deletions(-) rename {pg/doc => doc}/02_moran.md (100%) rename {pg/doc => doc}/03_overlap_sum.md (100%) delete mode 100644 pg/.gitignore delete mode 100644 pg/Makefile delete mode 100644 pg/crankshaft--0.0.1.sql delete mode 100644 pg/test/0.0.1/expected/01_install_test.out delete mode 100644 python/Makefile create mode 100644 src/pg/.gitignore create mode 100644 src/pg/Makefile rename {pg => src/pg}/README.md (100%) rename {pg => src/pg}/crankshaft.control (100%) rename {pg/sql/0.0.1 => src/pg/sql}/00_header.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/01_random_seeds.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/02_moran.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/03_overlap_sum.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/04_dot_density.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/90_permissions.sql (100%) rename {pg/test/0.0.1/results => src/pg/test/expected}/01_install_test.out (75%) rename {pg/test/0.0.1 => src/pg/test}/expected/02_moran_test.out (100%) rename {pg/test/0.0.1 => src/pg/test}/expected/03_overlap_sum_test.out (100%) rename {pg/test/0.0.1 => src/pg/test}/expected/04_dot_density_test.out (100%) rename {pg => src/pg}/test/fixtures/polyg_values.sql (100%) rename {pg => src/pg}/test/fixtures/ppoints.sql (100%) rename {pg => src/pg}/test/fixtures/ppoints2.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/01_install_test.sql (75%) rename {pg/test/0.0.1 => src/pg/test}/sql/02_moran_test.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/03_overlap_sum_test.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/04_dot_density_test.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/90_permissions.sql (100%) rename {python => src/py}/.gitignore (54%) create mode 100644 src/py/Makefile rename {python => src/py}/README.md (100%) rename {python => src/py}/crankshaft/crankshaft/__init__.py (100%) rename {python => src/py}/crankshaft/crankshaft/clustering/__init__.py (100%) rename {python => src/py}/crankshaft/crankshaft/clustering/moran.py (100%) rename {python => src/py}/crankshaft/crankshaft/random_seeds.py (100%) rename {python => src/py}/crankshaft/setup.py (100%) rename {python => src/py}/crankshaft/test/fixtures/moran.json (100%) rename {python => src/py}/crankshaft/test/fixtures/neighbors.json (100%) rename {python => src/py}/crankshaft/test/helper.py (100%) rename {python => src/py}/crankshaft/test/mock_plpy.py (100%) rename {python => src/py}/crankshaft/test/test_clustering_moran.py (100%) diff --git a/Makefile b/Makefile index d1d9734..5cae2cb 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -EXT_DIR = pg -PYP_DIR = python +EXT_DIR = src/pg +PYP_DIR = src/py .PHONY: install .PHONY: run_tests diff --git a/README.md b/README.md index 61e8738..5a12c2b 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,87 @@ CartoDB Spatial Analysis extension for PostgreSQL. ## Code organization -* *pg* contains the PostgreSQL extension source code -* *python* Python module +* *doc* documentation +* *src* source code +* - *src/pg* contains the PostgreSQL extension source code +* - *src/py* Python module source code +* *release* reselesed versions ## Requirements -* pip +* pip, virtualenv, PostgreSQL + +# Working Process + +## Development + +Work in `src/pg/sql`, `src/py/crankshaft`; +use topic branch. + +Update local installation with `sudo make install` +(this will update the 'dev' version of the extension in 'src/pg/') + +Run the tests with `PGUSER=postgres make test` + +Update extension in working database with + +* `ALTER EXTENSION crankshaft VERSION TO 'current';` + `ALTER EXTENSION crankshaft VERSION TO 'dev';` + +Note: we keep the current development version install as 'dev' always; +we update through the 'current' alias to allow changing the extension +contents but not the version identifier. This will fail if the +changes involve incompatible function changes such as a different +return type; in that case the offending function (or the whole extension) +should be dropped manually before the update. + +If the extension has not previously been installed in a database +we can: + +Add tests... + +* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` + +Test + +Commit, push, create PR, wait for CI tests, CR, ... + +## Release + +To release current development version +(working directory should be clean in dev branch) + +(process to be gradually automated) + +For backwards compatible changes (no return value, num of arguments, etc. changes...) +new version number increasing either patch level (no new functionality) +or minor level (new functionality) => 'X.Y.Z'. +Update version in src/pg/crankshaft.control +Copy release/crankshaft--current.sql to release/crankshaft--X.Y.Z.sql +Prepare incremental downgrade, upgrade scripts.... + +Python: ... + +Install the new release + +`make install-release` + +Test the new release + +`make test-release` + +Push the release + +Wait for CI tests + +Merge into master + +Deploy: install extension and python to production hosts, +update extension in databases (limited to team users, data observatory, ...) + +Release manager role: ... + +.sql release scripts +commit +tests: staging.... +merge, tag, deploy... diff --git a/pg/doc/02_moran.md b/doc/02_moran.md similarity index 100% rename from pg/doc/02_moran.md rename to doc/02_moran.md diff --git a/pg/doc/03_overlap_sum.md b/doc/03_overlap_sum.md similarity index 100% rename from pg/doc/03_overlap_sum.md rename to doc/03_overlap_sum.md diff --git a/pg/.gitignore b/pg/.gitignore deleted file mode 100644 index 820df46..0000000 --- a/pg/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -regression.diffs -regression.out -results/ diff --git a/pg/Makefile b/pg/Makefile deleted file mode 100644 index 99605f5..0000000 --- a/pg/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# Makefile to generate the extension out of separate sql source files. -# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. - -EXTENSION = crankshaft -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") - -# The new version to be generated from templates -NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql - -# DATA is a special variable used by postgres build infrastructure -# These are the files to be installed in the server shared dir, -# for installation from scratch, upgrades and downgrades. -# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html -DATA = $(NEW_EXTENSION_ARTIFACT) - -SOURCES_DATA_DIR = sql/$(EXTVERSION) -SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) - -# The extension installation artifacts are stored in the base subdirectory -$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) - rm -f $@ - cat $(SOURCES_DATA_DIR)/*.sql >> $@ - -REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) -TEST_DIR = test/$(EXTVERSION) -REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' - -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) - -# This seems to be needed at least for PG 9.3.11 -all: $(DATA) diff --git a/pg/crankshaft--0.0.1.sql b/pg/crankshaft--0.0.1.sql deleted file mode 100644 index 436beea..0000000 --- a/pg/crankshaft--0.0.1.sql +++ /dev/null @@ -1,148 +0,0 @@ ---DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit --- Internal function. --- Set the seeds of the RNGs (Random Number Generators) --- used internally. -CREATE OR REPLACE FUNCTION -_cdb_random_seeds (seed_value INTEGER) RETURNS VOID -AS $$ - from crankshaft import random_seeds - random_seeds.set_random_seeds(seed_value) -$$ LANGUAGE plpythonu; --- Moran's I -CREATE OR REPLACE FUNCTION - cdb_moran_local ( - t TEXT, - attr TEXT, - significance float DEFAULT 0.05, - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn') -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) -AS $$ - from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) -$$ LANGUAGE plpythonu; - --- Moran's I Local Rate -CREATE OR REPLACE FUNCTION - cdb_moran_local_rate(t TEXT, - numerator TEXT, - denominator TEXT, - significance FLOAT DEFAULT 0.05, - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn') -RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) -AS $$ - from crankshaft.clustering import moran_local_rate - # TODO: use named parameters or a dictionary - return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) -$$ LANGUAGE plpythonu; --- Function by Stuart Lynn for a simple interpolation of a value --- from a polygon table over an arbitrary polygon --- (weighted by the area proportion overlapped) --- Aereal weighting is a very simple form of aereal interpolation. --- --- Parameters: --- * geom a Polygon geometry which defines the area where a value will be --- estimated as the area-weighted sum of a given table/column --- * target_table_name table name of the table that provides the values --- * target_column column name of the column that provides the values --- * schema_name optional parameter to defina the schema the target table --- belongs to, which is necessary if its not in the search_path. --- Note that target_table_name should never include the schema in it. --- Return value: --- Aereal-weighted interpolation of the column values over the geometry -CREATE OR REPLACE -FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) - RETURNS numeric AS -$$ -DECLARE - result numeric; - qualified_name text; -BEGIN - IF schema_name IS NULL THEN - qualified_name := Format('%I', target_table_name); - ELSE - qualified_name := Format('%I.%s', schema_name, target_table_name); - END IF; - EXECUTE Format(' - SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) - FROM %s AS a - WHERE $1 && a.the_geom - ', target_column, qualified_name) - USING geom - INTO result; - RETURN result; -END; -$$ LANGUAGE plpgsql; --- --- Creates N points randomly distributed arround the polygon --- --- @param g - the geometry to be turned in to points --- --- @param no_points - the number of points to generate --- --- @params max_iter_per_point - the function generates points in the polygon's bounding box --- and discards points which don't lie in the polygon. max_iter_per_point specifies how many --- misses per point the funciton accepts before giving up. --- --- Returns: Multipoint with the requested points -CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) -RETURNS GEOMETRY AS $$ -DECLARE - extent GEOMETRY; - test_point Geometry; - width NUMERIC; - height NUMERIC; - x0 NUMERIC; - y0 NUMERIC; - xp NUMERIC; - yp NUMERIC; - no_left INTEGER; - remaining_iterations INTEGER; - points GEOMETRY[]; - bbox_line GEOMETRY; - intersection_line GEOMETRY; -BEGIN - extent := ST_Envelope(geom); - width := ST_XMax(extent) - ST_XMIN(extent); - height := ST_YMax(extent) - ST_YMIN(extent); - x0 := ST_XMin(extent); - y0 := ST_YMin(extent); - no_left := no_points; - - LOOP - if(no_left=0) THEN - EXIT; - END IF; - yp = y0 + height*random(); - bbox_line = ST_MakeLine( - ST_SetSRID(ST_MakePoint(yp, x0),4326), - ST_SetSRID(ST_MakePoint(yp, x0+width),4326) - ); - intersection_line = ST_Intersection(bbox_line,geom); - test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); - points := points || test_point; - no_left = no_left - 1 ; - END LOOP; - RETURN ST_Collect(points); -END; -$$ -LANGUAGE plpgsql VOLATILE; --- Make sure by default there are no permissions for publicuser --- NOTE: this happens at extension creation time, as part of an implicit transaction. --- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; - --- Grant permissions on the schema to publicuser (but just the schema) -GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; - --- Revoke execute permissions on all functions in the schema by default --- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/pg/test/0.0.1/expected/01_install_test.out b/pg/test/0.0.1/expected/01_install_test.out deleted file mode 100644 index c14537c..0000000 --- a/pg/test/0.0.1/expected/01_install_test.out +++ /dev/null @@ -1,6 +0,0 @@ --- Install dependencies -CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis; -CREATE EXTENSION cartodb; --- Install the extension -CREATE EXTENSION crankshaft; diff --git a/python/Makefile b/python/Makefile deleted file mode 100644 index 07b41dd..0000000 --- a/python/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -# Install the package (needs root privileges) -install: - pip install ./crankshaft --upgrade - -# Test from source code -test: - (cd crankshaft && nosetests test/) - -# Test currently installed package -testinstalled: - nosetests crankshaft/test/ diff --git a/src/pg/.gitignore b/src/pg/.gitignore new file mode 100644 index 0000000..b58a014 --- /dev/null +++ b/src/pg/.gitignore @@ -0,0 +1,6 @@ +regression.diffs +regression.out +results/ +crankshaft--dev.sql +crankshaft--dev--current.sql +crankshaft--current--dev.sql diff --git a/src/pg/Makefile b/src/pg/Makefile new file mode 100644 index 0000000..4ff713d --- /dev/null +++ b/src/pg/Makefile @@ -0,0 +1,41 @@ +# Generation of a new development version 'dev' (with an alias 'current' for +# updating easily by upgrading to 'current', then 'dev') + +# sudo make install -- generate the 'dev' version from current source +# and make it available to PostgreSQL +# PGUSER=postgres make installcheck -- test the 'dev' extension + +EXTENSION = crankshaft + +DATA = $(EXTENSION)--dev.sql \ + $(EXTENSION)--current--dev.sql \ + $(EXTENSION)--dev--current.sql + +SOURCES_DATA_DIR = sql +SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) + +$(DATA): $(SOURCES_DATA) + cat $(SOURCES_DATA_DIR)/*.sql > $@ + +TEST_DIR = test +REGRESS = $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +# This seems to be needed at least for PG 9.3.11 +all: $(DATA) + +# WIP: goals for releasing the extension... + +EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") + +../release/$(EXTENSION).control: $(EXTENSION).control + cp $< $@ + +release: ../release/$(EXTENSION).control + cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql + # pending: create upgrade/downgrade scripts, + # commit, push, tag.... diff --git a/pg/README.md b/src/pg/README.md similarity index 100% rename from pg/README.md rename to src/pg/README.md diff --git a/pg/crankshaft.control b/src/pg/crankshaft.control similarity index 100% rename from pg/crankshaft.control rename to src/pg/crankshaft.control diff --git a/pg/sql/0.0.1/00_header.sql b/src/pg/sql/00_header.sql similarity index 100% rename from pg/sql/0.0.1/00_header.sql rename to src/pg/sql/00_header.sql diff --git a/pg/sql/0.0.1/01_random_seeds.sql b/src/pg/sql/01_random_seeds.sql similarity index 100% rename from pg/sql/0.0.1/01_random_seeds.sql rename to src/pg/sql/01_random_seeds.sql diff --git a/pg/sql/0.0.1/02_moran.sql b/src/pg/sql/02_moran.sql similarity index 100% rename from pg/sql/0.0.1/02_moran.sql rename to src/pg/sql/02_moran.sql diff --git a/pg/sql/0.0.1/03_overlap_sum.sql b/src/pg/sql/03_overlap_sum.sql similarity index 100% rename from pg/sql/0.0.1/03_overlap_sum.sql rename to src/pg/sql/03_overlap_sum.sql diff --git a/pg/sql/0.0.1/04_dot_density.sql b/src/pg/sql/04_dot_density.sql similarity index 100% rename from pg/sql/0.0.1/04_dot_density.sql rename to src/pg/sql/04_dot_density.sql diff --git a/pg/sql/0.0.1/90_permissions.sql b/src/pg/sql/90_permissions.sql similarity index 100% rename from pg/sql/0.0.1/90_permissions.sql rename to src/pg/sql/90_permissions.sql diff --git a/pg/test/0.0.1/results/01_install_test.out b/src/pg/test/expected/01_install_test.out similarity index 75% rename from pg/test/0.0.1/results/01_install_test.out rename to src/pg/test/expected/01_install_test.out index c14537c..e40d267 100644 --- a/pg/test/0.0.1/results/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -3,4 +3,4 @@ CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; CREATE EXTENSION cartodb; -- Install the extension -CREATE EXTENSION crankshaft; +CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/pg/test/0.0.1/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out similarity index 100% rename from pg/test/0.0.1/expected/02_moran_test.out rename to src/pg/test/expected/02_moran_test.out diff --git a/pg/test/0.0.1/expected/03_overlap_sum_test.out b/src/pg/test/expected/03_overlap_sum_test.out similarity index 100% rename from pg/test/0.0.1/expected/03_overlap_sum_test.out rename to src/pg/test/expected/03_overlap_sum_test.out diff --git a/pg/test/0.0.1/expected/04_dot_density_test.out b/src/pg/test/expected/04_dot_density_test.out similarity index 100% rename from pg/test/0.0.1/expected/04_dot_density_test.out rename to src/pg/test/expected/04_dot_density_test.out diff --git a/pg/test/fixtures/polyg_values.sql b/src/pg/test/fixtures/polyg_values.sql similarity index 100% rename from pg/test/fixtures/polyg_values.sql rename to src/pg/test/fixtures/polyg_values.sql diff --git a/pg/test/fixtures/ppoints.sql b/src/pg/test/fixtures/ppoints.sql similarity index 100% rename from pg/test/fixtures/ppoints.sql rename to src/pg/test/fixtures/ppoints.sql diff --git a/pg/test/fixtures/ppoints2.sql b/src/pg/test/fixtures/ppoints2.sql similarity index 100% rename from pg/test/fixtures/ppoints2.sql rename to src/pg/test/fixtures/ppoints2.sql diff --git a/pg/test/0.0.1/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql similarity index 75% rename from pg/test/0.0.1/sql/01_install_test.sql rename to src/pg/test/sql/01_install_test.sql index 54117e5..fc3ea80 100644 --- a/pg/test/0.0.1/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -4,4 +4,4 @@ CREATE EXTENSION postgis; CREATE EXTENSION cartodb; -- Install the extension -CREATE EXTENSION crankshaft; +CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/pg/test/0.0.1/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql similarity index 100% rename from pg/test/0.0.1/sql/02_moran_test.sql rename to src/pg/test/sql/02_moran_test.sql diff --git a/pg/test/0.0.1/sql/03_overlap_sum_test.sql b/src/pg/test/sql/03_overlap_sum_test.sql similarity index 100% rename from pg/test/0.0.1/sql/03_overlap_sum_test.sql rename to src/pg/test/sql/03_overlap_sum_test.sql diff --git a/pg/test/0.0.1/sql/04_dot_density_test.sql b/src/pg/test/sql/04_dot_density_test.sql similarity index 100% rename from pg/test/0.0.1/sql/04_dot_density_test.sql rename to src/pg/test/sql/04_dot_density_test.sql diff --git a/pg/test/0.0.1/sql/90_permissions.sql b/src/pg/test/sql/90_permissions.sql similarity index 100% rename from pg/test/0.0.1/sql/90_permissions.sql rename to src/pg/test/sql/90_permissions.sql diff --git a/python/.gitignore b/src/py/.gitignore similarity index 54% rename from python/.gitignore rename to src/py/.gitignore index 0d20b64..fb50fe5 100644 --- a/python/.gitignore +++ b/src/py/.gitignore @@ -1 +1,2 @@ *.pyc +dev/ diff --git a/src/py/Makefile b/src/py/Makefile new file mode 100644 index 0000000..27d1446 --- /dev/null +++ b/src/py/Makefile @@ -0,0 +1,9 @@ +# Install the package locally for development +install: + virtualenv dev + ./dev/bin/pip install ./crankshaft --upgrade + ./dev/bin/pip install nose + +# Test develpment install +testinstalled: + ./dev/bin/nosetests crankshaft/test/ diff --git a/python/README.md b/src/py/README.md similarity index 100% rename from python/README.md rename to src/py/README.md diff --git a/python/crankshaft/crankshaft/__init__.py b/src/py/crankshaft/crankshaft/__init__.py similarity index 100% rename from python/crankshaft/crankshaft/__init__.py rename to src/py/crankshaft/crankshaft/__init__.py diff --git a/python/crankshaft/crankshaft/clustering/__init__.py b/src/py/crankshaft/crankshaft/clustering/__init__.py similarity index 100% rename from python/crankshaft/crankshaft/clustering/__init__.py rename to src/py/crankshaft/crankshaft/clustering/__init__.py diff --git a/python/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py similarity index 100% rename from python/crankshaft/crankshaft/clustering/moran.py rename to src/py/crankshaft/crankshaft/clustering/moran.py diff --git a/python/crankshaft/crankshaft/random_seeds.py b/src/py/crankshaft/crankshaft/random_seeds.py similarity index 100% rename from python/crankshaft/crankshaft/random_seeds.py rename to src/py/crankshaft/crankshaft/random_seeds.py diff --git a/python/crankshaft/setup.py b/src/py/crankshaft/setup.py similarity index 100% rename from python/crankshaft/setup.py rename to src/py/crankshaft/setup.py diff --git a/python/crankshaft/test/fixtures/moran.json b/src/py/crankshaft/test/fixtures/moran.json similarity index 100% rename from python/crankshaft/test/fixtures/moran.json rename to src/py/crankshaft/test/fixtures/moran.json diff --git a/python/crankshaft/test/fixtures/neighbors.json b/src/py/crankshaft/test/fixtures/neighbors.json similarity index 100% rename from python/crankshaft/test/fixtures/neighbors.json rename to src/py/crankshaft/test/fixtures/neighbors.json diff --git a/python/crankshaft/test/helper.py b/src/py/crankshaft/test/helper.py similarity index 100% rename from python/crankshaft/test/helper.py rename to src/py/crankshaft/test/helper.py diff --git a/python/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py similarity index 100% rename from python/crankshaft/test/mock_plpy.py rename to src/py/crankshaft/test/mock_plpy.py diff --git a/python/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py similarity index 100% rename from python/crankshaft/test/test_clustering_moran.py rename to src/py/crankshaft/test/test_clustering_moran.py From 8e972128eb90105dfd4cdabda5b0bb2c2e3519e4 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 Mar 2016 15:00:50 +0100 Subject: [PATCH 02/81] Modify sql code to user the python virtualenv --- src/pg/sql/01_py.sql | 18 ++++++++++++++++++ ...01_random_seeds.sql => 02_random_seeds.sql} | 1 + src/pg/sql/{02_moran.sql => 10_moran.sql} | 2 ++ .../{03_overlap_sum.sql => 20_overlap_sum.sql} | 0 .../{04_dot_density.sql => 30_dot_density.sql} | 0 5 files changed, 21 insertions(+) create mode 100644 src/pg/sql/01_py.sql rename src/pg/sql/{01_random_seeds.sql => 02_random_seeds.sql} (80%) rename src/pg/sql/{02_moran.sql => 10_moran.sql} (89%) rename src/pg/sql/{03_overlap_sum.sql => 20_overlap_sum.sql} (100%) rename src/pg/sql/{04_dot_density.sql => 30_dot_density.sql} (100%) diff --git a/src/pg/sql/01_py.sql b/src/pg/sql/01_py.sql new file mode 100644 index 0000000..f1eaa31 --- /dev/null +++ b/src/pg/sql/01_py.sql @@ -0,0 +1,18 @@ +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + # activate virtualenv + # TODO: parameterize with environment variables or something + venv_path = '/home/ubuntu/crankshaft/src/py/dev' + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), + dict(__file__=activate_path)) + + # import something from virtualenv + # from crankshaft import random_seeds + + # do some stuff + # random_seeds.set_random_seeds(123) + # plpy.notice('here we are') +$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/01_random_seeds.sql b/src/pg/sql/02_random_seeds.sql similarity index 80% rename from src/pg/sql/01_random_seeds.sql rename to src/pg/sql/02_random_seeds.sql index 2b62be3..9a0cca6 100644 --- a/src/pg/sql/01_random_seeds.sql +++ b/src/pg/sql/02_random_seeds.sql @@ -4,6 +4,7 @@ CREATE OR REPLACE FUNCTION _cdb_random_seeds (seed_value INTEGER) RETURNS VOID AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft import random_seeds random_seeds.set_random_seeds(seed_value) $$ LANGUAGE plpythonu; diff --git a/src/pg/sql/02_moran.sql b/src/pg/sql/10_moran.sql similarity index 89% rename from src/pg/sql/02_moran.sql rename to src/pg/sql/10_moran.sql index d061b45..49c70c2 100644 --- a/src/pg/sql/02_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -11,6 +11,7 @@ CREATE OR REPLACE FUNCTION w_type TEXT DEFAULT 'knn') RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) @@ -29,6 +30,7 @@ CREATE OR REPLACE FUNCTION w_type TEXT DEFAULT 'knn') RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate # TODO: use named parameters or a dictionary return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) diff --git a/src/pg/sql/03_overlap_sum.sql b/src/pg/sql/20_overlap_sum.sql similarity index 100% rename from src/pg/sql/03_overlap_sum.sql rename to src/pg/sql/20_overlap_sum.sql diff --git a/src/pg/sql/04_dot_density.sql b/src/pg/sql/30_dot_density.sql similarity index 100% rename from src/pg/sql/04_dot_density.sql rename to src/pg/sql/30_dot_density.sql From 1810f02242c494a5ecb7f1ab726e732cbecf65cf Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 Mar 2016 15:03:17 +0100 Subject: [PATCH 03/81] Use SciPy from system package python-scipy --- README.md | 1 + src/py/Makefile | 6 +++--- src/py/crankshaft/setup.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5a12c2b..c8869e7 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. ## Requirements * pip, virtualenv, PostgreSQL +* python-scipy system package # Working Process diff --git a/src/py/Makefile b/src/py/Makefile index 27d1446..16be269 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,8 +1,8 @@ # Install the package locally for development install: - virtualenv dev - ./dev/bin/pip install ./crankshaft --upgrade - ./dev/bin/pip install nose + virtualenv --system-site-packages dev + ./dev/bin/pip install -I ./crankshaft + ./dev/bin/pip install -I nose # Test develpment install testinstalled: diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index c0f8c50..aaa76a6 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.11.0','numpy==1.6.1','scipy==0.17.0'], + install_requires=['pysal==1.9.1'], - requires=['pysal', 'numpy'], + requires=['pysal', 'numpy' ], test_suite='test' ) From 0056f411b50be904c57c9d48ffd8a4ac886f5bc8 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 Mar 2016 19:04:21 +0100 Subject: [PATCH 04/81] Set the path to virtualenvs in the Makefile Also, version the virtualenv --- src/pg/Makefile | 11 ++++++++- src/pg/sql/01_py.sql | 18 --------------- src/pg/sql/01_version.sql | 12 ++++++++++ src/pg/sql/02_py.sql | 23 +++++++++++++++++++ ...2_random_seeds.sql => 03_random_seeds.sql} | 0 5 files changed, 45 insertions(+), 19 deletions(-) delete mode 100644 src/pg/sql/01_py.sql create mode 100644 src/pg/sql/01_version.sql create mode 100644 src/pg/sql/02_py.sql rename src/pg/sql/{02_random_seeds.sql => 03_random_seeds.sql} (100%) diff --git a/src/pg/Makefile b/src/pg/Makefile index 4ff713d..ded7121 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -5,6 +5,8 @@ # and make it available to PostgreSQL # PGUSER=postgres make installcheck -- test the 'dev' extension +SED = sed + EXTENSION = crankshaft DATA = $(EXTENSION)--dev.sql \ @@ -14,8 +16,14 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) +VIRTUALENV_PATH = $(realpath ../py/) +ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) + +REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ + -e 's/@@VIRTUALENV_PATH@@/$(ESC_VIRVIRTUALENV_PATH)/g' + $(DATA): $(SOURCES_DATA) - cat $(SOURCES_DATA_DIR)/*.sql > $@ + $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@ TEST_DIR = test REGRESS = $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))) @@ -28,6 +36,7 @@ include $(PGXS) # This seems to be needed at least for PG 9.3.11 all: $(DATA) + # WIP: goals for releasing the extension... EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") diff --git a/src/pg/sql/01_py.sql b/src/pg/sql/01_py.sql deleted file mode 100644 index f1eaa31..0000000 --- a/src/pg/sql/01_py.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Use the crankshaft python module -CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() -RETURNS VOID -AS $$ - # activate virtualenv - # TODO: parameterize with environment variables or something - venv_path = '/home/ubuntu/crankshaft/src/py/dev' - activate_path = venv_path + '/bin/activate_this.py' - exec(open(activate_path).read(), - dict(__file__=activate_path)) - - # import something from virtualenv - # from crankshaft import random_seeds - - # do some stuff - # random_seeds.set_random_seeds(123) - # plpy.notice('here we are') -$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/01_version.sql b/src/pg/sql/01_version.sql new file mode 100644 index 0000000..90e6123 --- /dev/null +++ b/src/pg/sql/01_version.sql @@ -0,0 +1,12 @@ +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '@@VERSION@@'::text; +$$ language 'sql' IMMUTABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' IMMUTABLE STRICT; diff --git a/src/pg/sql/02_py.sql b/src/pg/sql/02_py.sql new file mode 100644 index 0000000..a233a0a --- /dev/null +++ b/src/pg/sql/02_py.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() +RETURNS text +AS $$ + BEGIN + -- RETURN '/opt/virtualenvs/crankshaft'; + RETURN '@@VIRTUALENV_PATH@@'; + END; +$$ language plpgsql IMMUTABLE STRICT; + +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + import os + # plpy.notice('%',str(os.environ)) + # activate virtualenv + crankshaft_version = plpy.execute('SELECT cdb_crankshaft.cdb_crankshaft_internal_version()')[0]['cdb_crankshaft_internal_version'] + base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] + default_venv_path = os.path.join(base_path, crankshaft_version) + venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), dict(__file__=activate_path)) +$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/02_random_seeds.sql b/src/pg/sql/03_random_seeds.sql similarity index 100% rename from src/pg/sql/02_random_seeds.sql rename to src/pg/sql/03_random_seeds.sql From b754ffe42a2720fd67ace6ab8ceea0784423e4b6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 9 Mar 2016 18:51:04 +0100 Subject: [PATCH 05/81] Add info about python dependencies --- src/py/README.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/py/README.md b/src/py/README.md index f342bf2..03926c3 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -7,3 +7,93 @@ cd crankshaft nosetests test/ ``` + +## Notes about python dependencies +* This extension is targeted at production databases. Therefore certain restrictions must be assumed about the production environment vs other experimental environments. +* We're using `pip` and `virtualenv` to generate a suitable isolated environment for python code that has all the dependencies +* Every dependency should be: + - Added to the `setup.py` file + - Installed through it + - Tested, when they have a test suite. + - Fixed in the `requirements.txt` +* At present we use Python version 2.7.3 + +--- + +### Sample session with virtualenv +#### Create and use a virtual env + + # Create the virtual environment for python + $ virtualenv myenv + + # Activate the virtualenv + $ source myenv/bin/activate + + # Install all the requirements + # expect this to take a while, as it will trigger a few compilations + (myenv) $ pip install -r requirements.txt + + # Add a new pip to the party + (myenv) $ pip install pandas + +#### Test the libraries with that virtual env +##### Test numpy library dependency: + + import numpy + numpy.test('full') + +output: +``` +====================================================================== +ERROR: test_multiarray.TestNewBufferProtocol.test_relaxed_strides +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/nose/case.py", line 197, in runTest + self.test(*self.arg) + File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/numpy/core/tests/test_multiarray.py", line 5366, in test_relaxed_strides + fd.write(c.data) +TypeError: 'buffer' does not have the buffer interface + +---------------------------------------------------------------------- +Ran 6153 tests in 84.561s + +FAILED (KNOWNFAIL=3, SKIP=5, errors=1) +Out[2]: +``` + +NOTE: this is expected to fail with Python 2.7.3, which is the version embedded in our postgresql installation + + +##### Run scipy tests + + import scipy + scipy.test('full') + +Output: +``` +Ran 21562 tests in 321.610s + +OK (KNOWNFAIL=130, SKIP=1840) +Out[2]: +``` +Ok, this looks good... + +##### Testing pysal +See [http://pysal.readthedocs.org/en/latest/developers/testing.html] + + import pysal + import nose + nose.runmodule('pysal') + +``` +Ran 537 tests in 42.182s + +FAILED (errors=48, failures=17) +An exception has occurred, use %tb to see the full traceback. +``` + +This doesn't look good... Taking a deeper look at the failures, many have the `IOError: [Errno 2] No such file or directory: 'streets.shp'` + +In the source code, there's the following [config](https://github.com/pysal/pysal/blob/master/setup.cfg) that seems to be missing in the pip package. By copying it to `lib/python2.7/site-packages` within the environment, it goes down to 17 failures. + +The remaining failures don't look good. I see two types: precision calculation errors and arrays/matrices missing 1 element when comparing... TODO: FIX this From 0206cc6c4457e6ca91f7a2cee1dee89e9df8b40f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 10 Mar 2016 19:13:46 +0100 Subject: [PATCH 06/81] Update documentation --- CONTRIBUTING.md | 84 ---------------------------------- README.md | 77 ++++++++++++++++++------------- src/pg/Makefile | 2 - src/py/README.md | 117 ++++++++++++++++++++++++++++++----------------- 4 files changed, 118 insertions(+), 162 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 63670ca..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,84 +0,0 @@ -# Contributing guide - -## How to add new functions - -Try to put as little logic in the SQL extension as possible and -just use it as a wrapper to the Python module functionality. - -Once a function is defined it should never change its signature in subsequent -versions. To change a function's signature a new function with a different -name must be created. - -### Version numbers - -The version of both the SQL extension and the Python package shall -follow the [Semantic Versioning 2.0](http://semver.org/) guidelines: - -* When backwards incompatibility is introduced the major number is incremented -* When functionally is added (in a backwards-compatible manner) the minor number - is incremented -* When only fixes are introduced (backwards-compatible) the patch number is - incremented - -### Python Package - -... - -### SQL Extension - -* Generate a **new subfolder version** for `sql` and `test` folders to define - the new functions and tests - - Use symlinks to avoid file duplication between versions that don't update them - - Add new files or modify copies of the old files to add new functions or - modify existing functions (remember to rename a function if the signature - changes) - - Add or modify the corresponding documentation files in the `doc` folder. - Since we expect to have highly technical functions here, an extense - background explanation would be of great help to users of this extension. - - Create tests for the new functions/behaviour - -* Generate the **upgrade and downgrade files** for the extension - -* Update the control file and the Makefile to generate the complete SQL - file for the new created version. After running `make` a new - file `crankshaft--X.Y.Z.sql` will be created for the current version. - Additional files for migrating to/from the previous version A.B.Z should be - created: - - `crankshaft--X.Y.Z--A.B.C.sql` - - `crankshaft--A.B.C--X.Y.Z.sql` - All these new files must be added to git and pushed. - -* Update the public docs! ;-) - -## Conventions - -# SQL - -Use snake case (i.e. `snake_case` and not `CamelCase`) for all -functions. Prefix functions intended for public use with `cdb_` -and private functions (to be used only internally inside -the extension) with `_cdb_`. - -# Python - -... - -## Testing - -Running just the Python tests: - -``` -(cd python && make test) -``` - -Installing the Extension and running just the PostgreSQL tests: - -``` -(cd pg && sudo make install && PGUSER=postgres make installcheck) -``` - -Installing and testing everything: - -``` -sudo make install && PGUSER=postgres make testinstalled -``` diff --git a/README.md b/README.md index c8869e7..c9dd529 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,32 @@ CartoDB Spatial Analysis extension for PostgreSQL. * *src* source code * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code -* *release* reselesed versions +* *release* reseleased versions ## Requirements * pip, virtualenv, PostgreSQL -* python-scipy system package +* python-scipy system package (see src/py/README.md) # Working Process ## Development Work in `src/pg/sql`, `src/py/crankshaft`; -use topic branch. +use a topic branch. See src/py/README.md +for the procedure to work with the Python local environment. + +Take into account: + +* Always remember to add tests for any new functionality + documentation. +* Add or modify the corresponding documentation files in the `doc` folder. + Since we expect to have highly technical functions here, an extense + background explanation would be of great help to users of this extension. +* Convention: Use snake case (i.e. `snake_case` and not `CamelCase`) for all + functions. Prefix functions intended for public use with `cdb_` + and private functions (to be used only internally inside + the extension) with `_cdb_`. Update local installation with `sudo make install` (this will update the 'dev' version of the extension in 'src/pg/') @@ -42,50 +55,48 @@ should be dropped manually before the update. If the extension has not previously been installed in a database we can: -Add tests... - * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` -Test +Once the tests are succeeding a new Pull-Request can be created. +CI-tests must be checked to be successfull. + +Before merging a topic branch peer code reviewing of the code is a must. -Commit, push, create PR, wait for CI tests, CR, ... ## Release -To release current development version -(working directory should be clean in dev branch) +The release process of a new version of the extension +shall by performed by the designated *Release Manager*. -(process to be gradually automated) +Note that we expect to gradually automate this process. -For backwards compatible changes (no return value, num of arguments, etc. changes...) -new version number increasing either patch level (no new functionality) -or minor level (new functionality) => 'X.Y.Z'. -Update version in src/pg/crankshaft.control -Copy release/crankshaft--current.sql to release/crankshaft--X.Y.Z.sql -Prepare incremental downgrade, upgrade scripts.... +Having checkout the topic branch of the PR to be released: -Python: ... +The version number in `pg/cranckshaft.control` must first be updated. +To do so [Semantic Versioning 2.0](http://semver.org/) is in order. -Install the new release +We now will explain the process for the case of backwards-compatible +releases (updating the minor or patch version numbers). -`make install-release` +TODO: document the complex case of major releases. -Test the new release +The next command must be executed to produce the main installation +script for the new release, `release/cranckshaft--X.Y.Z.sql`. -`make test-release` +``` +make release +``` -Push the release +Then, the release manager shall produce upgrade and downgrade scripts +to migrate to/from the previous release. In the case of minor/patch +releases this simply consist in extracting the functions that have changed +and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` +file. -Wait for CI tests +TODO: configure the local enviroment to be used by the release; +currently should be directory `src/py/X.Y.Z`, but this must be fixed; +a possibility to explore is to use the `cdb_conf` table. -Merge into master +TODO: testing procedure for the new release -Deploy: install extension and python to production hosts, -update extension in databases (limited to team users, data observatory, ...) - -Release manager role: ... - -.sql release scripts -commit -tests: staging.... -merge, tag, deploy... +TODO: push, merge, tag, deploy procedures. diff --git a/src/pg/Makefile b/src/pg/Makefile index ded7121..ed65eba 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -46,5 +46,3 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/def release: ../release/$(EXTENSION).control cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql - # pending: create upgrade/downgrade scripts, - # commit, push, tag.... diff --git a/src/py/README.md b/src/py/README.md index 03926c3..d55b7d7 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -20,80 +20,111 @@ nosetests test/ --- -### Sample session with virtualenv +We have two possible approaches being considered as to how manage +the Python virtual environment: using a pure virtual enviroment +or combine it with some system packages that include depencencies +for the *hard-to-compile* packages (and pin them in somewhat old versions). + +### Alternative A: pure virtual environment + +In this case we will install all the packages needed in the +virtual environment. +This will involve, specially for the numerical packages compiling +and linking code that uses a number of third party libraries, +and requires having theses depencencies solved for the production +environments. + #### Create and use a virtual env +We'll use a virtual enviroment directory `dev` +under the `src/pg` directory. + # Create the virtual environment for python - $ virtualenv myenv + $ virtualenv dev # Activate the virtualenv - $ source myenv/bin/activate + $ source dev/bin/activate # Install all the requirements # expect this to take a while, as it will trigger a few compilations - (myenv) $ pip install -r requirements.txt + (dev) $ pip install -r requirements.txt # Add a new pip to the party - (myenv) $ pip install pandas + (dev) $ pip install pandas #### Test the libraries with that virtual env + ##### Test numpy library dependency: import numpy numpy.test('full') -output: -``` -====================================================================== -ERROR: test_multiarray.TestNewBufferProtocol.test_relaxed_strides ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/nose/case.py", line 197, in runTest - self.test(*self.arg) - File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/numpy/core/tests/test_multiarray.py", line 5366, in test_relaxed_strides - fd.write(c.data) -TypeError: 'buffer' does not have the buffer interface - ----------------------------------------------------------------------- -Ran 6153 tests in 84.561s - -FAILED (KNOWNFAIL=3, SKIP=5, errors=1) -Out[2]: -``` - -NOTE: this is expected to fail with Python 2.7.3, which is the version embedded in our postgresql installation - - ##### Run scipy tests import scipy scipy.test('full') -Output: -``` -Ran 21562 tests in 321.610s - -OK (KNOWNFAIL=130, SKIP=1840) -Out[2]: -``` -Ok, this looks good... - ##### Testing pysal + See [http://pysal.readthedocs.org/en/latest/developers/testing.html] +This will require putting this into `dev/lib/python2.7/site-packages/setup.cfg`: + +``` +[nosetests] +ignore-files=collection +exclude-dir=pysal/contrib + +[wheel] +universal=1 +``` + +And copying some files before executing the tests: +(we'll use a temporary directory from where the tests will be executed because +some tests expect some files in the current directory). Next must be executed +from + +``` +cp dev/lib/python2.7/site-packages/pysal/examples/geodanet/* dev/local/lib/python2.7/site-packages/pysal/examples +mkdir -p test_tmp && cd test_tmp && cp ../dev/lib/python2.7/site-packages/pysal/examples/geodanet/* ./ +``` + +Then, execute the tests with: + import pysal import nose nose.runmodule('pysal') -``` -Ran 537 tests in 42.182s -FAILED (errors=48, failures=17) -An exception has occurred, use %tb to see the full traceback. +### Alternative B: using some packaged modules + +This option avoids troublesome compilations/linkings, at the cost +of freezing some module versions as available in system packages, +namely numpy 1.6.1 and scipy 0.9.0. (in turn, this implies +the most recent version of PySAL we can use is 1.9.1) + + +TODO: to use this alternative the python-scipy package must be +installed (this will have to be included in server provisioning) + +``` +apt-get install -y python-scipy ``` -This doesn't look good... Taking a deeper look at the failures, many have the `IOError: [Errno 2] No such file or directory: 'streets.shp'` +#### Create and use a virtual env -In the source code, there's the following [config](https://github.com/pysal/pysal/blob/master/setup.cfg) that seems to be missing in the pip package. By copying it to `lib/python2.7/site-packages` within the environment, it goes down to 17 failures. +We'll use a `dev` enviroment as before, but will configure it to +use also system modules. -The remaining failures don't look good. I see two types: precision calculation errors and arrays/matrices missing 1 element when comparing... TODO: FIX this + + # Create the virtual environment for python + $ virtualenv --system-site-packages dev + + # Activate the virtualenv + $ source dev/bin/activate + + # Install all the requirements + # expect this to take a while, as it will trigger a few compilations + (dev) $ pip install -I ./crankshaft + +Then we can proceed to testing as in Alternative A. From e801c9cb60cdb2774238da313bdbfd6c58bb21c7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 15 Mar 2016 18:48:46 +0100 Subject: [PATCH 07/81] Release tasks using release-specific virtual environments Refine the development process and define the procedure for releasing new versions. --- .gitignore | 1 + Makefile | 26 ++++++++++-- NEWS.md | 3 ++ README.md | 87 +++++++++++++++++++++++++++++--------- TODO.md | 9 ---- release/.gitignore | 0 release/python/.gitignore | 0 src/pg/Makefile | 59 +++++++++++++++++++++----- src/pg/README.md | 7 --- src/py/.gitignore | 1 - src/py/Makefile | 11 ++--- src/py/crankshaft/setup.py | 2 +- 12 files changed, 149 insertions(+), 57 deletions(-) create mode 100644 .gitignore create mode 100644 NEWS.md delete mode 100644 TODO.md create mode 100644 release/.gitignore create mode 100644 release/python/.gitignore delete mode 100644 src/pg/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..37819f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +envs/ diff --git a/Makefile b/Makefile index 5cae2cb..43b8480 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,31 @@ PYP_DIR = src/py .PHONY: install .PHONY: run_tests +.PHONY: release +.PHONY: deploy + +# Generate and install developmet versions of the extension +# and python package. +# The extension is named 'dev' with a 'current' alias for easily upgrading. +# The Python package is installed in a virtual environment envs/dev/ +# Requires sudo. install: $(MAKE) -C $(PYP_DIR) install $(MAKE) -C $(EXT_DIR) install -testinstalled: - $(MAKE) -C $(PYP_DIR) testinstalled - $(MAKE) -C $(EXT_DIR) installcheck +# Run the tests for the installed development extension and +# python package +test: + $(MAKE) -C $(PYP_DIR) test + $(MAKE) -C $(EXT_DIR) test + +# Generate a new release into release +release: + $(MAKE) -C $(EXT_DIR) release + +# Install the current release. +# The Python package is installed in a virtual environment envs/X.Y.Z/ +# Requires sudo. +deploy: + $(MAKE) -C $(EXT_DIR) deploy diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..201da51 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,3 @@ +0.0.1 (2016-03-15) +------------------ +* Preliminar release diff --git a/README.md b/README.md index c9dd529..75f7c2e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code * *release* reseleased versions +* *env* base directory for Python virtual environments ## Requirements @@ -17,16 +18,24 @@ CartoDB Spatial Analysis extension for PostgreSQL. # Working Process +We use the default `develop` branch as the basis for development. +This branch and `master` are maintained by the *Release Manager*. +The `master` branch is used to merge and tag releases to be +deployed in production. + +In addition to these two permanent branches, temporal *topic* +branches will be used for all modifications. + ## Development -Work in `src/pg/sql`, `src/py/crankshaft`; -use a topic branch. See src/py/README.md -for the procedure to work with the Python local environment. +A topic branch should be created out of the `develop` branch +and be used for the development process; see src/py/README.md +Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. Take into account: -* Always remember to add tests for any new functionality - documentation. +* Always remember to add tests (`src/pg/test`, `src/py/crankshaft/test`) + for any new functionality. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. @@ -35,12 +44,22 @@ Take into account: and private functions (to be used only internally inside the extension) with `_cdb_`. -Update local installation with `sudo make install` -(this will update the 'dev' version of the extension in 'src/pg/') +Update the local development installation with `sudo make install`. +This will update the 'dev' version of the extension in 'src/pg/' and +make it available to PostgreSQL. +It will also install the python package (crankshaft) in a virtual +environment `env/dev`. -Run the tests with `PGUSER=postgres make test` +Run the tests with `make test` -Update extension in working database with +To use the python extension for custom tests, activate the virtual +environment with: + +``` +source envs/dev/bin/activate +``` + +Update extension in a working database with: * `ALTER EXTENSION crankshaft VERSION TO 'current';` `ALTER EXTENSION crankshaft VERSION TO 'dev';` @@ -57,31 +76,37 @@ we can: * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` +Note: the development extension uses the development pyhton virtual +environment automatically. + Once the tests are succeeding a new Pull-Request can be created. -CI-tests must be checked to be successfull. - -Before merging a topic branch peer code reviewing of the code is a must. +CI-tests must be checked to be successful. +Before proceeding to the release process peer code reviewing of the code is a must. ## Release The release process of a new version of the extension shall by performed by the designated *Release Manager*. -Note that we expect to gradually automate this process. +Note that we expect to gradually automate more of this process. -Having checkout the topic branch of the PR to be released: +Having checked the topic branch of the PR to be released it shall be +merged back into the `develop` branch to prepare the new release. The version number in `pg/cranckshaft.control` must first be updated. To do so [Semantic Versioning 2.0](http://semver.org/) is in order. +Thew `NEWS.md` will be updated. + We now will explain the process for the case of backwards-compatible releases (updating the minor or patch version numbers). TODO: document the complex case of major releases. The next command must be executed to produce the main installation -script for the new release, `release/cranckshaft--X.Y.Z.sql`. +script for the new release, `release/cranckshaft--X.Y.Z.sql` and +also to copy the python package to `release/python/X.Y.Z/crankshaft`. ``` make release @@ -93,10 +118,32 @@ releases this simply consist in extracting the functions that have changed and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` file. -TODO: configure the local enviroment to be used by the release; -currently should be directory `src/py/X.Y.Z`, but this must be fixed; -a possibility to explore is to use the `cdb_conf` table. +The new release can be deployed for staging/smoke tests with this command: -TODO: testing procedure for the new release +``` +sudo make deploy +``` -TODO: push, merge, tag, deploy procedures. +This will make the 'X.Y.Z' version of the extension to PostgreSQL. +The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z` + +It can be activated with: + +``` +source envs/X.Y.Z/bin/activate +``` + +But note that this is needed only for using the package directly; +the 'X.Y.Z' version of the extension will automatically use the +python package from this virtual environment. + +The `sudo make deploy` operation can be also used for installing +the new version after it has been released. + +TODO: testing procedure for the new release. + +TODO: procedure for staging deployment. + +TODO: procedure for merging to master, tagging and deploying +in production. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 8a708e2..0000000 --- a/TODO.md +++ /dev/null @@ -1,9 +0,0 @@ -* [x] Support versioning -* [x] Test use of `plpy` from python Package -* [x] Add `pysal` etc. dependencies -* [x] Define documentation practices (general, per extension/package?) -* [x] Add initial function set (WIP) -* Unify style of function comments -* [x] Add integration tests -* Make target to open a new version development (create symlinks, etc.) -* [x] Should add cartodb ext. as a dependency? diff --git a/release/.gitignore b/release/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/release/python/.gitignore b/release/python/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/src/pg/Makefile b/src/pg/Makefile index ed65eba..fea7386 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -1,9 +1,13 @@ -# Generation of a new development version 'dev' (with an alias 'current' for -# updating easily by upgrading to 'current', then 'dev') - -# sudo make install -- generate the 'dev' version from current source -# and make it available to PostgreSQL -# PGUSER=postgres make installcheck -- test the 'dev' extension +# Development tasks: +# +# * install generates the control & script files into src/pg/ +# and installs then into the PostgreSQL extensions directory; +# requires sudo. In additionof the current development version +# named 'dev', an alias 'current' is generating for ease of +# update (upgrade to 'current', then to 'dev'). +# the python module is installed in a virtualenv in envs/dev/ +# * test runs the tests for the currently generated Development +# extension. SED = sed @@ -16,7 +20,7 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) -VIRTUALENV_PATH = $(realpath ../py/) +VIRTUALENV_PATH = $(realpath ../../envs) ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ @@ -36,13 +40,46 @@ include $(PGXS) # This seems to be needed at least for PG 9.3.11 all: $(DATA) +test: export PGUSER=postgres +test: installcheck -# WIP: goals for releasing the extension... +# Release tasks +PACKAGE = crankshaft EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -../release/$(EXTENSION).control: $(EXTENSION).control +../../release/$(EXTENSION).control: $(EXTENSION).control cp $< $@ -release: ../release/$(EXTENSION).control - cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql +# Prepare new release from the currently installed development version, +# for the current version X.Y.Z (defined in the control file) +# producing the extension script and control files in releases/ +# and the python package in releases/python/X.Y.Z/crankshaft/ +release: ../../release/$(EXTENSION).control + cp $(EXTENSION)--dev.sql ../../release/$(EXTENSION)--$(EXTVERSION).sql + mkdir -p ../../release/python/$(EXTVERSION) + cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ + $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py + +# Install the current relese into the PostgreSQL extensions directory +# and the Python package in a virtual environment envs/X.Y.Z +deploy: + $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' + $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' + # TODO: install also upgrade/downgrade files (manually generated) + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(EXTVERSION) + $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I -U ../../release/python/$(EXTVERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose + +clean-dev: + rm $(EXTNAME)--*.sql + +clean-releases: + rm -rf ../../release/python/* + rm ../../release/$(EXTNAME)--*.sql + rm ../../release/$(EXTNAME).control + +clean-environments: + rm -rf ../../envs/* + +clean-all: clean-dev clean-releases clean-environments diff --git a/src/pg/README.md b/src/pg/README.md deleted file mode 100644 index 511fdae..0000000 --- a/src/pg/README.md +++ /dev/null @@ -1,7 +0,0 @@ - -# Running the tests: - -``` -sudo make install -PGUSER=postgres make installcheck -``` diff --git a/src/py/.gitignore b/src/py/.gitignore index fb50fe5..0d20b64 100644 --- a/src/py/.gitignore +++ b/src/py/.gitignore @@ -1,2 +1 @@ *.pyc -dev/ diff --git a/src/py/Makefile b/src/py/Makefile index 16be269..ab787ee 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,9 +1,10 @@ # Install the package locally for development install: - virtualenv --system-site-packages dev - ./dev/bin/pip install -I ./crankshaft - ./dev/bin/pip install -I nose + virtualenv --system-site-packages ../../envs/dev + # source ../../envs/dev/bin/activate + ../../envs/dev/bin/pip install -I ./crankshaft + ../../envs/dev/bin/pip install -I nose # Test develpment install -testinstalled: - ./dev/bin/nosetests crankshaft/test/ +test: + ../../envs/dev/bin/nosetests crankshaft/test/ diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index aaa76a6..8d5e622 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='crankshaft', - version='0.0.1', + version='0.0.0', description='CartoDB Spatial Analysis Python Library', From 82a738fe40fb0ab2f4f15f3a205748b4dc07b2ff Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 15 Mar 2016 19:22:27 +0100 Subject: [PATCH 08/81] Fix make clean tasks --- src/pg/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/Makefile b/src/pg/Makefile index fea7386..4e8a444 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -72,12 +72,12 @@ deploy: $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose clean-dev: - rm $(EXTNAME)--*.sql + rm -f $(EXTENSION)--*.sql clean-releases: rm -rf ../../release/python/* - rm ../../release/$(EXTNAME)--*.sql - rm ../../release/$(EXTNAME).control + rm -f ../../release/$(EXTNAME)--*.sql + rm -f ../../release/$(EXTNAME).control clean-environments: rm -rf ../../envs/* From 24e4037995f61d8d7387fcdc950901080b168339 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 11:11:16 +0100 Subject: [PATCH 09/81] Fix version number of released extension script --- src/pg/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/Makefile b/src/pg/Makefile index 4e8a444..d81023c 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -55,8 +55,8 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/def # for the current version X.Y.Z (defined in the control file) # producing the extension script and control files in releases/ # and the python package in releases/python/X.Y.Z/crankshaft/ -release: ../../release/$(EXTENSION).control - cp $(EXTENSION)--dev.sql ../../release/$(EXTENSION)--$(EXTVERSION).sql +release: ../../release/$(EXTENSION).control $(SOURCES_DATA) + $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql mkdir -p ../../release/python/$(EXTVERSION) cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py From a518034e65b9e3a5ae0b2f1931f72310a851b22c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 11:13:26 +0100 Subject: [PATCH 10/81] Fix .pyc files need not only be ignored inside src/py --- .gitignore | 1 + src/py/.gitignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 src/py/.gitignore diff --git a/.gitignore b/.gitignore index 37819f7..8d2abce 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ envs/ +*.pyc diff --git a/src/py/.gitignore b/src/py/.gitignore deleted file mode 100644 index 0d20b64..0000000 --- a/src/py/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc From d13f167d47c9dfd90198ddd239b1d7ae17a33577 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 14:38:18 +0100 Subject: [PATCH 11/81] Add RELEASE_VERSION option to make deploy Now make deploy installs by default the current version, but can be made to install any prior specific version using a environmnt varialbe RELEASE_VERSION --- Makefile | 2 ++ src/pg/Makefile | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 43b8480..738b83e 100644 --- a/Makefile +++ b/Makefile @@ -29,5 +29,7 @@ release: # Install the current release. # The Python package is installed in a virtual environment envs/X.Y.Z/ # Requires sudo. +# Use the RELEASE_VERSION environment variable to deploy a specific version: +# sudo make deploy RELEASE_VERSION=1.0.0 deploy: $(MAKE) -C $(EXT_DIR) deploy diff --git a/src/pg/Makefile b/src/pg/Makefile index d81023c..ebe7009 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -61,15 +61,17 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py +RELEASE_VERSION ?= $(EXTVERSION) + # Install the current relese into the PostgreSQL extensions directory # and the Python package in a virtual environment envs/X.Y.Z deploy: $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' # TODO: install also upgrade/downgrade files (manually generated) - virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(EXTVERSION) - $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I -U ../../release/python/$(EXTVERSION)/$(PACKAGE) - $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose clean-dev: rm -f $(EXTENSION)--*.sql From dea6e2f1a7fcf3abdf341441a445fff72f7d3436 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:40:40 +0100 Subject: [PATCH 12/81] Refactor the Makefile Separate concerns properly for each subdirectory's Makefile --- Makefile | 45 ++++++++++++++++++++++++++++++++++++++++----- Makefile.global | 6 ++++++ src/pg/Makefile | 31 ++----------------------------- src/py/Makefile | 12 ++++++++++++ 4 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 Makefile.global diff --git a/Makefile b/Makefile index 738b83e..979c2b5 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +include ./Makefile.global + EXT_DIR = src/pg PYP_DIR = src/py @@ -6,30 +8,63 @@ PYP_DIR = src/py .PHONY: release .PHONY: deploy - # Generate and install developmet versions of the extension # and python package. # The extension is named 'dev' with a 'current' alias for easily upgrading. # The Python package is installed in a virtual environment envs/dev/ # Requires sudo. -install: +install: ## Generate and install development version of the extension; requires sudo. $(MAKE) -C $(PYP_DIR) install $(MAKE) -C $(EXT_DIR) install # Run the tests for the installed development extension and # python package -test: +test: ## Run the tests for the development version of the extension $(MAKE) -C $(PYP_DIR) test $(MAKE) -C $(EXT_DIR) test # Generate a new release into release -release: +release: ## Generate a new release of the extension. Only for telease manager $(MAKE) -C $(EXT_DIR) release + $(MAKE) -C $(PYP_DIR) release # Install the current release. # The Python package is installed in a virtual environment envs/X.Y.Z/ # Requires sudo. # Use the RELEASE_VERSION environment variable to deploy a specific version: # sudo make deploy RELEASE_VERSION=1.0.0 -deploy: +deploy: ## Deploy a released extension. Only for release manager. Requires sudo. $(MAKE) -C $(EXT_DIR) deploy + $(MAKE) -C $(PYP_DIR) deploy + +# Cleanup development extension script files +clean-dev: ## clean up development extension script files + rm -f src/pg/$(EXTENSION)--*.sql + +# # Cleanup all releases +# clean-releases: +# rm -rf release/python/* +# rm -f release/$(EXTENSION)--*.sql +# rm -f release/$(EXTENSION).control + +# Cleanup current/specific version +clean-release: ## clean up current release + rm -rf release/python/$(RELEASE_VERSION) + rm -f release/$(RELEASE_VERSION)--*.sql + +# Cleanup all virtual environments +clean-environments: ## clean up all virtual environments + rm -rf envs/* + +clean-all: clean-dev clean-release clean-environments + +help: + @IFS=$$'\n' ; \ + help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//'`); \ + for help_line in $${help_lines[@]}; do \ + IFS=$$'#' ; \ + help_split=($$help_line) ; \ + help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \ + help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \ + printf "%-30s %s\n" $$help_command $$help_info ; \ + done diff --git a/Makefile.global b/Makefile.global new file mode 100644 index 0000000..77f6c69 --- /dev/null +++ b/Makefile.global @@ -0,0 +1,6 @@ +SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +EXTENSION = crankshaft +PACKAGE = crankshaft +EXTVERSION = $(shell grep default_version $(SELF_DIR)/src/pg/$(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +RELEASE_VERSION ?= $(EXTVERSION) +SED = sed diff --git a/src/pg/Makefile b/src/pg/Makefile index ebe7009..8a745c4 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -1,3 +1,5 @@ +include ../../Makefile.global + # Development tasks: # # * install generates the control & script files into src/pg/ @@ -9,10 +11,6 @@ # * test runs the tests for the currently generated Development # extension. -SED = sed - -EXTENSION = crankshaft - DATA = $(EXTENSION)--dev.sql \ $(EXTENSION)--current--dev.sql \ $(EXTENSION)--dev--current.sql @@ -45,9 +43,6 @@ test: installcheck # Release tasks -PACKAGE = crankshaft -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") - ../../release/$(EXTENSION).control: $(EXTENSION).control cp $< $@ @@ -57,31 +52,9 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/def # and the python package in releases/python/X.Y.Z/crankshaft/ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql - mkdir -p ../../release/python/$(EXTVERSION) - cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ - $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py - -RELEASE_VERSION ?= $(EXTVERSION) # Install the current relese into the PostgreSQL extensions directory # and the Python package in a virtual environment envs/X.Y.Z deploy: $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' - # TODO: install also upgrade/downgrade files (manually generated) - virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose - -clean-dev: - rm -f $(EXTENSION)--*.sql - -clean-releases: - rm -rf ../../release/python/* - rm -f ../../release/$(EXTNAME)--*.sql - rm -f ../../release/$(EXTNAME).control - -clean-environments: - rm -rf ../../envs/* - -clean-all: clean-dev clean-releases clean-environments diff --git a/src/py/Makefile b/src/py/Makefile index ab787ee..90b22b8 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,3 +1,5 @@ +include ../../Makefile.global + # Install the package locally for development install: virtualenv --system-site-packages ../../envs/dev @@ -8,3 +10,13 @@ install: # Test develpment install test: ../../envs/dev/bin/nosetests crankshaft/test/ + +release: ../../release/$(EXTENSION).control $(SOURCES_DATA) + mkdir -p ../../release/python/$(EXTVERSION) + cp -r ./$(PACKAGE) ../../release/python/$(EXTVERSION)/ + $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py + +deploy: + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose From 4ffb2c966447a60b8676e029efd32397dae14524 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:45:13 +0100 Subject: [PATCH 13/81] Review and fix the documentation --- README.md | 59 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 75f7c2e..5c52541 100644 --- a/README.md +++ b/README.md @@ -24,33 +24,43 @@ The `master` branch is used to merge and tag releases to be deployed in production. In addition to these two permanent branches, temporal *topic* -branches will be used for all modifications. +branches must be used for all modifications. ## Development -A topic branch should be created out of the `develop` branch +For any modification of crankshaft, including adding new features, +a topic branch must be created out of the `develop` branch and be used for the development process; see src/py/README.md +for further details about the Pyhton package development. Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. Take into account: -* Always remember to add tests (`src/pg/test`, `src/py/crankshaft/test`) - for any new functionality. +* Test must be added for any new functionality + (`src/pg/test`, `src/py/crankshaft/test`) as well as for + detected any bugs corrected. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. -* Convention: Use snake case (i.e. `snake_case` and not `CamelCase`) for all - functions. Prefix functions intended for public use with `cdb_` +* Convention: snake case(i.e. `snake_case` and not `CamelCase`) + shall be used for all function names. + Prefix function names intended for public use with `cdb_` and private functions (to be used only internally inside the extension) with `_cdb_`. -Update the local development installation with `sudo make install`. -This will update the 'dev' version of the extension in 'src/pg/' and +Once the code is ready to be tested, update the local development installation +with `sudo make install`. +This will update the 'dev' version of the extension in `src/pg/` and make it available to PostgreSQL. It will also install the python package (crankshaft) in a virtual environment `env/dev`. -Run the tests with `make test` +The version number of the Python package, defined in +`src/pg/crankshaft/setup.py` will be overridden when +the package is released and always match the extension version number, +but for development it shall be kept as '0.0.0'. + +Run the tests with `make test`. To use the python extension for custom tests, activate the virtual environment with: @@ -71,23 +81,27 @@ changes involve incompatible function changes such as a different return type; in that case the offending function (or the whole extension) should be dropped manually before the update. -If the extension has not previously been installed in a database -we can: +If the extension has not previously been installed in a database, +it can be installed directly with: * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` -Note: the development extension uses the development pyhton virtual +Note: the development extension uses the development python virtual environment automatically. -Once the tests are succeeding a new Pull-Request can be created. -CI-tests must be checked to be successful. +Once the tests are succeeding a new Pull-Request can be created +to the develop branch. CI-tests must be checked to be successful. -Before proceeding to the release process peer code reviewing of the code is a must. +Before proceeding to the release process peer code reviewing of the code is +a must. + +When the code is accepted by the peer reviewing (and CI tests succeed) +a request for release will be send to the Release Manager. ## Release The release process of a new version of the extension -shall by performed by the designated *Release Manager*. +shall be performed by the designated *Release Manager*. Note that we expect to gradually automate more of this process. @@ -124,9 +138,9 @@ The new release can be deployed for staging/smoke tests with this command: sudo make deploy ``` -This will make the 'X.Y.Z' version of the extension to PostgreSQL. -The corresponding Python extension will be installed in a -virtual environment in `envs/X.Y.Z` +This will copy the current 'X.Y.Z' released version of the extension to +PostgreSQL. The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z`. It can be activated with: @@ -141,6 +155,13 @@ python package from this virtual environment. The `sudo make deploy` operation can be also used for installing the new version after it has been released. +To install a specific version 'X.Y.Z' different from the current one +(which must be present in `releases/`) you can: + +``` +sudo make deploy RELEASE_VERSION=X.Y.Z +``` + TODO: testing procedure for the new release. TODO: procedure for staging deployment. From ef3bcaeee891310ef0cd376068f652900345518f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:52:47 +0100 Subject: [PATCH 14/81] Restore commented-out make target --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 979c2b5..9c79e00 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,11 @@ deploy: ## Deploy a released extension. Only for release manager. Requires sudo. clean-dev: ## clean up development extension script files rm -f src/pg/$(EXTENSION)--*.sql -# # Cleanup all releases -# clean-releases: -# rm -rf release/python/* -# rm -f release/$(EXTENSION)--*.sql -# rm -f release/$(EXTENSION).control +# Cleanup all releases +clean-releases: + rm -rf release/python/* + rm -f release/$(EXTENSION)--*.sql + rm -f release/$(EXTENSION).control # Cleanup current/specific version clean-release: ## clean up current release From 935c7f99639dae006b30179f1c542846b0789007 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:54:39 +0100 Subject: [PATCH 15/81] Add missing Makefile comment --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9c79e00..6c3e219 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ clean-dev: ## clean up development extension script files rm -f src/pg/$(EXTENSION)--*.sql # Cleanup all releases -clean-releases: +clean-releases: ## clean up all releases rm -rf release/python/* rm -f release/$(EXTENSION)--*.sql rm -f release/$(EXTENSION).control From 4706442a1ddb539646f1130d204e8bed7e7dcc8f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:56:19 +0100 Subject: [PATCH 16/81] Add documentation about useful make targets --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 5c52541..d0305a1 100644 --- a/README.md +++ b/README.md @@ -168,3 +168,35 @@ TODO: procedure for staging deployment. TODO: procedure for merging to master, tagging and deploying in production. + +## Relevant tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +# Development tasks + +* `sudo make install` will generate the extension scripts for the development + version ('dev'/'current') and install the python package into the + development virtual environment `envs/dev`. + Intended for use by developers. + +* `make test` will run the tests for the installed development extension. + Intended for use by developers. + +# Release tasks + +* `make release` will generate a new release (version number defined in + `src/pg/crankshaft.control`) into `release/`. + Intended for use by the release manager. + +* `sudo make deploy` will install the current release X.Y.Z from the + `release/` files into PostgreSQL and a Python virtual environment + `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. + +* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version + previously generated in `release/` + into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. +``` From bad09ffd7be56da0ea50d97300c526d182b7a757 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 16:30:03 +0100 Subject: [PATCH 17/81] Remove abandoned alternatives from the documentation --- src/py/README.md | 70 ++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 56 deletions(-) diff --git a/src/py/README.md b/src/py/README.md index d55b7d7..b9bf64d 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -20,37 +20,29 @@ nosetests test/ --- -We have two possible approaches being considered as to how manage -the Python virtual environment: using a pure virtual enviroment -or combine it with some system packages that include depencencies -for the *hard-to-compile* packages (and pin them in somewhat old versions). +To avoid troublesome compilations/linkings we will use +the available system package `python-scipy`. +This package and its dependencies provide numpy 1.6.1 +and scipy 0.9.0. To be able to use these versions we cannot +PySAL 1.10 or later, so we'll stick to 1.9.1. -### Alternative A: pure virtual environment +``` +apt-get install -y python-scipy +``` -In this case we will install all the packages needed in the -virtual environment. -This will involve, specially for the numerical packages compiling -and linking code that uses a number of third party libraries, -and requires having theses depencencies solved for the production -environments. +We'll use virtual environments to install our packages, +but configued to use also system modules so that the +mentioned scipy and numpy are used. -#### Create and use a virtual env - -We'll use a virtual enviroment directory `dev` -under the `src/pg` directory. - - # Create the virtual environment for python - $ virtualenv dev + # Create a virtual environment for python + $ virtualenv --system-site-packages dev # Activate the virtualenv $ source dev/bin/activate # Install all the requirements # expect this to take a while, as it will trigger a few compilations - (dev) $ pip install -r requirements.txt - - # Add a new pip to the party - (dev) $ pip install pandas + (dev) $ pip install -I ./crankshaft #### Test the libraries with that virtual env @@ -94,37 +86,3 @@ Then, execute the tests with: import pysal import nose nose.runmodule('pysal') - - -### Alternative B: using some packaged modules - -This option avoids troublesome compilations/linkings, at the cost -of freezing some module versions as available in system packages, -namely numpy 1.6.1 and scipy 0.9.0. (in turn, this implies -the most recent version of PySAL we can use is 1.9.1) - - -TODO: to use this alternative the python-scipy package must be -installed (this will have to be included in server provisioning) - -``` -apt-get install -y python-scipy -``` - -#### Create and use a virtual env - -We'll use a `dev` enviroment as before, but will configure it to -use also system modules. - - - # Create the virtual environment for python - $ virtualenv --system-site-packages dev - - # Activate the virtualenv - $ source dev/bin/activate - - # Install all the requirements - # expect this to take a while, as it will trigger a few compilations - (dev) $ pip install -I ./crankshaft - -Then we can proceed to testing as in Alternative A. From a7e42e93cc5781e0208eb0e4589543bf4ca4ec6e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 16:41:54 +0100 Subject: [PATCH 18/81] Rename cdb_crankshaft_internal_version as internal function --- src/pg/sql/01_version.sql | 2 +- src/pg/sql/02_py.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/01_version.sql b/src/pg/sql/01_version.sql index 90e6123..5c701fd 100644 --- a/src/pg/sql/01_version.sql +++ b/src/pg/sql/01_version.sql @@ -6,7 +6,7 @@ $$ language 'sql' IMMUTABLE STRICT; -- Internal identifier of the installed extension instence -- e.g. 'dev' for current development version -CREATE OR REPLACE FUNCTION cdb_crankshaft_internal_version() +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() RETURNS text AS $$ SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; $$ language 'sql' IMMUTABLE STRICT; diff --git a/src/pg/sql/02_py.sql b/src/pg/sql/02_py.sql index a233a0a..7da5f47 100644 --- a/src/pg/sql/02_py.sql +++ b/src/pg/sql/02_py.sql @@ -14,7 +14,7 @@ AS $$ import os # plpy.notice('%',str(os.environ)) # activate virtualenv - crankshaft_version = plpy.execute('SELECT cdb_crankshaft.cdb_crankshaft_internal_version()')[0]['cdb_crankshaft_internal_version'] + crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] default_venv_path = os.path.join(base_path, crankshaft_version) venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) From 689ec8a925297b0f85e5def6223c063db4b53ba0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 17:09:50 +0100 Subject: [PATCH 19/81] Change version function from IMMUTABLE to STABLE These functions' results will change when the extension is updated. --- src/pg/sql/01_version.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/01_version.sql b/src/pg/sql/01_version.sql index 5c701fd..f73c764 100644 --- a/src/pg/sql/01_version.sql +++ b/src/pg/sql/01_version.sql @@ -2,11 +2,11 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft_version() RETURNS text AS $$ SELECT '@@VERSION@@'::text; -$$ language 'sql' IMMUTABLE STRICT; +$$ language 'sql' STABLE STRICT; -- Internal identifier of the installed extension instence -- e.g. 'dev' for current development version CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() RETURNS text AS $$ SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; -$$ language 'sql' IMMUTABLE STRICT; +$$ language 'sql' STABLE STRICT; From b5281d06813214b2045e4a8eb26d4c6f9cfef1a3 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 17:19:21 +0100 Subject: [PATCH 20/81] Documentation clarifications and corrections. --- README.md | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d0305a1..5003292 100644 --- a/README.md +++ b/README.md @@ -18,27 +18,42 @@ CartoDB Spatial Analysis extension for PostgreSQL. # Working Process +We distinguish two roles regarding the development cycle of crankshaft: + +* *developers* will implement new functionality and bugfixes into + the codebase and will request for new releases of the extension. +* A *release manager* will attend these requests and will handle + the release process. The release process is sequential: + no concurrent releases will ever be in the works. + We use the default `develop` branch as the basis for development. This branch and `master` are maintained by the *Release Manager*. The `master` branch is used to merge and tag releases to be deployed in production. -In addition to these two permanent branches, temporal *topic* -branches must be used for all modifications. +Developers shall create a new topic branch from `develop` for any new feature +or bugfix and commit their changes to it and eventually merge back into +the `develop` branch. When a new release is required a Pull Request +will be open againt the `develop` branch. + +The `develop` pull requests will be handled by the release manage, +who will merge into master where new releases are prepared and tagged. +The `master` branch is the sole responsibility of the release masters +and developers must not commit or merge into it. ## Development -For any modification of crankshaft, including adding new features, -a topic branch must be created out of the `develop` branch -and be used for the development process; see src/py/README.md -for further details about the Pyhton package development. +For any modification of crankshaft, such as adding new features, +refactoring or bug-fixing, topic branch must be created out of the `develop` +branch and be used for the development process. Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. + Take into account: -* Test must be added for any new functionality - (`src/pg/test`, `src/py/crankshaft/test`) as well as for - detected any bugs corrected. +* Tests must be added for any new functionality + (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to + detect any bugs that are being fixed. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. @@ -89,14 +104,17 @@ it can be installed directly with: Note: the development extension uses the development python virtual environment automatically. -Once the tests are succeeding a new Pull-Request can be created -to the develop branch. CI-tests must be checked to be successful. - Before proceeding to the release process peer code reviewing of the code is a must. -When the code is accepted by the peer reviewing (and CI tests succeed) -a request for release will be send to the Release Manager. +Once the feature or bugfix is completed, all the tests are passing +and the code has been accepted by peer reviewing, +the topic branch can be merged back into the `develop` branch and a +new Pull-Request can be created on it. +CI-tests must be checked to be successful. + +The release manage will take hold of the PR at this moment to proceed +to the release process for a new revision of the extension. ## Release @@ -105,8 +123,8 @@ shall be performed by the designated *Release Manager*. Note that we expect to gradually automate more of this process. -Having checked the topic branch of the PR to be released it shall be -merged back into the `develop` branch to prepare the new release. +Having checked PR to be released it shall be +merged back into the `master` branch to prepare the new release. The version number in `pg/cranckshaft.control` must first be updated. To do so [Semantic Versioning 2.0](http://semver.org/) is in order. From 9f03a9b0754a790a000701cdfd744f242bd4e2a2 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 17:42:28 +0100 Subject: [PATCH 21/81] Reorganize the documentation into separate files Keep a "Quickstart Guide" in the README, add separate detailed sections for development (CONTRIBUTING) and release/deployment (RELEASE). --- CONTRIBUTING.md | 90 +++++++++++++++++++++++ DEPLOYING.md | 43 ----------- README.md | 186 +++++------------------------------------------- RELEASE.md | 93 ++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 210 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 DEPLOYING.md create mode 100644 RELEASE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c6422a2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,90 @@ +# Development process + +Please read the Working Process/Quickstart Guide in README.md first. + +For any modification of crankshaft, such as adding new features, +refactoring or bug-fixing, topic branch must be created out of the `develop` +branch and be used for the development process. + +Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. + +Take into account: + +* Tests must be added for any new functionality + (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to + detect any bugs that are being fixed. +* Add or modify the corresponding documentation files in the `doc` folder. + Since we expect to have highly technical functions here, an extense + background explanation would be of great help to users of this extension. +* Convention: snake case(i.e. `snake_case` and not `CamelCase`) + shall be used for all function names. + Prefix function names intended for public use with `cdb_` + and private functions (to be used only internally inside + the extension) with `_cdb_`. + +Once the code is ready to be tested, update the local development installation +with `sudo make install`. +This will update the 'dev' version of the extension in `src/pg/` and +make it available to PostgreSQL. +It will also install the python package (crankshaft) in a virtual +environment `env/dev`. + +The version number of the Python package, defined in +`src/pg/crankshaft/setup.py` will be overridden when +the package is released and always match the extension version number, +but for development it shall be kept as '0.0.0'. + +Run the tests with `make test`. + +To use the python extension for custom tests, activate the virtual +environment with: + +``` +source envs/dev/bin/activate +``` + +Update extension in a working database with: + +* `ALTER EXTENSION crankshaft VERSION TO 'current';` + `ALTER EXTENSION crankshaft VERSION TO 'dev';` + +Note: we keep the current development version install as 'dev' always; +we update through the 'current' alias to allow changing the extension +contents but not the version identifier. This will fail if the +changes involve incompatible function changes such as a different +return type; in that case the offending function (or the whole extension) +should be dropped manually before the update. + +If the extension has not previously been installed in a database, +it can be installed directly with: + +* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` + +Note: the development extension uses the development python virtual +environment automatically. + +Before proceeding to the release process peer code reviewing of the code is +a must. + +Once the feature or bugfix is completed, all the tests are passing +and the code has been accepted by peer reviewing, +the topic branch can be merged back into the `develop` branch and a +new Pull-Request can be created on it. +CI-tests must be checked to be successful. + +The release manage will take hold of the PR at this moment to proceed +to the release process for a new revision of the extension. + +## Relevant development tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +* `sudo make install` will generate the extension scripts for the development + version ('dev'/'current') and install the python package into the + development virtual environment `envs/dev`. + Intended for use by developers. + +* `make test` will run the tests for the installed development extension. + Intended for use by developers. +``` diff --git a/DEPLOYING.md b/DEPLOYING.md deleted file mode 100644 index 5b21f30..0000000 --- a/DEPLOYING.md +++ /dev/null @@ -1,43 +0,0 @@ -# Workflow - -... (branching/merging flow) - -# Deployment - -... - -Deployment to db servers: the next command will install both the Python -package and the extension. - -``` -sudo make install -``` - -Installing only the Python package: - -``` -sudo pip install python/crankshaft --upgrade -``` - -Caveat: note that `pip install ./crankshaft` will install -from local files, but `pip install crankshaft` will not. - -CI: Install and run the tests on the installed extension and package: - -``` -(sudo make install && PGUSER=postgres make testinstalled) -``` - -Installing the extension in user databases: -Once installed in a server, the extension can be added -to a database with the next SQL command: - -``` -CREATE EXTENSION crankshaft; -``` - -To upgrade the extension to an specific version X.Y.Z: - -``` -ALTER EXTENSION crankshaft UPGRADE TO 'X.Y.Z'; -``` diff --git a/README.md b/README.md index 5003292..cf8de5a 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. * pip, virtualenv, PostgreSQL * python-scipy system package (see src/py/README.md) -# Working Process +# Working Process -- Quickstart Guide We distinguish two roles regarding the development cycle of crankshaft: @@ -41,180 +41,32 @@ who will merge into master where new releases are prepared and tagged. The `master` branch is the sole responsibility of the release masters and developers must not commit or merge into it. -## Development +## Development Guidelines -For any modification of crankshaft, such as adding new features, -refactoring or bug-fixing, topic branch must be created out of the `develop` -branch and be used for the development process. +For a detailed description of the development process please see +the CONTRIBUTING.md guide. -Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. +Any modification to the source code (`src/pg/sql` for the SQL extension, +`src/py/crankshaft` for the Python package) shall always be done +in a topic branch created from the `develop` branch. -Take into account: +Tests, documentation and peer code reviewing are required for all +modifications. -* Tests must be added for any new functionality - (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to - detect any bugs that are being fixed. -* Add or modify the corresponding documentation files in the `doc` folder. - Since we expect to have highly technical functions here, an extense - background explanation would be of great help to users of this extension. -* Convention: snake case(i.e. `snake_case` and not `CamelCase`) - shall be used for all function names. - Prefix function names intended for public use with `cdb_` - and private functions (to be used only internally inside - the extension) with `_cdb_`. - -Once the code is ready to be tested, update the local development installation -with `sudo make install`. -This will update the 'dev' version of the extension in `src/pg/` and -make it available to PostgreSQL. -It will also install the python package (crankshaft) in a virtual -environment `env/dev`. - -The version number of the Python package, defined in -`src/pg/crankshaft/setup.py` will be overridden when -the package is released and always match the extension version number, -but for development it shall be kept as '0.0.0'. - -Run the tests with `make test`. - -To use the python extension for custom tests, activate the virtual -environment with: +The tests (both for SQL and Pyhton) are executed by running, +from the top directory: ``` -source envs/dev/bin/activate +sudo make install +make test ``` -Update extension in a working database with: - -* `ALTER EXTENSION crankshaft VERSION TO 'current';` - `ALTER EXTENSION crankshaft VERSION TO 'dev';` - -Note: we keep the current development version install as 'dev' always; -we update through the 'current' alias to allow changing the extension -contents but not the version identifier. This will fail if the -changes involve incompatible function changes such as a different -return type; in that case the offending function (or the whole extension) -should be dropped manually before the update. - -If the extension has not previously been installed in a database, -it can be installed directly with: - -* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` - -Note: the development extension uses the development python virtual -environment automatically. - -Before proceeding to the release process peer code reviewing of the code is -a must. - -Once the feature or bugfix is completed, all the tests are passing -and the code has been accepted by peer reviewing, -the topic branch can be merged back into the `develop` branch and a -new Pull-Request can be created on it. -CI-tests must be checked to be successful. - -The release manage will take hold of the PR at this moment to proceed -to the release process for a new revision of the extension. +To request a new release, which will be handled by them +release manager, a Pull Request must be created in the `develop` +branch. ## Release -The release process of a new version of the extension -shall be performed by the designated *Release Manager*. - -Note that we expect to gradually automate more of this process. - -Having checked PR to be released it shall be -merged back into the `master` branch to prepare the new release. - -The version number in `pg/cranckshaft.control` must first be updated. -To do so [Semantic Versioning 2.0](http://semver.org/) is in order. - -Thew `NEWS.md` will be updated. - -We now will explain the process for the case of backwards-compatible -releases (updating the minor or patch version numbers). - -TODO: document the complex case of major releases. - -The next command must be executed to produce the main installation -script for the new release, `release/cranckshaft--X.Y.Z.sql` and -also to copy the python package to `release/python/X.Y.Z/crankshaft`. - -``` -make release -``` - -Then, the release manager shall produce upgrade and downgrade scripts -to migrate to/from the previous release. In the case of minor/patch -releases this simply consist in extracting the functions that have changed -and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` -file. - -The new release can be deployed for staging/smoke tests with this command: - -``` -sudo make deploy -``` - -This will copy the current 'X.Y.Z' released version of the extension to -PostgreSQL. The corresponding Python extension will be installed in a -virtual environment in `envs/X.Y.Z`. - -It can be activated with: - -``` -source envs/X.Y.Z/bin/activate -``` - -But note that this is needed only for using the package directly; -the 'X.Y.Z' version of the extension will automatically use the -python package from this virtual environment. - -The `sudo make deploy` operation can be also used for installing -the new version after it has been released. - -To install a specific version 'X.Y.Z' different from the current one -(which must be present in `releases/`) you can: - -``` -sudo make deploy RELEASE_VERSION=X.Y.Z -``` - -TODO: testing procedure for the new release. - -TODO: procedure for staging deployment. - -TODO: procedure for merging to master, tagging and deploying -in production. - -## Relevant tasks available in the Makefile - -``` -* `make help` show a short description of the available targets - -# Development tasks - -* `sudo make install` will generate the extension scripts for the development - version ('dev'/'current') and install the python package into the - development virtual environment `envs/dev`. - Intended for use by developers. - -* `make test` will run the tests for the installed development extension. - Intended for use by developers. - -# Release tasks - -* `make release` will generate a new release (version number defined in - `src/pg/crankshaft.control`) into `release/`. - Intended for use by the release manager. - -* `sudo make deploy` will install the current release X.Y.Z from the - `release/` files into PostgreSQL and a Python virtual environment - `envs/X.Y.Z`. - Intended for use by the release manager and deployment jobs. - -* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version - previously generated in `release/` - into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. - Intended for use by the release manager and deployment jobs. -``` +The release and deployment process is described in the +RELEASE.md guide and it is the responsibility of the designated +release manager. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..0db48a2 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,93 @@ +# Release & Deployment Process + +Please read the Working Process/Quickstart Guide in README.md +and the Development guidelines in CONTRIBUTING.md. + +The release process of a new version of the extension +shall be performed by the designated *Release Manager*. + +Note that we expect to gradually automate more of this process. + +Having checked PR to be released it shall be +merged back into the `master` branch to prepare the new release. + +The version number in `pg/cranckshaft.control` must first be updated. +To do so [Semantic Versioning 2.0](http://semver.org/) is in order. + +Thew `NEWS.md` will be updated. + +We now will explain the process for the case of backwards-compatible +releases (updating the minor or patch version numbers). + +TODO: document the complex case of major releases. + +The next command must be executed to produce the main installation +script for the new release, `release/cranckshaft--X.Y.Z.sql` and +also to copy the python package to `release/python/X.Y.Z/crankshaft`. + +``` +make release +``` + +Then, the release manager shall produce upgrade and downgrade scripts +to migrate to/from the previous release. In the case of minor/patch +releases this simply consist in extracting the functions that have changed +and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` +file. + +The new release can be deployed for staging/smoke tests with this command: + +``` +sudo make deploy +``` + +This will copy the current 'X.Y.Z' released version of the extension to +PostgreSQL. The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z`. + +It can be activated with: + +``` +source envs/X.Y.Z/bin/activate +``` + +But note that this is needed only for using the package directly; +the 'X.Y.Z' version of the extension will automatically use the +python package from this virtual environment. + +The `sudo make deploy` operation can be also used for installing +the new version after it has been released. + +To install a specific version 'X.Y.Z' different from the current one +(which must be present in `releases/`) you can: + +``` +sudo make deploy RELEASE_VERSION=X.Y.Z +``` + +TODO: testing procedure for the new release. + +TODO: procedure for staging deployment. + +TODO: procedure for merging to master, tagging and deploying +in production. + +## Relevant release & deployment tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +* `make release` will generate a new release (version number defined in + `src/pg/crankshaft.control`) into `release/`. + Intended for use by the release manager. + +* `sudo make deploy` will install the current release X.Y.Z from the + `release/` files into PostgreSQL and a Python virtual environment + `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. + +* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version + previously generated in `release/` + into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. +``` From 47e0253652f496b173744f57d1e9c3e1c9187201 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 18:18:59 +0100 Subject: [PATCH 22/81] Fixes to the documentation --- CONTRIBUTING.md | 11 ++++++----- README.md | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6422a2..bcdde4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,11 +66,12 @@ environment automatically. Before proceeding to the release process peer code reviewing of the code is a must. -Once the feature or bugfix is completed, all the tests are passing -and the code has been accepted by peer reviewing, -the topic branch can be merged back into the `develop` branch and a -new Pull-Request can be created on it. -CI-tests must be checked to be successful. +Once the feature or bugfix is completed and all the tests are passing +a Pull-Request shall be created on the topic branch, reviewed by a peer +and then merged back into the `develop` branch when all CI tests pass. + +When the changes in the `develop` branch are to be released in a new +version of the extension, a PR must be created on the `develop` branch. The release manage will take hold of the PR at this moment to proceed to the release process for a new revision of the extension. diff --git a/README.md b/README.md index cf8de5a..3ecb1d0 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ We distinguish two roles regarding the development cycle of crankshaft: no concurrent releases will ever be in the works. We use the default `develop` branch as the basis for development. -This branch and `master` are maintained by the *Release Manager*. The `master` branch is used to merge and tag releases to be deployed in production. From 4903af6cdcdd08bde5c15f00e0a0ee1f9f8877e0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 18:41:49 +0100 Subject: [PATCH 23/81] Add existing release 0.0.1 The existing 0.0.1 files are placed into their location in release/ --- release/crankshaft--0.0.1.sql | 148 ++++++++ release/crankshaft.control | 5 + .../0.0.1/crankshaft/crankshaft/__init__.py | 2 + .../crankshaft/clustering/__init__.py | 1 + .../crankshaft/crankshaft/clustering/moran.py | 321 ++++++++++++++++++ .../crankshaft/crankshaft/random_seeds.py | 10 + release/python/0.0.1/crankshaft/setup.py | 48 +++ .../0.0.1/crankshaft/test/fixtures/moran.json | 52 +++ .../crankshaft/test/fixtures/neighbors.json | 54 +++ .../python/0.0.1/crankshaft/test/helper.py | 13 + .../python/0.0.1/crankshaft/test/mock_plpy.py | 34 ++ .../crankshaft/test/test_clustering_moran.py | 144 ++++++++ 12 files changed, 832 insertions(+) create mode 100644 release/crankshaft--0.0.1.sql create mode 100644 release/crankshaft.control create mode 100644 release/python/0.0.1/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.0.1/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.0.1/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.0.1/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.0.1/crankshaft/setup.py create mode 100644 release/python/0.0.1/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.0.1/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.0.1/crankshaft/test/helper.py create mode 100644 release/python/0.0.1/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.0.1/crankshaft/test/test_clustering_moran.py diff --git a/release/crankshaft--0.0.1.sql b/release/crankshaft--0.0.1.sql new file mode 100644 index 0000000..436beea --- /dev/null +++ b/release/crankshaft--0.0.1.sql @@ -0,0 +1,148 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I +CREATE OR REPLACE FUNCTION + cdb_moran_local ( + t TEXT, + attr TEXT, + significance float DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate +CREATE OR REPLACE FUNCTION + cdb_moran_local_rate(t TEXT, + numerator TEXT, + denominator TEXT, + significance FLOAT DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control new file mode 100644 index 0000000..74777ac --- /dev/null +++ b/release/crankshaft.control @@ -0,0 +1,5 @@ +comment = 'CartoDB Spatial Analysis extension' +default_version = '0.0.1' +requires = 'plpythonu, postgis, cartodb' +superuser = true +schema = cdb_crankshaft diff --git a/release/python/0.0.1/crankshaft/crankshaft/__init__.py b/release/python/0.0.1/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..d07e330 --- /dev/null +++ b/release/python/0.0.1/crankshaft/crankshaft/__init__.py @@ -0,0 +1,2 @@ +import random_seeds +import clustering diff --git a/release/python/0.0.1/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.0.1/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..0df080f --- /dev/null +++ b/release/python/0.0.1/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1 @@ +from moran import * diff --git a/release/python/0.0.1/crankshaft/crankshaft/clustering/moran.py b/release/python/0.0.1/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..8882235 --- /dev/null +++ b/release/python/0.0.1/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,321 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import numpy as np +import pysal as ps +import plpy + +# High level interface --------------------------------------- + +def moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + # TODO: ensure that the significance output can be smaller that 1e-3 (0.001) + # TODO: make a wishlist of output features (zscores, pvalues, raw local lisa, what else?) + + plpy.notice('** Constructing query') + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "attr1": attr, + "geom_col": geom_column, + "table": t, + "num_ngbrs": num_ngbrs} + + q = get_query(w_type, qvals) + + try: + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Exiting function') + return zip([None], [None], [None], [None]) + + y = get_attributes(r, 1) + w = get_weight(r, w_type) + + # calculate LISA values + lisa = ps.Moran_Local(y, w) + + # find units of significance + lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) + + +def moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type): + """ + Moran's I Local Rate + Andy Eschbacher + """ + + plpy.notice('** Constructing query') + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_column, + "table": t, + "num_ngbrs": num_ngbrs} + + q = get_query(w_type, qvals) + + try: + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Error: %s' % plpy.SPIError) + plpy.notice('** Exiting function') + return zip([None], [None], [None], [None]) + + plpy.notice('r.nrows() = %d' % r.nrows()) + + ## collect attributes + numer = get_attributes(r, 1) + denom = get_attributes(r, 2) + + w = get_weight(r, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, w, permutations=permutations) + + # find units of significance + lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + + plpy.notice('** Finished calculations') + + ## TODO: Decide on which return values here + return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order, lisa.y) + +def moran_local_bv(t, attr1, attr2, significance, num_ngbrs, permutations, geom_column, id_col, w_type): + plpy.notice('** Constructing query') + + qvals = {"num_ngbrs": num_ngbrs, + "attr1": attr1, + "attr2": attr2, + "table": t, + "geom_col": geom_column, + "id_col": id_col} + + q = get_query(w_type, qvals) + + try: + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Error: %s' % plpy.SPIError) + plpy.notice('** Exiting function') + return zip([None], [None], [None], [None]) + + ## collect attributes + attr1_vals = get_attributes(r, 1) + attr2_vals = get_attributes(r, 2) + + # create weights + w = get_weight(r, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, w) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) + + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + :param coord (int): quadrant of a specific measurement + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + :param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + :param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM \"{table}\" As j " \ + "WHERE %(attr_where_j)s " \ + "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs} OFFSET 1 ) " \ + ") As neighbors " \ + "FROM \"{table}\" As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + :param params: dict of information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM \"{table}\" As j " \ + "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM \"{table}\" As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_query(w_type, query_vals): + """Return requested query. + :param w_type: type of neighbors to calculate (knn or queen) + :param query_vals: values used to construct the query + """ + + if w_type == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +def get_attributes(query_res, attr_num): + """ + :param query_res: query results with attributes and neighbors + :param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +## Build weight object +def get_weight(query_res, w_type='queen', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + :param query_res: query results with attributes and neighbors + """ + if w_type == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + elif w_type == 'queen': + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def quad_position(quads): + """ + Produce Moran's I classification based of n + """ + + lisa_sig = np.array([map_quads(q) for q in quads]) + + return lisa_sig + +def lisa_sig_vals(pvals, quads, threshold): + """ + Produce Moran's I classification based of n + """ + + sig = (pvals <= threshold) + + lisa_sig = np.empty(len(sig), np.chararray) + + for idx, val in enumerate(sig): + if val: + lisa_sig[idx] = map_quads(quads[idx]) + else: + lisa_sig[idx] = 'Not significant' + + return lisa_sig diff --git a/release/python/0.0.1/crankshaft/crankshaft/random_seeds.py b/release/python/0.0.1/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..b7c8eed --- /dev/null +++ b/release/python/0.0.1/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,10 @@ +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.0.1/crankshaft/setup.py b/release/python/0.0.1/crankshaft/setup.py new file mode 100644 index 0000000..f045b62 --- /dev/null +++ b/release/python/0.0.1/crankshaft/setup.py @@ -0,0 +1,48 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.0.01', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + install_requires=['pysal==1.11.0','numpy==1.6.1','scipy==0.17.0'], + + requires=['pysal', 'numpy'], + + test_suite='test' +) diff --git a/release/python/0.0.1/crankshaft/test/fixtures/moran.json b/release/python/0.0.1/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..0530c18 --- /dev/null +++ b/release/python/0.0.1/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "Not significant"], +[0.6152779669180425, "Not significant"], +[-0.14657336660125297, "Not significant"], +[0.6967858120189607, "Not significant"], +[0.07949310115714454, "Not significant"], +[0.4703198759258987, "Not significant"], +[0.4421125200498064, "Not significant"], +[0.5724288737143592, "Not significant"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "Not significant"], +[-0.01466729201304962, "Not significant"], +[0.3481559372544409, "Not significant"], +[0.06547094736902978, "Not significant"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "Not significant"], +[0.15971286468915544, "Not significant"], +[1.0543588860308968, "Not significant"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "Not significant"], +[0.08438455015300014, "Not significant"], +[0.06547094736902978, "Not significant"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "Not significant"], +[0.795275137550483, "Not significant"], +[0.18562939195219, "LL"], +[0.3010757406693439, "Not significant"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "Not significant"], +[-0.07116352791516614, "Not significant"], +[-0.09945240794119009, "Not significant"], +[0.18562939195219, "LL"], +[0.1832733440191868, "Not significant"], +[-0.39054253768447705, "Not significant"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "Not significant"], +[0.2584386102554792, "Not significant"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "Not significant"], +[0.051367269430983485, "Not significant"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "Not significant"], +[0.04191046803899837, "Not significant"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "Not significant"], +[0.5410013139159929, "Not significant"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] diff --git a/release/python/0.0.1/crankshaft/test/fixtures/neighbors.json b/release/python/0.0.1/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.0.1/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.0.1/crankshaft/test/helper.py b/release/python/0.0.1/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.0.1/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.0.1/crankshaft/test/mock_plpy.py b/release/python/0.0.1/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..63c88f6 --- /dev/null +++ b/release/python/0.0.1/crankshaft/test/mock_plpy.py @@ -0,0 +1,34 @@ +import re + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.0.1/crankshaft/test/test_clustering_moran.py b/release/python/0.0.1/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..2e730a1 --- /dev/null +++ b/release/python/0.0.1/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,144 @@ +import unittest +import numpy as np + +import unittest + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions.""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "table": "a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads.""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_query_attr_select(self): + """Test query_attr_select.""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(cc.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test query_attr_where.""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND "\ + "idx_replace.\"{attr2}\" IS NOT NULL AND "\ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(cc.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn function.""" + + ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM \"a_list\" As j WHERE j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 OFFSET 1 ) ) " \ + "As neighbors FROM \"a_list\" As i WHERE i.\"andy\" IS NOT " \ + "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ + "BY i.\"cartodb_id\" ASC;" + + self.assertEqual(cc.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors function.""" + + ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ + "j.\"cartodb_id\" FROM \"a_list\" As j WHERE ST_Touches(" \ + "i.\"the_geom\", j.\"the_geom\") AND j.\"andy\" IS NOT NULL " \ + "AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0)) As " \ + "neighbors FROM \"a_list\" As i WHERE i.\"andy\" IS NOT NULL " \ + "AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER BY " \ + "i.\"cartodb_id\" ASC;" + + self.assertEqual(cc.queen(self.params), ans) + + def test_get_query(self): + """Test get_query.""" + + ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ + "j.\"cartodb_id\" FROM \"a_list\" As j WHERE j.\"andy\" IS " \ + "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ + "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ + "OFFSET 1 ) ) As neighbors FROM \"a_list\" As i WHERE " \ + "i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(cc.get_query('knn', self.params), ans) + + def test_get_attributes(self): + """Test get_attributes.""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight.""" + + self.assertEqual(True, True) + + + def test_quad_position(self): + """Test lisa_sig_vals.""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('table', 'value', 0.05, 5, 99, 'the_geom', 'cartodb_id', 'knn') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('table', 'numerator', 'denominator', 0.05, 5, 99, 'the_geom', 'cartodb_id', 'knn') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) From 5a7d3178dd43a34a0cea2a7dc29360c710e84c13 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 19:22:21 +0100 Subject: [PATCH 24/81] Release 0.0.2 This version is the first with the new versioning approach which uses separate per-version Pyhton virtual enironments. --- NEWS.md | 6 +- release/crankshaft--0.0.1--0.0.2.sql | 74 ++++ release/crankshaft--0.0.2--0.0.1.sql | 44 +++ release/crankshaft--0.0.2.sql | 186 ++++++++++ release/crankshaft.control | 2 +- .../0.0.2/crankshaft/crankshaft/__init__.py | 2 + .../crankshaft/clustering/__init__.py | 1 + .../crankshaft/crankshaft/clustering/moran.py | 321 ++++++++++++++++++ .../crankshaft/crankshaft/random_seeds.py | 10 + release/python/0.0.2/crankshaft/setup.py | 48 +++ .../0.0.2/crankshaft/test/fixtures/moran.json | 52 +++ .../crankshaft/test/fixtures/neighbors.json | 54 +++ .../python/0.0.2/crankshaft/test/helper.py | 13 + .../python/0.0.2/crankshaft/test/mock_plpy.py | 34 ++ .../crankshaft/test/test_clustering_moran.py | 144 ++++++++ src/pg/crankshaft.control | 2 +- 16 files changed, 990 insertions(+), 3 deletions(-) create mode 100644 release/crankshaft--0.0.1--0.0.2.sql create mode 100644 release/crankshaft--0.0.2--0.0.1.sql create mode 100644 release/crankshaft--0.0.2.sql create mode 100644 release/python/0.0.2/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.0.2/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.0.2/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.0.2/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.0.2/crankshaft/setup.py create mode 100644 release/python/0.0.2/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.0.2/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.0.2/crankshaft/test/helper.py create mode 100644 release/python/0.0.2/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.0.2/crankshaft/test/test_clustering_moran.py diff --git a/NEWS.md b/NEWS.md index 201da51..0b8c2da 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ -0.0.1 (2016-03-15) +0.0.2 (2016-03-16) +------------------ +* New versioning approach using per-version Python virtual environments + +0.0.1 (2016-02-22) ------------------ * Preliminar release diff --git a/release/crankshaft--0.0.1--0.0.2.sql b/release/crankshaft--0.0.1--0.0.2.sql new file mode 100644 index 0000000..60c6ecd --- /dev/null +++ b/release/crankshaft--0.0.1--0.0.2.sql @@ -0,0 +1,74 @@ +CREATE OR REPLACE FUNCTION cdb_crankshaft.cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.2'::text; +$$ language 'sql' STABLE STRICT; + +CREATE OR REPLACE FUNCTION cdb_crankshaft._cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +CREATE OR REPLACE FUNCTION cdb_crankshaft._cdb_crankshaft_virtualenvs_path() +RETURNS text +AS $$ + BEGIN + RETURN '/home/ubuntu/crankshaft/envs'; + END; +$$ language plpgsql IMMUTABLE STRICT; + +CREATE OR REPLACE FUNCTION cdb_crankshaft._cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + import os + # plpy.notice('%',str(os.environ)) + # activate virtualenv + crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] + base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] + default_venv_path = os.path.join(base_path, crankshaft_version) + venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), dict(__file__=activate_path)) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION +cdb_crankshaft._cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I +CREATE OR REPLACE FUNCTION +cdb_crankshaft.cdb_moran_local ( + t TEXT, + attr TEXT, + significance float DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION +cdb_crankshaft.cdb_moran_local_rate(t TEXT, + numerator TEXT, + denominator TEXT, + significance FLOAT DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; diff --git a/release/crankshaft--0.0.2--0.0.1.sql b/release/crankshaft--0.0.2--0.0.1.sql new file mode 100644 index 0000000..1bfdcba --- /dev/null +++ b/release/crankshaft--0.0.2--0.0.1.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION +cdb_crankshaft._cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION +cdb_crankshaft.cdb_moran_local ( + t TEXT, + attr TEXT, + significance float DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +CREATE OR REPLACE FUNCTION +cdb_crankshaft.cdb_moran_local_rate(t TEXT, + numerator TEXT, + denominator TEXT, + significance FLOAT DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +DROP FUNCTION IF EXISTS cdb_crankshaft.cdb_crankshaft_version(); +DROP FUNCTION IF EXISTS cdb_crankshaft._cdb_crankshaft_internal_version(); +DROP FUNCTION IF EXISTS cdb_crankshaft._cdb_crankshaft_activate_py(); diff --git a/release/crankshaft--0.0.2.sql b/release/crankshaft--0.0.2.sql new file mode 100644 index 0000000..daf8d27 --- /dev/null +++ b/release/crankshaft--0.0.2.sql @@ -0,0 +1,186 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.2'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() +RETURNS text +AS $$ + BEGIN + -- RETURN '/opt/virtualenvs/crankshaft'; + RETURN '/home/ubuntu/crankshaft/envs'; + END; +$$ language plpgsql IMMUTABLE STRICT; + +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + import os + # plpy.notice('%',str(os.environ)) + # activate virtualenv + crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] + base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] + default_venv_path = os.path.join(base_path, crankshaft_version) + venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), dict(__file__=activate_path)) +$$ LANGUAGE plpythonu; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I +CREATE OR REPLACE FUNCTION + cdb_moran_local ( + t TEXT, + attr TEXT, + significance float DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate +CREATE OR REPLACE FUNCTION + cdb_moran_local_rate(t TEXT, + numerator TEXT, + denominator TEXT, + significance FLOAT DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 74777ac..49c0d22 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.1' +default_version = '0.0.2' requires = 'plpythonu, postgis, cartodb' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.0.2/crankshaft/crankshaft/__init__.py b/release/python/0.0.2/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..d07e330 --- /dev/null +++ b/release/python/0.0.2/crankshaft/crankshaft/__init__.py @@ -0,0 +1,2 @@ +import random_seeds +import clustering diff --git a/release/python/0.0.2/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.0.2/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..0df080f --- /dev/null +++ b/release/python/0.0.2/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1 @@ +from moran import * diff --git a/release/python/0.0.2/crankshaft/crankshaft/clustering/moran.py b/release/python/0.0.2/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..8882235 --- /dev/null +++ b/release/python/0.0.2/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,321 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import numpy as np +import pysal as ps +import plpy + +# High level interface --------------------------------------- + +def moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + # TODO: ensure that the significance output can be smaller that 1e-3 (0.001) + # TODO: make a wishlist of output features (zscores, pvalues, raw local lisa, what else?) + + plpy.notice('** Constructing query') + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "attr1": attr, + "geom_col": geom_column, + "table": t, + "num_ngbrs": num_ngbrs} + + q = get_query(w_type, qvals) + + try: + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Exiting function') + return zip([None], [None], [None], [None]) + + y = get_attributes(r, 1) + w = get_weight(r, w_type) + + # calculate LISA values + lisa = ps.Moran_Local(y, w) + + # find units of significance + lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) + + +def moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type): + """ + Moran's I Local Rate + Andy Eschbacher + """ + + plpy.notice('** Constructing query') + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_column, + "table": t, + "num_ngbrs": num_ngbrs} + + q = get_query(w_type, qvals) + + try: + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Error: %s' % plpy.SPIError) + plpy.notice('** Exiting function') + return zip([None], [None], [None], [None]) + + plpy.notice('r.nrows() = %d' % r.nrows()) + + ## collect attributes + numer = get_attributes(r, 1) + denom = get_attributes(r, 2) + + w = get_weight(r, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, w, permutations=permutations) + + # find units of significance + lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + + plpy.notice('** Finished calculations') + + ## TODO: Decide on which return values here + return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order, lisa.y) + +def moran_local_bv(t, attr1, attr2, significance, num_ngbrs, permutations, geom_column, id_col, w_type): + plpy.notice('** Constructing query') + + qvals = {"num_ngbrs": num_ngbrs, + "attr1": attr1, + "attr2": attr2, + "table": t, + "geom_col": geom_column, + "id_col": id_col} + + q = get_query(w_type, qvals) + + try: + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Error: %s' % plpy.SPIError) + plpy.notice('** Exiting function') + return zip([None], [None], [None], [None]) + + ## collect attributes + attr1_vals = get_attributes(r, 1) + attr2_vals = get_attributes(r, 2) + + # create weights + w = get_weight(r, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, w) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) + + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + :param coord (int): quadrant of a specific measurement + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + :param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + :param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM \"{table}\" As j " \ + "WHERE %(attr_where_j)s " \ + "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs} OFFSET 1 ) " \ + ") As neighbors " \ + "FROM \"{table}\" As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + :param params: dict of information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM \"{table}\" As j " \ + "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM \"{table}\" As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_query(w_type, query_vals): + """Return requested query. + :param w_type: type of neighbors to calculate (knn or queen) + :param query_vals: values used to construct the query + """ + + if w_type == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +def get_attributes(query_res, attr_num): + """ + :param query_res: query results with attributes and neighbors + :param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +## Build weight object +def get_weight(query_res, w_type='queen', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + :param query_res: query results with attributes and neighbors + """ + if w_type == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + elif w_type == 'queen': + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def quad_position(quads): + """ + Produce Moran's I classification based of n + """ + + lisa_sig = np.array([map_quads(q) for q in quads]) + + return lisa_sig + +def lisa_sig_vals(pvals, quads, threshold): + """ + Produce Moran's I classification based of n + """ + + sig = (pvals <= threshold) + + lisa_sig = np.empty(len(sig), np.chararray) + + for idx, val in enumerate(sig): + if val: + lisa_sig[idx] = map_quads(quads[idx]) + else: + lisa_sig[idx] = 'Not significant' + + return lisa_sig diff --git a/release/python/0.0.2/crankshaft/crankshaft/random_seeds.py b/release/python/0.0.2/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..b7c8eed --- /dev/null +++ b/release/python/0.0.2/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,10 @@ +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.0.2/crankshaft/setup.py b/release/python/0.0.2/crankshaft/setup.py new file mode 100644 index 0000000..7a709a1 --- /dev/null +++ b/release/python/0.0.2/crankshaft/setup.py @@ -0,0 +1,48 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.0.2', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + install_requires=['pysal==1.9.1'], + + requires=['pysal', 'numpy' ], + + test_suite='test' +) diff --git a/release/python/0.0.2/crankshaft/test/fixtures/moran.json b/release/python/0.0.2/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..0530c18 --- /dev/null +++ b/release/python/0.0.2/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "Not significant"], +[0.6152779669180425, "Not significant"], +[-0.14657336660125297, "Not significant"], +[0.6967858120189607, "Not significant"], +[0.07949310115714454, "Not significant"], +[0.4703198759258987, "Not significant"], +[0.4421125200498064, "Not significant"], +[0.5724288737143592, "Not significant"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "Not significant"], +[-0.01466729201304962, "Not significant"], +[0.3481559372544409, "Not significant"], +[0.06547094736902978, "Not significant"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "Not significant"], +[0.15971286468915544, "Not significant"], +[1.0543588860308968, "Not significant"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "Not significant"], +[0.08438455015300014, "Not significant"], +[0.06547094736902978, "Not significant"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "Not significant"], +[0.795275137550483, "Not significant"], +[0.18562939195219, "LL"], +[0.3010757406693439, "Not significant"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "Not significant"], +[-0.07116352791516614, "Not significant"], +[-0.09945240794119009, "Not significant"], +[0.18562939195219, "LL"], +[0.1832733440191868, "Not significant"], +[-0.39054253768447705, "Not significant"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "Not significant"], +[0.2584386102554792, "Not significant"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "Not significant"], +[0.051367269430983485, "Not significant"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "Not significant"], +[0.04191046803899837, "Not significant"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "Not significant"], +[0.5410013139159929, "Not significant"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] diff --git a/release/python/0.0.2/crankshaft/test/fixtures/neighbors.json b/release/python/0.0.2/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.0.2/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.0.2/crankshaft/test/helper.py b/release/python/0.0.2/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.0.2/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.0.2/crankshaft/test/mock_plpy.py b/release/python/0.0.2/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..63c88f6 --- /dev/null +++ b/release/python/0.0.2/crankshaft/test/mock_plpy.py @@ -0,0 +1,34 @@ +import re + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.0.2/crankshaft/test/test_clustering_moran.py b/release/python/0.0.2/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..2e730a1 --- /dev/null +++ b/release/python/0.0.2/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,144 @@ +import unittest +import numpy as np + +import unittest + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions.""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "table": "a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads.""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_query_attr_select(self): + """Test query_attr_select.""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(cc.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test query_attr_where.""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND "\ + "idx_replace.\"{attr2}\" IS NOT NULL AND "\ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(cc.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn function.""" + + ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM \"a_list\" As j WHERE j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 OFFSET 1 ) ) " \ + "As neighbors FROM \"a_list\" As i WHERE i.\"andy\" IS NOT " \ + "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ + "BY i.\"cartodb_id\" ASC;" + + self.assertEqual(cc.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors function.""" + + ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ + "j.\"cartodb_id\" FROM \"a_list\" As j WHERE ST_Touches(" \ + "i.\"the_geom\", j.\"the_geom\") AND j.\"andy\" IS NOT NULL " \ + "AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0)) As " \ + "neighbors FROM \"a_list\" As i WHERE i.\"andy\" IS NOT NULL " \ + "AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER BY " \ + "i.\"cartodb_id\" ASC;" + + self.assertEqual(cc.queen(self.params), ans) + + def test_get_query(self): + """Test get_query.""" + + ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ + "j.\"cartodb_id\" FROM \"a_list\" As j WHERE j.\"andy\" IS " \ + "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ + "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ + "OFFSET 1 ) ) As neighbors FROM \"a_list\" As i WHERE " \ + "i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(cc.get_query('knn', self.params), ans) + + def test_get_attributes(self): + """Test get_attributes.""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight.""" + + self.assertEqual(True, True) + + + def test_quad_position(self): + """Test lisa_sig_vals.""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('table', 'value', 0.05, 5, 99, 'the_geom', 'cartodb_id', 'knn') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('table', 'numerator', 'denominator', 0.05, 5, 99, 'the_geom', 'cartodb_id', 'knn') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 74777ac..49c0d22 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.1' +default_version = '0.0.2' requires = 'plpythonu, postgis, cartodb' superuser = true schema = cdb_crankshaft From 58c141d2175d751c5dc29128906b8229297296a2 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Wed, 16 Mar 2016 19:40:06 +0100 Subject: [PATCH 25/81] Allow to pass free queries as `select * from table limit 100` in moran --- src/pg/test/expected/02_moran_test.out | 130 ++---------------- src/pg/test/sql/02_moran_test.sql | 4 +- .../crankshaft/crankshaft/clustering/moran.py | 20 +-- .../crankshaft/test/test_clustering_moran.py | 14 +- 4 files changed, 33 insertions(+), 135 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 6ca4900..66ccaaa 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -121,70 +121,18 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints - JOIN cdb_crankshaft.cdb_moran_local('ppoints', 'value') m + JOIN cdb_crankshaft.cdb_moran_local('SELECT * FROM ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; NOTICE: ** Constructing query CONTEXT: PL/Python function "cdb_moran_local" -NOTICE: ** Query returned with 52 rows +NOTICE: ** Query failed: "SELECT i."cartodb_id" As id, i."value"::numeric As attr1, (SELECT ARRAY(SELECT j."cartodb_id" FROM "(SELECT * FROM ppoints)" As j WHERE j."value" IS NOT NULL ORDER BY j."the_geom" <-> i."the_geom" ASC LIMIT 5 OFFSET 1 ) ) As neighbors FROM "(SELECT * FROM ppoints)" As i WHERE i."value" IS NOT NULL ORDER BY i."cartodb_id" ASC;" CONTEXT: PL/Python function "cdb_moran_local" -NOTICE: ** Finished calculations +NOTICE: ** Exiting function CONTEXT: PL/Python function "cdb_moran_local" - code | quads -------+----------------- - 01 | HH - 02 | HL - 03 | Not significant - 04 | Not significant - 05 | Not significant - 06 | Not significant - 07 | Not significant - 08 | Not significant - 09 | Not significant - 10 | Not significant - 11 | LL - 12 | Not significant - 13 | Not significant - 14 | Not significant - 15 | Not significant - 16 | HH - 17 | Not significant - 18 | Not significant - 19 | Not significant - 20 | HH - 21 | LL - 22 | Not significant - 23 | Not significant - 24 | Not significant - 25 | HH - 26 | HH - 27 | Not significant - 28 | Not significant - 29 | LL - 30 | Not significant - 31 | HH - 32 | Not significant - 33 | Not significant - 34 | Not significant - 35 | LL - 36 | Not significant - 37 | Not significant - 38 | HL - 39 | Not significant - 40 | Not significant - 41 | HL - 42 | LH - 43 | Not significant - 44 | Not significant - 45 | LH - 46 | Not significant - 47 | Not significant - 48 | HH - 49 | Not significant - 50 | Not significant - 51 | LL - 52 | LL -(52 rows) + code | quads +------+------- +(0 rows) SELECT cdb_crankshaft._cdb_random_seeds(1234); _cdb_random_seeds @@ -194,67 +142,17 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.cdb_moran_local_rate('ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.cdb_moran_local_rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; NOTICE: ** Constructing query CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Query returned with 51 rows +NOTICE: ** Query failed: "SELECT i."cartodb_id" As id, i."denominator"::numeric As attr1, i."numerator"::numeric As attr2, (SELECT ARRAY(SELECT j."cartodb_id" FROM "(SELECT * FROM ppoints2)" As j WHERE j."denominator" IS NOT NULL AND j."numerator" IS NOT NULL AND j."numerator" <> 0 ORDER BY j."the_geom" <-> i."the_geom" ASC LIMIT 5 OFFSET 1 ) ) As neighbors FROM "(SELECT * FROM ppoints2)" As i WHERE i."denominator" IS NOT NULL AND i."numerator" IS NOT NULL AND i."numerator" <> 0 ORDER BY i."cartodb_id" ASC;" CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Finished calculations +NOTICE: ** Error: CONTEXT: PL/Python function "cdb_moran_local_rate" - code | quads -------+----------------- - 01 | LL - 02 | Not significant - 03 | Not significant - 04 | Not significant - 05 | Not significant - 06 | Not significant - 07 | Not significant - 08 | Not significant - 09 | LL - 10 | Not significant - 11 | HH - 12 | Not significant - 13 | Not significant - 14 | Not significant - 15 | Not significant - 16 | Not significant - 17 | LL - 18 | Not significant - 19 | Not significant - 20 | LL - 21 | Not significant - 22 | Not significant - 23 | Not significant - 24 | Not significant - 25 | LL - 26 | LL - 27 | Not significant - 28 | Not significant - 29 | LH - 30 | Not significant - 31 | LL - 32 | Not significant - 33 | Not significant - 34 | Not significant - 35 | LH - 36 | Not significant - 37 | Not significant - 38 | LH - 39 | Not significant - 40 | Not significant - 41 | LH - 42 | HL - 43 | Not significant - 44 | Not significant - 45 | LL - 46 | Not significant - 47 | Not significant - 48 | LL - 49 | Not significant - 50 | Not significant - 51 | Not significant -(51 rows) - +NOTICE: ** Exiting function +CONTEXT: PL/Python function "cdb_moran_local_rate" +ERROR: length of returned sequence did not match number of columns in row +CONTEXT: while creating return value +PL/Python function "cdb_moran_local_rate" diff --git a/src/pg/test/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql index f2e7257..a0bc401 100644 --- a/src/pg/test/sql/02_moran_test.sql +++ b/src/pg/test/sql/02_moran_test.sql @@ -8,7 +8,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints - JOIN cdb_crankshaft.cdb_moran_local('ppoints', 'value') m + JOIN cdb_crankshaft.cdb_moran_local('SELECT * FROM ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; @@ -16,6 +16,6 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.cdb_moran_local_rate('ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.cdb_moran_local_rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 8882235..9dd976e 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -11,7 +11,7 @@ import plpy # High level interface --------------------------------------- -def moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type): +def moran_local(subquery, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type): """ Moran's I implementation for PL/Python Andy Eschbacher @@ -27,7 +27,7 @@ def moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_ qvals = {"id_col": id_col, "attr1": attr, "geom_col": geom_column, - "table": t, + "subquery": subquery, "num_ngbrs": num_ngbrs} q = get_query(w_type, qvals) @@ -54,7 +54,7 @@ def moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_ return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) -def moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type): +def moran_local_rate(subquery, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type): """ Moran's I Local Rate Andy Eschbacher @@ -69,7 +69,7 @@ def moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permuta "numerator": numerator, "denominator": denominator, "geom_col": geom_column, - "table": t, + "subquery": subquery, "num_ngbrs": num_ngbrs} q = get_query(w_type, qvals) @@ -171,7 +171,7 @@ def query_attr_select(params): """ attrs = [k for k in params - if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs')] + if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs', 'subquery')] template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " @@ -187,7 +187,7 @@ def query_attr_where(params): Create portion of WHERE clauses for weeding out NULL-valued geometries """ attrs = sorted([k for k in params - if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs')]) + if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs', 'subquery')]) attr_string = [] @@ -217,12 +217,12 @@ def knn(params): "i.\"{id_col}\" As id, " \ "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM \"{table}\" As j " \ + "FROM \"({subquery})\" As j " \ "WHERE %(attr_where_j)s " \ "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ "LIMIT {num_ngbrs} OFFSET 1 ) " \ ") As neighbors " \ - "FROM \"{table}\" As i " \ + "FROM \"({subquery})\" As i " \ "WHERE " \ "%(attr_where_i)s " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements @@ -245,11 +245,11 @@ def queen(params): "i.\"{id_col}\" As id, " \ "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM \"{table}\" As j " \ + "FROM \"({subquery})\" As j " \ "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ "%(attr_where_j)s)" \ ") As neighbors " \ - "FROM \"{table}\" As i " \ + "FROM \"({subquery})\" As i " \ "WHERE " \ "%(attr_where_i)s " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 2e730a1..b48b8d6 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -23,7 +23,7 @@ class MoranTest(unittest.TestCase): self.params = {"id_col": "cartodb_id", "attr1": "andy", "attr2": "jay_z", - "table": "a_list", + "subquery": "SELECT * FROM a_list", "geom_col": "the_geom", "num_ngbrs": 321} self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) @@ -60,10 +60,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM \"a_list\" As j WHERE j.\"andy\" IS NOT NULL AND " \ + "FROM \"(SELECT * FROM a_list)\" As j WHERE j.\"andy\" IS NOT NULL AND " \ "j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 ORDER BY " \ "j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 OFFSET 1 ) ) " \ - "As neighbors FROM \"a_list\" As i WHERE i.\"andy\" IS NOT " \ + "As neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE i.\"andy\" IS NOT " \ "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ "BY i.\"cartodb_id\" ASC;" @@ -74,10 +74,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM \"a_list\" As j WHERE ST_Touches(" \ + "j.\"cartodb_id\" FROM \"(SELECT * FROM a_list)\" As j WHERE ST_Touches(" \ "i.\"the_geom\", j.\"the_geom\") AND j.\"andy\" IS NOT NULL " \ "AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0)) As " \ - "neighbors FROM \"a_list\" As i WHERE i.\"andy\" IS NOT NULL " \ + "neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE i.\"andy\" IS NOT NULL " \ "AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER BY " \ "i.\"cartodb_id\" ASC;" @@ -88,10 +88,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM \"a_list\" As j WHERE j.\"andy\" IS " \ + "j.\"cartodb_id\" FROM \"(SELECT * FROM a_list)\" As j WHERE j.\"andy\" IS " \ "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ - "OFFSET 1 ) ) As neighbors FROM \"a_list\" As i WHERE " \ + "OFFSET 1 ) ) As neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE " \ "i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ "i.\"jay_z\" <> 0 ORDER BY i.\"cartodb_id\" ASC;" From 3b31da783a9551b5a462cc8fa4f35080f9f1fd6e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 21 Mar 2016 08:40:37 -0400 Subject: [PATCH 26/81] adding mac ds_store ignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 8d2abce..1161ea2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ envs/ *.pyc +.DS_Store From 0aa4d0a50e09caf09a4489f10378be6f272ddaae Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Mon, 21 Mar 2016 08:51:10 -0400 Subject: [PATCH 27/81] typo fixes, linking, etc. --- CONTRIBUTING.md | 2 +- README.md | 10 +++++----- src/py/README.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bcdde4a..9cfb951 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Development process -Please read the Working Process/Quickstart Guide in README.md first. +Please read the Working Process/Quickstart Guide in [README.md](https://github.com/CartoDB/crankshaft/blob/master/README.md) first. For any modification of crankshaft, such as adding new features, refactoring or bug-fixing, topic branch must be created out of the `develop` diff --git a/README.md b/README.md index 3ecb1d0..68a64fb 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. ## Requirements * pip, virtualenv, PostgreSQL -* python-scipy system package (see src/py/README.md) +* python-scipy system package (see [src/py/README.md](https://github.com/CartoDB/crankshaft/blob/master/src/py/README.md)) # Working Process -- Quickstart Guide @@ -33,7 +33,7 @@ deployed in production. Developers shall create a new topic branch from `develop` for any new feature or bugfix and commit their changes to it and eventually merge back into the `develop` branch. When a new release is required a Pull Request -will be open againt the `develop` branch. +will be open against the `develop` branch. The `develop` pull requests will be handled by the release manage, who will merge into master where new releases are prepared and tagged. @@ -43,7 +43,7 @@ and developers must not commit or merge into it. ## Development Guidelines For a detailed description of the development process please see -the CONTRIBUTING.md guide. +the [CONTRIBUTING.md](https://github.com/CartoDB/crankshaft/blob/master/CONTRIBUTING.md) guide. Any modification to the source code (`src/pg/sql` for the SQL extension, `src/py/crankshaft` for the Python package) shall always be done @@ -52,7 +52,7 @@ in a topic branch created from the `develop` branch. Tests, documentation and peer code reviewing are required for all modifications. -The tests (both for SQL and Pyhton) are executed by running, +The tests (both for SQL and Python) are executed by running, from the top directory: ``` @@ -67,5 +67,5 @@ branch. ## Release The release and deployment process is described in the -RELEASE.md guide and it is the responsibility of the designated +[RELEASE.md](https://github.com/CartoDB/crankshaft/blob/master/RELEASE.md) guide and it is the responsibility of the designated release manager. diff --git a/src/py/README.md b/src/py/README.md index b9bf64d..29a3145 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -8,7 +8,7 @@ cd crankshaft nosetests test/ ``` -## Notes about python dependencies +## Notes about Python dependencies * This extension is targeted at production databases. Therefore certain restrictions must be assumed about the production environment vs other experimental environments. * We're using `pip` and `virtualenv` to generate a suitable isolated environment for python code that has all the dependencies * Every dependency should be: From 3eda8ecd166d21b79aa2ad8ad537c2e3ffdef2a3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 22 Mar 2016 10:34:22 -0400 Subject: [PATCH 28/81] new signatures for moran (w/o significance) --- src/pg/sql/10_moran.sql | 104 ++++++++++++++++++++++++++++++---------- 1 file changed, 78 insertions(+), 26 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index 49c70c2..dba7069 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -1,37 +1,89 @@ --- Moran's I +-- Moran's I (global) CREATE OR REPLACE FUNCTION - cdb_moran_local ( - t TEXT, - attr TEXT, - significance float DEFAULT 0.05, - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn') + cdb_moran ( + subquery TEXT, + attr_name TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) + # TODO: use named parameters or a dictionary + return moran(subquery, attr, num_ngbrs, permutations, geom_col, id_col, w_type) $$ LANGUAGE plpythonu; +-- Moran's I Local +CREATE OR REPLACE FUNCTION + cdb_moran_local ( + subquery TEXT, + attr TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + +-- Moran's I Rate (global) +CREATE OR REPLACE FUNCTION + cdb_moran_rate ( + subquery TEXT, + numerator TEXT, + denominator TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + + -- Moran's I Local Rate CREATE OR REPLACE FUNCTION - cdb_moran_local_rate(t TEXT, - numerator TEXT, - denominator TEXT, - significance FLOAT DEFAULT 0.05, - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn') -RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) + cdb_moran_local_rate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate - # TODO: use named parameters or a dictionary - return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; + +-- -- Moran's I Local Bivariate +-- CREATE OR REPLACE FUNCTION +-- cdb_moran_local_bv( +-- subquery TEXT, +-- attr1 TEXT, +-- attr2 TEXT, +-- permutations INT DEFAULT 99, +-- geom_col TEXT DEFAULT 'the_geom', +-- id_col TEXT DEFAULT 'cartodb_id', +-- w_type TEXT DEFAULT 'knn', +-- num_ngbrs INT DEFAULT 5) +-- RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +-- AS $$ +-- from crankshaft.clustering import moran_local_bv +-- # TODO: use named parameters or a dictionary +-- return moran_local_bv(t, attr1, attr2, permutations, geom_col, id_col, w_type, num_ngbrs) +-- $$ LANGUAGE plpythonu; \ No newline at end of file From 1578b17eb87c60c91e67980f829ed58a9a54b2ce Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 22 Mar 2016 10:42:06 -0400 Subject: [PATCH 29/81] updated function flow without significance --- .../crankshaft/crankshaft/clustering/moran.py | 163 +++++++++++++----- 1 file changed, 117 insertions(+), 46 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 9dd976e..2d65db5 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -11,7 +11,51 @@ import plpy # High level interface --------------------------------------- -def moran_local(subquery, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type): +def moran(subquery, attr_name, permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (global) + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": attr_name, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + q = get_query(w_type, qvals) + + plpy.notice('** Query: %s' % q) + + try: + r = plpy.execute(q) + if (len(r) == 0) & (w_type != 'knn'): + plpy.notice('** Query returned with 0 rows, trying kNN weights') + q = get_query('knn', qvals) + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.error('** Moran rate failed executing query to build weight object') + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Error: %s' % plpy.SPIError) + plpy.notice('** Exiting function') + return zip([None], [None]) + + ## if there are no neighbors, exit + if len(r) == 0: + return zip([None], [None]) + + ## collect attributes + attr_vals = get_attributes(r, 1) + + ## calculate weights + weight = get_weight(r, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, permutations=permutations) + + return zip([moran_global.I],[moran_global.EI]) + +def moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs): """ Moran's I implementation for PL/Python Andy Eschbacher @@ -25,8 +69,8 @@ def moran_local(subquery, attr, significance, num_ngbrs, permutations, geom_colu # resulting in a collection of not as near neighbors qvals = {"id_col": id_col, - "attr1": attr, - "geom_col": geom_column, + "attr1": attr, + "geom_col": geom_col, "subquery": subquery, "num_ngbrs": num_ngbrs} @@ -38,23 +82,68 @@ def moran_local(subquery, attr, significance, num_ngbrs, permutations, geom_colu except plpy.SPIError: plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Exiting function') - return zip([None], [None], [None], [None]) + return zip([None], [None], [None], [None], [None]) y = get_attributes(r, 1) w = get_weight(r, w_type) # calculate LISA values - lisa = ps.Moran_Local(y, w) + lisa = ps.esda.moran.Moran_Local(y, w) - # find units of significance - lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + # find quadrants for each geometry + quads = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + return zip(lisa.Is, quads, lisa.p_sim, w.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": numerator, + "attr2": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + q = get_query(w_type, qvals) + + plpy.notice('** Query: %s' % q) + + try: + r = plpy.execute(q) + if len(r) == 0: + plpy.notice('** Query returned with 0 rows, trying kNN weights') + q = get_query('knn', qvals) + r = plpy.execute(q) + plpy.notice('** Query returned with %d rows' % len(r)) + except plpy.SPIError: + plpy.error('Moran rate failed executing query to build weight object') + plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Error: %s' % plpy.SPIError) + plpy.notice('** Exiting function') + return zip([None], [None]) + + ## if there are no values returned, exit + if len(r) == 0: + return zip([None], [None]) + + ## collect attributes + numer = get_attributes(r, 1) + denom = get_attributes(r, 2) + + w = get_weight(r, w_type, num_ngbrs) + + ## calculate moran global rate + mr = ps.esda.moran.Moran_Rate(numer, denom, w, permutations=permutations) plpy.notice('** Finished calculations') - return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) + return zip([mr.I],[mr.EI]) - -def moran_local_rate(subquery, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type): +def moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs): """ Moran's I Local Rate Andy Eschbacher @@ -68,7 +157,7 @@ def moran_local_rate(subquery, numerator, denominator, significance, num_ngbrs, qvals = {"id_col": id_col, "numerator": numerator, "denominator": denominator, - "geom_col": geom_column, + "geom_col": geom_col, "subquery": subquery, "num_ngbrs": num_ngbrs} @@ -81,7 +170,7 @@ def moran_local_rate(subquery, numerator, denominator, significance, num_ngbrs, plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Error: %s' % plpy.SPIError) plpy.notice('** Exiting function') - return zip([None], [None], [None], [None]) + return zip([None], [None], [None], [None], [None]) plpy.notice('r.nrows() = %d' % r.nrows()) @@ -95,21 +184,20 @@ def moran_local_rate(subquery, numerator, denominator, significance, num_ngbrs, lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, w, permutations=permutations) # find units of significance - lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + quads = quad_position(lisa.q) plpy.notice('** Finished calculations') - ## TODO: Decide on which return values here - return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order, lisa.y) + return zip(lisa.Is, quads, lisa.p_sim, w.id_order, lisa.y) -def moran_local_bv(t, attr1, attr2, significance, num_ngbrs, permutations, geom_column, id_col, w_type): +def moran_local_bv(subquery, attr1, attr2, permutations, geom_col, id_col, w_type, num_ngbrs): plpy.notice('** Constructing query') qvals = {"num_ngbrs": num_ngbrs, "attr1": attr1, "attr2": attr2, - "table": t, - "geom_col": geom_column, + "subquery": subquery, + "geom_col": geom_col, "id_col": id_col} q = get_query(w_type, qvals) @@ -136,7 +224,7 @@ def moran_local_bv(t, attr1, attr2, significance, num_ngbrs, permutations, geom_ plpy.notice("len of Is: %d" % len(lisa.Is)) # find clustering of significance - lisa_sig = lisa_sig_vals(lisa.p_sim, lisa.q, significance) + lisa_sig = quad_position(lisa.q) plpy.notice('** Finished calculations') @@ -171,7 +259,7 @@ def query_attr_select(params): """ attrs = [k for k in params - if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs', 'subquery')] + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " @@ -187,7 +275,7 @@ def query_attr_where(params): Create portion of WHERE clauses for weeding out NULL-valued geometries """ attrs = sorted([k for k in params - if k not in ('id_col', 'geom_col', 'table', 'num_ngbrs', 'subquery')]) + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) attr_string = [] @@ -217,12 +305,12 @@ def knn(params): "i.\"{id_col}\" As id, " \ "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM \"({subquery})\" As j " \ + "FROM ({subquery}) As j " \ "WHERE %(attr_where_j)s " \ "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ "LIMIT {num_ngbrs} OFFSET 1 ) " \ ") As neighbors " \ - "FROM \"({subquery})\" As i " \ + "FROM ({subquery}) As i " \ "WHERE " \ "%(attr_where_i)s " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements @@ -245,11 +333,11 @@ def queen(params): "i.\"{id_col}\" As id, " \ "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM \"({subquery})\" As j " \ + "FROM ({subquery}) As j " \ "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ "%(attr_where_j)s)" \ ") As neighbors " \ - "FROM \"({subquery})\" As i " \ + "FROM ({subquery}) As i " \ "WHERE " \ "%(attr_where_i)s " \ "ORDER BY i.\"{id_col}\" ASC;" % replacements @@ -285,10 +373,10 @@ def get_weight(query_res, w_type='queen', num_ngbrs=5): if w_type == 'knn': row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs weights = {x['id']: row_normed_weights for x in query_res} - elif w_type == 'queen': + else: weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) - if len(x['neighbors']) > 0 - else [] for x in query_res} + if len(x['neighbors']) > 0 + else [] for x in query_res} neighbors = {x['id']: x['neighbors'] for x in query_res} @@ -301,21 +389,4 @@ def quad_position(quads): lisa_sig = np.array([map_quads(q) for q in quads]) - return lisa_sig - -def lisa_sig_vals(pvals, quads, threshold): - """ - Produce Moran's I classification based of n - """ - - sig = (pvals <= threshold) - - lisa_sig = np.empty(len(sig), np.chararray) - - for idx, val in enumerate(sig): - if val: - lisa_sig[idx] = map_quads(quads[idx]) - else: - lisa_sig[idx] = 'Not significant' - - return lisa_sig + return lisa_sig \ No newline at end of file From eecbe39547ff2e8aa3e36050b5cf7d39452dd032 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 22 Mar 2016 10:42:44 -0400 Subject: [PATCH 30/81] updating tests --- src/pg/test/expected/02_moran_test.out | 122 ++++++++++++++++-- src/py/crankshaft/test/fixtures/moran.json | 70 +++++----- .../crankshaft/test/test_clustering_moran.py | 19 +-- 3 files changed, 157 insertions(+), 54 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 66ccaaa..92cb218 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -126,13 +126,65 @@ SELECT ppoints.code, m.quads ORDER BY ppoints.code; NOTICE: ** Constructing query CONTEXT: PL/Python function "cdb_moran_local" -NOTICE: ** Query failed: "SELECT i."cartodb_id" As id, i."value"::numeric As attr1, (SELECT ARRAY(SELECT j."cartodb_id" FROM "(SELECT * FROM ppoints)" As j WHERE j."value" IS NOT NULL ORDER BY j."the_geom" <-> i."the_geom" ASC LIMIT 5 OFFSET 1 ) ) As neighbors FROM "(SELECT * FROM ppoints)" As i WHERE i."value" IS NOT NULL ORDER BY i."cartodb_id" ASC;" +NOTICE: ** Query returned with 52 rows CONTEXT: PL/Python function "cdb_moran_local" -NOTICE: ** Exiting function +NOTICE: ** Finished calculations CONTEXT: PL/Python function "cdb_moran_local" code | quads ------+------- -(0 rows) + 01 | HH + 02 | HL + 03 | LL + 04 | LL + 05 | LH + 06 | LL + 07 | HH + 08 | HH + 09 | HH + 10 | LL + 11 | LL + 12 | LL + 13 | HL + 14 | LL + 15 | LL + 16 | HH + 17 | HH + 18 | LL + 19 | HH + 20 | HH + 21 | LL + 22 | HH + 23 | LL + 24 | LL + 25 | HH + 26 | HH + 27 | LL + 28 | HH + 29 | LL + 30 | LL + 31 | HH + 32 | LL + 33 | HL + 34 | LH + 35 | LL + 36 | LL + 37 | HL + 38 | HL + 39 | HH + 40 | HH + 41 | HL + 42 | LH + 43 | LH + 44 | LL + 45 | LH + 46 | LL + 47 | LL + 48 | HH + 49 | LH + 50 | HH + 51 | LL + 52 | LL +(52 rows) SELECT cdb_crankshaft._cdb_random_seeds(1234); _cdb_random_seeds @@ -147,12 +199,62 @@ SELECT ppoints2.code, m.quads ORDER BY ppoints2.code; NOTICE: ** Constructing query CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Query failed: "SELECT i."cartodb_id" As id, i."denominator"::numeric As attr1, i."numerator"::numeric As attr2, (SELECT ARRAY(SELECT j."cartodb_id" FROM "(SELECT * FROM ppoints2)" As j WHERE j."denominator" IS NOT NULL AND j."numerator" IS NOT NULL AND j."numerator" <> 0 ORDER BY j."the_geom" <-> i."the_geom" ASC LIMIT 5 OFFSET 1 ) ) As neighbors FROM "(SELECT * FROM ppoints2)" As i WHERE i."denominator" IS NOT NULL AND i."numerator" IS NOT NULL AND i."numerator" <> 0 ORDER BY i."cartodb_id" ASC;" +NOTICE: ** Query returned with 51 rows CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Error: +NOTICE: ** Finished calculations CONTEXT: PL/Python function "cdb_moran_local_rate" -NOTICE: ** Exiting function -CONTEXT: PL/Python function "cdb_moran_local_rate" -ERROR: length of returned sequence did not match number of columns in row -CONTEXT: while creating return value -PL/Python function "cdb_moran_local_rate" + code | quads +------+------- + 01 | LL + 02 | LH + 03 | HH + 04 | HH + 05 | LL + 06 | HH + 07 | LL + 08 | LL + 09 | LL + 10 | HH + 11 | HH + 12 | HL + 13 | LL + 14 | HH + 15 | LL + 16 | LL + 17 | LL + 18 | LH + 19 | LL + 20 | LL + 21 | HH + 22 | LL + 23 | HL + 24 | LL + 25 | LL + 26 | LL + 27 | LL + 28 | LL + 29 | LH + 30 | HH + 31 | LL + 32 | LL + 33 | LL + 34 | LL + 35 | LH + 36 | HL + 37 | LH + 38 | LH + 39 | LL + 40 | LL + 41 | LH + 42 | HL + 43 | LL + 44 | HL + 45 | LL + 46 | HL + 47 | LL + 48 | LL + 49 | HL + 50 | LL + 51 | HH +(51 rows) + diff --git a/src/py/crankshaft/test/fixtures/moran.json b/src/py/crankshaft/test/fixtures/moran.json index 0530c18..2f75cf1 100644 --- a/src/py/crankshaft/test/fixtures/moran.json +++ b/src/py/crankshaft/test/fixtures/moran.json @@ -1,52 +1,52 @@ [[0.9319096128346788, "HH"], [-1.135787401862846, "HL"], -[0.11732030672508517, "Not significant"], -[0.6152779669180425, "Not significant"], -[-0.14657336660125297, "Not significant"], -[0.6967858120189607, "Not significant"], -[0.07949310115714454, "Not significant"], -[0.4703198759258987, "Not significant"], -[0.4421125200498064, "Not significant"], -[0.5724288737143592, "Not significant"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], [0.8970743435692062, "LL"], -[0.18327334401918674, "Not significant"], -[-0.01466729201304962, "Not significant"], -[0.3481559372544409, "Not significant"], -[0.06547094736902978, "Not significant"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], [0.15482141569329988, "HH"], -[0.4373841193538136, "Not significant"], -[0.15971286468915544, "Not significant"], -[1.0543588860308968, "Not significant"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], [1.7372866900020818, "HH"], [1.091998586053999, "LL"], -[0.1171572584252222, "Not significant"], -[0.08438455015300014, "Not significant"], -[0.06547094736902978, "Not significant"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], [0.15482141569329985, "HH"], [1.1627044812890683, "HH"], -[0.06547094736902978, "Not significant"], -[0.795275137550483, "Not significant"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], [0.18562939195219, "LL"], -[0.3010757406693439, "Not significant"], +[0.3010757406693439, "LL"], [2.8205795942839376, "HH"], -[0.11259190602909264, "Not significant"], -[-0.07116352791516614, "Not significant"], -[-0.09945240794119009, "Not significant"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], [0.18562939195219, "LL"], -[0.1832733440191868, "Not significant"], -[-0.39054253768447705, "Not significant"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], [-0.1672071289487642, "HL"], -[0.3337669247916343, "Not significant"], -[0.2584386102554792, "Not significant"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], [-0.19733845476322634, "HL"], [-0.9379282899805409, "LH"], -[-0.028770969951095866, "Not significant"], -[0.051367269430983485, "Not significant"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], [-0.2172548045913472, "LH"], -[0.05136726943098351, "Not significant"], -[0.04191046803899837, "Not significant"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], [0.7482357030403517, "HH"], -[-0.014585767863118111, "Not significant"], -[0.5410013139159929, "Not significant"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], [1.0223932668429925, "LL"], -[1.4179402898927476, "LL"]] +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index b48b8d6..c7cc71a 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -60,10 +60,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM \"(SELECT * FROM a_list)\" As j WHERE j.\"andy\" IS NOT NULL AND " \ + "FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS NOT NULL AND " \ "j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 ORDER BY " \ "j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 OFFSET 1 ) ) " \ - "As neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE i.\"andy\" IS NOT " \ + "As neighbors FROM (SELECT * FROM a_list) As i WHERE i.\"andy\" IS NOT " \ "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ "BY i.\"cartodb_id\" ASC;" @@ -74,10 +74,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM \"(SELECT * FROM a_list)\" As j WHERE ST_Touches(" \ + "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE ST_Touches(" \ "i.\"the_geom\", j.\"the_geom\") AND j.\"andy\" IS NOT NULL " \ "AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0)) As " \ - "neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE i.\"andy\" IS NOT NULL " \ + "neighbors FROM (SELECT * FROM a_list) As i WHERE i.\"andy\" IS NOT NULL " \ "AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER BY " \ "i.\"cartodb_id\" ASC;" @@ -88,10 +88,10 @@ class MoranTest(unittest.TestCase): ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM \"(SELECT * FROM a_list)\" As j WHERE j.\"andy\" IS " \ + "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS " \ "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ - "OFFSET 1 ) ) As neighbors FROM \"(SELECT * FROM a_list)\" As i WHERE " \ + "OFFSET 1 ) ) As neighbors FROM (SELECT * FROM a_list) As i WHERE " \ "i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ "i.\"jay_z\" <> 0 ORDER BY i.\"cartodb_id\" ASC;" @@ -125,7 +125,7 @@ class MoranTest(unittest.TestCase): data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.moran_local('table', 'value', 0.05, 5, 99, 'the_geom', 'cartodb_id', 'knn') + result = cc.moran_local('table', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) result = [(row[0], row[1]) for row in result] expected = self.moran_data for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): @@ -137,8 +137,9 @@ class MoranTest(unittest.TestCase): data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.moran_local_rate('table', 'numerator', 'denominator', 0.05, 5, 99, 'the_geom', 'cartodb_id', 'knn') + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 99, 'the_geom', 'cartodb_id', 'knn', 5) + print 'result == None? ', result == None result = [(row[0], row[1]) for row in result] expected = self.moran_data for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): - self.assertAlmostEqual(res_val, exp_val) + self.assertAlmostEqual(res_val, exp_val) \ No newline at end of file From 3f20275d3d9a63f43d599093333a51ea0ca35451 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 17:09:52 -0400 Subject: [PATCH 31/81] adopting new format (wip) --- doc/02_moran.md | 100 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/doc/02_moran.md b/doc/02_moran.md index 85384cb..c91eb3f 100644 --- a/doc/02_moran.md +++ b/doc/02_moran.md @@ -1,4 +1,102 @@ -### Moran's I +## Name + +CDB_AreasOfInterest -- returns a table with a cluster/outlier classification, the significance of a classification, an autocorrelation statistic (Local Moran's I), and the geometry id for each geometry in the original dataset. + +## Synopsis + +```sql +table(numeric moran_val, text quadrant, numeric significance, int ids, numeric column_values) CDB_AreasOfInterest(text query, text column_name) + +table(numeric moran_val, text quadrant, numeric significance, int ids, numeric column_values) CDB_AreasOfInterest(text query, text column_name, int permutations, text geom_column, text id_column, text weight_type, int num_ngbrs) +``` + +## Description + +CDB_AreasOfInterest is a table-returning function that classifies the geometries in a table by an attribute and gives a significance for that classification. This information can be used to find "Areas of Interest" by using the correlation of a geometry's attribute with that of its neighbors. Areas can be clusters, outliers, or neither (depending on which significance value is used). + +Inputs: + +* `query` (required): an arbitrary query against tables you have access to (e.g., in your account, shared in your organization, or through the Data Observatory). This string must contain the following columns: an id `INT` (e.g., `cartodb_id`), geometry (e.g., `the_geom`), and the numeric attribute which is specified in `column_name` +* `column_name` (required): column to perform the area of interest analysis tool on. The data must be numeric (e.g., `float`, `int`, etc.) +* `permutations` (optional): used to calculate the significance of a classification. Defaults to 99, which is sufficient in most situations. +* `geom_column` (optional): the name of the geometry column. Data must be of type `geometry`. +* `id_column` (optional): the name of the id column (e.g., `cartodb_id`). Data must be of type `int` or `bigint` and have a unique condition on the data. +* `weight_type` (optional): the type of weight used for determining what defines a neighborhood. Options are `knn` or `queen`. +* `num_ngbrs` (optional): the number of neighbors in a neighborhood around a geometry. Only used if `knn` is chosen above. + +Outputs: + +* `moran_val`: underlying correlation statistic used in analysis +* `quadrant`: human-readable interpretation of classification +* `significance`: significance of classification (closer to 0 is more significant) +* `ids`: id of original geometry (used for joining against original table if desired -- see examples) +* `column_values`: original column values from `column_name` + +Availability: crankshaft v0.0.1 and above + +## Examples + +```sql +SELECT + t.the_geom_webmercator, + t.cartodb_id, + aoi.significance, + aoi.quadrant As aoi_quadrant +FROM + observatory.acs2013 As t +JOIN + crankshaft.CDB_AreasOfInterest('SELECT * FROM observatory.acs2013', + 'gini_index') +``` + +## API Usage + +Example + +```text +http://eschbacher.cartodb.com/api/v2/sql?q=SELECT * FROM crankshaft.CDB_AreasOfInterest('SELECT * FROM observatory.acs2013','gini_index') +``` + +Result +```json +{ + time: 0.120, + total_rows: 100, + rows: [{ + moran_vals: 0.7213, + quadrant: 'High area', + significance: 0.03, + ids: 1, + column_value: 0.22 + }, + { + moran_vals: -0.7213, + quadrant: 'Low outlier', + significance: 0.13, + ids: 2, + column_value: 0.03 + }, + ... + ] +} +``` + +## See Also + +crankshaft's areas of interest functions: + +* [CDB_AreasOfInterest_Global]() +* [CDB_AreasOfInterest_Rate_Local]() +* [CDB_AreasOfInterest_Rate_Global]() + + +PostGIS clustering functions: + +* [ST_ClusterIntersecting](http://postgis.net/docs/manual-2.2/ST_ClusterIntersecting.html) +* [ST_ClusterWithin](http://postgis.net/docs/manual-2.2/ST_ClusterWithin.html) + + +-- removing below, working into above #### What is Moran's I and why is it significant for CartoDB? From 00579cd838060b1de8f487c775dc88564c062285 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 23 Mar 2016 17:10:08 -0400 Subject: [PATCH 32/81] adding template --- doc/docs_template.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 doc/docs_template.md diff --git a/doc/docs_template.md b/doc/docs_template.md new file mode 100644 index 0000000..9d5b550 --- /dev/null +++ b/doc/docs_template.md @@ -0,0 +1,24 @@ + +## Name + +## Synopsis + +## Description + +Availability: v... + +## Examples + +```SQL +-- example of the function in use +SELECT cdb_awesome_function(the_geom, 'total_pop') +FROM table_name +``` + +## API Usage + +_asdf_ + +## See Also + +_Other function pages_ From bc67ae8f69e710e0eddaf6f63a5304b68131558e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 29 Mar 2016 12:18:52 -0700 Subject: [PATCH 33/81] changed name of functions for observatory --- src/pg/sql/10_moran.sql | 10 +++++----- src/pg/test/expected/02_moran_test.out | 18 +++++++++--------- src/pg/test/sql/02_moran_test.sql | 6 +++--- src/pg/test/sql/90_permissions.sql | 2 +- .../crankshaft/crankshaft/clustering/moran.py | 5 ++--- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index dba7069..3089fc6 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -1,6 +1,6 @@ -- Moran's I (global) CREATE OR REPLACE FUNCTION - cdb_moran ( + CDB_AreasOfInterest_Global ( subquery TEXT, attr_name TEXT, permutations INT DEFAULT 99, @@ -17,7 +17,7 @@ $$ LANGUAGE plpythonu; -- Moran's I Local CREATE OR REPLACE FUNCTION - cdb_moran_local ( + CDB_AreasOfInterest_Local( subquery TEXT, attr TEXT, permutations INT DEFAULT 99, @@ -34,7 +34,7 @@ $$ LANGUAGE plpythonu; -- Moran's I Rate (global) CREATE OR REPLACE FUNCTION - cdb_moran_rate ( + CDB_AreasOfInterest_Global_Rate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -53,7 +53,7 @@ $$ LANGUAGE plpythonu; -- Moran's I Local Rate CREATE OR REPLACE FUNCTION - cdb_moran_local_rate( + CDB_AreasOfInterest_Local_Rate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -86,4 +86,4 @@ $$ LANGUAGE plpythonu; -- from crankshaft.clustering import moran_local_bv -- # TODO: use named parameters or a dictionary -- return moran_local_bv(t, attr1, attr2, permutations, geom_col, id_col, w_type, num_ngbrs) --- $$ LANGUAGE plpythonu; \ No newline at end of file +-- $$ LANGUAGE plpythonu; diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 92cb218..20b92cd 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -110,7 +110,7 @@ INSERT INTO ppoints2 VALUES (24,'0101000020E61000009C5F91C5095C17C0C78784B15A4F4540'::geometry,'24','07',0.3, 1.0), (29,'0101000020E6100000C34D4A5B48E712C092E680892C684240'::geometry,'29','01',0.3, 1.0), (52,'0101000020E6100000406A545EB29A07C04E5F0BDA39A54140'::geometry,'52','19',0.0, 1.01) --- Moral functions perform some nondeterministic computations +-- Areas of Interest functions perform some nondeterministic computations -- (to estimate the significance); we will set the seeds for the RNGs -- that affect those results to have repeateble results SELECT cdb_crankshaft._cdb_random_seeds(1234); @@ -121,15 +121,15 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints - JOIN cdb_crankshaft.cdb_moran_local('SELECT * FROM ppoints', 'value') m + JOIN cdb_crankshaft.CDB_AreasOfInterest_Local('SELECT * FROM ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; NOTICE: ** Constructing query -CONTEXT: PL/Python function "cdb_moran_local" +CONTEXT: PL/Python function "cdb_areasofinterest_local" NOTICE: ** Query returned with 52 rows -CONTEXT: PL/Python function "cdb_moran_local" +CONTEXT: PL/Python function "cdb_areasofinterest_local" NOTICE: ** Finished calculations -CONTEXT: PL/Python function "cdb_moran_local" +CONTEXT: PL/Python function "cdb_areasofinterest_local" code | quads ------+------- 01 | HH @@ -194,15 +194,15 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.cdb_moran_local_rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; NOTICE: ** Constructing query -CONTEXT: PL/Python function "cdb_moran_local_rate" +CONTEXT: PL/Python function "cdb_areasofinterest_local_rate" NOTICE: ** Query returned with 51 rows -CONTEXT: PL/Python function "cdb_moran_local_rate" +CONTEXT: PL/Python function "cdb_areasofinterest_local_rate" NOTICE: ** Finished calculations -CONTEXT: PL/Python function "cdb_moran_local_rate" +CONTEXT: PL/Python function "cdb_areasofinterest_local_rate" code | quads ------+------- 01 | LL diff --git a/src/pg/test/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql index a0bc401..de9c6cf 100644 --- a/src/pg/test/sql/02_moran_test.sql +++ b/src/pg/test/sql/02_moran_test.sql @@ -1,14 +1,14 @@ \i test/fixtures/ppoints.sql \i test/fixtures/ppoints2.sql --- Moral functions perform some nondeterministic computations +-- Areas of Interest functions perform some nondeterministic computations -- (to estimate the significance); we will set the seeds for the RNGs -- that affect those results to have repeateble results SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints - JOIN cdb_crankshaft.cdb_moran_local('SELECT * FROM ppoints', 'value') m + JOIN cdb_crankshaft.CDB_AreasOfInterest_Local('SELECT * FROM ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; @@ -16,6 +16,6 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.cdb_moran_local_rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; diff --git a/src/pg/test/sql/90_permissions.sql b/src/pg/test/sql/90_permissions.sql index 2e7a89c..187f795 100644 --- a/src/pg/test/sql/90_permissions.sql +++ b/src/pg/test/sql/90_permissions.sql @@ -9,7 +9,7 @@ SET search_path TO public,cartodb,cdb_crankshaft; -- Exercise public functions SELECT ppoints.code, m.quads FROM ppoints - JOIN cdb_moran_local('ppoints', 'value') m + JOIN CDB_AreasOfInterest_Local('ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; SELECT round(cdb_overlap_sum( diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 2d65db5..728e72f 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -14,6 +14,7 @@ import plpy def moran(subquery, attr_name, permutations, geom_col, id_col, w_type, num_ngbrs): """ Moran's I (global) + Implementation building neighors with a PostGIS database and Moran's I core clusters with PySAL. Andy Eschbacher """ qvals = {"id_col": id_col, @@ -172,8 +173,6 @@ def moran_local_rate(subquery, numerator, denominator, permutations, geom_col, i plpy.notice('** Exiting function') return zip([None], [None], [None], [None], [None]) - plpy.notice('r.nrows() = %d' % r.nrows()) - ## collect attributes numer = get_attributes(r, 1) denom = get_attributes(r, 2) @@ -389,4 +388,4 @@ def quad_position(quads): lisa_sig = np.array([map_quads(q) for q in quads]) - return lisa_sig \ No newline at end of file + return lisa_sig From 06f5cf9951719ad413e22a3a3e924360ff180046 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 29 Mar 2016 12:34:23 -0700 Subject: [PATCH 34/81] standarizing error reporting --- src/py/crankshaft/crankshaft/clustering/moran.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 728e72f..8db23e4 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -35,7 +35,7 @@ def moran(subquery, attr_name, permutations, geom_col, id_col, w_type, num_ngbrs r = plpy.execute(q) plpy.notice('** Query returned with %d rows' % len(r)) except plpy.SPIError: - plpy.error('** Moran rate failed executing query to build weight object') + plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Error: %s' % plpy.SPIError) plpy.notice('** Exiting function') @@ -81,6 +81,7 @@ def moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbr r = plpy.execute(q) plpy.notice('** Query returned with %d rows' % len(r)) except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Exiting function') return zip([None], [None], [None], [None], [None]) @@ -121,7 +122,7 @@ def moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, r = plpy.execute(q) plpy.notice('** Query returned with %d rows' % len(r)) except plpy.SPIError: - plpy.error('Moran rate failed executing query to build weight object') + plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Error: %s' % plpy.SPIError) plpy.notice('** Exiting function') @@ -168,6 +169,7 @@ def moran_local_rate(subquery, numerator, denominator, permutations, geom_col, i r = plpy.execute(q) plpy.notice('** Query returned with %d rows' % len(r)) except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Error: %s' % plpy.SPIError) plpy.notice('** Exiting function') @@ -205,6 +207,7 @@ def moran_local_bv(subquery, attr1, attr2, permutations, geom_col, id_col, w_typ r = plpy.execute(q) plpy.notice('** Query returned with %d rows' % len(r)) except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % q) plpy.notice('** Error: %s' % plpy.SPIError) plpy.notice('** Exiting function') From 8dd8ab37a50bb5c0511f7b14b4bbcf75b6e8046b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 29 Mar 2016 22:49:31 -0700 Subject: [PATCH 35/81] refactored from pylint --- .../crankshaft/crankshaft/clustering/moran.py | 209 +++++++++--------- .../crankshaft/test/test_clustering_moran.py | 19 +- 2 files changed, 112 insertions(+), 116 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 8db23e4..3b407c0 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -11,10 +11,12 @@ import plpy # High level interface --------------------------------------- -def moran(subquery, attr_name, permutations, geom_col, id_col, w_type, num_ngbrs): +def moran(subquery, attr_name, + permutations, geom_col, id_col, w_type, num_ngbrs): """ Moran's I (global) - Implementation building neighors with a PostGIS database and Moran's I core clusters with PySAL. + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. Andy Eschbacher """ qvals = {"id_col": id_col, @@ -23,48 +25,39 @@ def moran(subquery, attr_name, permutations, geom_col, id_col, w_type, num_ngbrs "subquery": subquery, "num_ngbrs": num_ngbrs} - q = get_query(w_type, qvals) + query = construct_neighbor_query(w_type, qvals) - plpy.notice('** Query: %s' % q) + plpy.notice('** Query: %s' % query) try: - r = plpy.execute(q) - if (len(r) == 0) & (w_type != 'knn'): - plpy.notice('** Query returned with 0 rows, trying kNN weights') - q = get_query('knn', qvals) - r = plpy.execute(q) - plpy.notice('** Query returned with %d rows' % len(r)) + result = plpy.execute(query) + ## if there are no neighbors, exit + if len(result) == 0: + return zip([None], [None]) + plpy.notice('** Query returned with %d rows' % len(result)) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') - plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - plpy.notice('** Exiting function') - return zip([None], [None]) - - ## if there are no neighbors, exit - if len(r) == 0: return zip([None], [None]) ## collect attributes - attr_vals = get_attributes(r, 1) + attr_vals = get_attributes(result) ## calculate weights - weight = get_weight(r, w_type, num_ngbrs) + weight = get_weight(result, w_type, num_ngbrs) ## calculate moran global moran_global = ps.esda.moran.Moran(attr_vals, weight, permutations=permutations) - return zip([moran_global.I],[moran_global.EI]) + return zip([moran_global.I], [moran_global.EI]) -def moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs): +def moran_local(subquery, attr, + permutations, geom_col, id_col, w_type, num_ngbrs): """ Moran's I implementation for PL/Python Andy Eschbacher """ - # TODO: ensure that the significance output can be smaller that 1e-3 (0.001) - # TODO: make a wishlist of output features (zscores, pvalues, raw local lisa, what else?) - - plpy.notice('** Constructing query') # geometries with attributes that are null are ignored # resulting in a collection of not as near neighbors @@ -75,30 +68,32 @@ def moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbr "subquery": subquery, "num_ngbrs": num_ngbrs} - q = get_query(w_type, qvals) + query = construct_neighbor_query(w_type, qvals) try: - r = plpy.execute(q) - plpy.notice('** Query returned with %d rows' % len(r)) + result = plpy.execute(query) + if len(result) == 0: + return zip([None], [None], [None], [None], [None]) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') - plpy.notice('** Query failed: "%s"' % q) - plpy.notice('** Exiting function') + plpy.notice('** Query failed: "%s"' % query) return zip([None], [None], [None], [None], [None]) - y = get_attributes(r, 1) - w = get_weight(r, w_type) + attr_vals = get_attributes(result) + weight = get_weight(result, w_type) # calculate LISA values - lisa = ps.esda.moran.Moran_Local(y, w) + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) # find quadrants for each geometry quads = quad_position(lisa.q) plpy.notice('** Finished calculations') - return zip(lisa.Is, quads, lisa.p_sim, w.id_order, lisa.y) + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) -def moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs): +def moran_rate(subquery, numerator, denominator, + permutations, geom_col, id_col, w_type, num_ngbrs): """ Moran's I Rate (global) Andy Eschbacher @@ -110,88 +105,82 @@ def moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, "subquery": subquery, "num_ngbrs": num_ngbrs} - q = get_query(w_type, qvals) + query = construct_neighbor_query(w_type, qvals) - plpy.notice('** Query: %s' % q) + plpy.notice('** Query: %s' % query) try: - r = plpy.execute(q) - if len(r) == 0: - plpy.notice('** Query returned with 0 rows, trying kNN weights') - q = get_query('knn', qvals) - r = plpy.execute(q) - plpy.notice('** Query returned with %d rows' % len(r)) + result = plpy.execute(query) + if len(result) == 0: + ## if there are no values returned, exit + return zip([None], [None]) + plpy.notice('** Query returned with %d rows' % len(result)) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') - plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - plpy.notice('** Exiting function') - return zip([None], [None]) - - ## if there are no values returned, exit - if len(r) == 0: return zip([None], [None]) ## collect attributes - numer = get_attributes(r, 1) - denom = get_attributes(r, 2) + numer = get_attributes(result, 1) + denom = get_attributes(result, 2) - w = get_weight(r, w_type, num_ngbrs) + weight = get_weight(result, w_type, num_ngbrs) ## calculate moran global rate - mr = ps.esda.moran.Moran_Rate(numer, denom, w, permutations=permutations) + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) - plpy.notice('** Finished calculations') + return zip([lisa_rate.I], [lisa_rate.EI]) - return zip([mr.I],[mr.EI]) - -def moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs): +def moran_local_rate(subquery, numerator, denominator, + permutations, geom_col, id_col, w_type, num_ngbrs): """ - Moran's I Local Rate - Andy Eschbacher + Moran's I Local Rate + Andy Eschbacher """ - - plpy.notice('** Constructing query') - - # geometries with attributes that are null are ignored + # geometries with values that are null are ignored # resulting in a collection of not as near neighbors - qvals = {"id_col": id_col, - "numerator": numerator, - "denominator": denominator, - "geom_col": geom_col, - "subquery": subquery, - "num_ngbrs": num_ngbrs} - - q = get_query(w_type, qvals) + query = construct_neighbor_query(w_type, + {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs}) try: - r = plpy.execute(q) - plpy.notice('** Query returned with %d rows' % len(r)) + result = plpy.execute(query) + plpy.notice('** Query returned with %d rows' % len(result)) + if len(result) == 0: + return zip([None], [None], [None], [None], [None]) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') - plpy.notice('** Query failed: "%s"' % q) + plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - plpy.notice('** Exiting function') return zip([None], [None], [None], [None], [None]) ## collect attributes - numer = get_attributes(r, 1) - denom = get_attributes(r, 2) + numer = get_attributes(result, 1) + denom = get_attributes(result, 2) - w = get_weight(r, w_type, num_ngbrs) + weight = get_weight(result, w_type, num_ngbrs) # calculate LISA values - lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, w, permutations=permutations) + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) # find units of significance - quads = quad_position(lisa.q) + quads = quad_position(lisa.q) - plpy.notice('** Finished calculations') + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) - return zip(lisa.Is, quads, lisa.p_sim, w.id_order, lisa.y) - -def moran_local_bv(subquery, attr1, attr2, permutations, geom_col, id_col, w_type, num_ngbrs): +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ plpy.notice('** Constructing query') qvals = {"num_ngbrs": num_ngbrs, @@ -201,27 +190,28 @@ def moran_local_bv(subquery, attr1, attr2, permutations, geom_col, id_col, w_typ "geom_col": geom_col, "id_col": id_col} - q = get_query(w_type, qvals) + query = construct_neighbor_query(w_type, qvals) try: - r = plpy.execute(q) - plpy.notice('** Query returned with %d rows' % len(r)) + result = plpy.execute(query) + plpy.notice('** Query returned with %d rows' % len(result)) + if len(result) == 0: + return zip([None], [None], [None], [None]) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') - plpy.notice('** Query failed: "%s"' % q) - plpy.notice('** Error: %s' % plpy.SPIError) - plpy.notice('** Exiting function') + plpy.notice('** Query failed: "%s"' % query) return zip([None], [None], [None], [None]) ## collect attributes - attr1_vals = get_attributes(r, 1) - attr2_vals = get_attributes(r, 2) + attr1_vals = get_attributes(result, 1) + attr2_vals = get_attributes(result, 2) # create weights - w = get_weight(r, w_type, num_ngbrs) + weight = get_weight(result, w_type, num_ngbrs) # calculate LISA values - lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, w) + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) plpy.notice("len of Is: %d" % len(lisa.Is)) @@ -230,7 +220,7 @@ def moran_local_bv(subquery, attr1, attr2, permutations, geom_col, id_col, w_typ plpy.notice('** Finished calculations') - return zip(lisa.Is, lisa_sig, lisa.p_sim, w.id_order) + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) # Low level functions ---------------------------------------- @@ -240,7 +230,7 @@ def map_quads(coord): Map a quadrant number to Moran's I designation HH=1, LH=2, LL=3, HL=4 Input: - :param coord (int): quadrant of a specific measurement + @param coord (int): quadrant of a specific measurement """ if coord == 1: return 'HH' @@ -256,7 +246,7 @@ def map_quads(coord): def query_attr_select(params): """ Create portion of SELECT statement for attributes inolved in query. - :param params: dict of information used in query (column names, + @param params: dict of information used in query (column names, table name, etc.) """ @@ -293,7 +283,7 @@ def query_attr_where(params): def knn(params): """SQL query for k-nearest neighbors. - :param vars: dict of values to fill template + @param vars: dict of values to fill template """ attr_select = query_attr_select(params) @@ -322,7 +312,7 @@ def knn(params): ## SQL query for finding queens neighbors (all contiguous polygons) def queen(params): """SQL query for queen neighbors. - :param params: dict of information to fill query + @param params dict: information to fill query """ attr_select = query_attr_select(params) attr_where = query_attr_where(params) @@ -348,10 +338,10 @@ def queen(params): ## to add more weight methods open a ticket or pull request -def get_query(w_type, query_vals): +def construct_neighbor_query(w_type, query_vals): """Return requested query. - :param w_type: type of neighbors to calculate (knn or queen) - :param query_vals: values used to construct the query + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query """ if w_type == 'knn': @@ -359,10 +349,10 @@ def get_query(w_type, query_vals): else: return queen(query_vals) -def get_attributes(query_res, attr_num): +def get_attributes(query_res, attr_num=1): """ - :param query_res: query results with attributes and neighbors - :param attr_num: attribute number (1, 2, ...) + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) """ return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) @@ -370,15 +360,15 @@ def get_attributes(query_res, attr_num): def get_weight(query_res, w_type='queen', num_ngbrs=5): """ Construct PySAL weight from return value of query - :param query_res: query results with attributes and neighbors + @param query_res: query results with attributes and neighbors """ if w_type == 'knn': row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs weights = {x['id']: row_normed_weights for x in query_res} else: weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) - if len(x['neighbors']) > 0 - else [] for x in query_res} + if len(x['neighbors']) > 0 + else [] for x in query_res} neighbors = {x['id']: x['neighbors'] for x in query_res} @@ -387,6 +377,11 @@ def get_weight(query_res, w_type='queen', num_ngbrs=5): def quad_position(quads): """ Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param ndarray: an array of quads classied by 'HH', 'LL', etc. """ lisa_sig = np.array([map_quads(q) for q in quads]) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index c7cc71a..95d959c 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -56,7 +56,7 @@ class MoranTest(unittest.TestCase): self.assertEqual(cc.query_attr_where(self.params), ans) def test_knn(self): - """Test knn function.""" + """Test knn neighbors constructor""" ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ @@ -70,7 +70,7 @@ class MoranTest(unittest.TestCase): self.assertEqual(cc.knn(self.params), ans) def test_queen(self): - """Test queen neighbors function.""" + """Test queen neighbors constructor""" ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ @@ -83,19 +83,20 @@ class MoranTest(unittest.TestCase): self.assertEqual(cc.queen(self.params), ans) - def test_get_query(self): - """Test get_query.""" + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS " \ "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ - "OFFSET 1 ) ) As neighbors FROM (SELECT * FROM a_list) As i WHERE " \ - "i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ - "i.\"jay_z\" <> 0 ORDER BY i.\"cartodb_id\" ASC;" + "OFFSET 1 ) ) As neighbors FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" - self.assertEqual(cc.get_query('knn', self.params), ans) + self.assertEqual(cc.construct_neighbor_query('knn', self.params), ans) def test_get_attributes(self): """Test get_attributes.""" @@ -142,4 +143,4 @@ class MoranTest(unittest.TestCase): result = [(row[0], row[1]) for row in result] expected = self.moran_data for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): - self.assertAlmostEqual(res_val, exp_val) \ No newline at end of file + self.assertAlmostEqual(res_val, exp_val) From e56519f599e2b4462be7a644824a8ca6da558c37 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 29 Mar 2016 23:39:29 -0700 Subject: [PATCH 36/81] removed unneded comments, make outputs more consistent --- .../crankshaft/crankshaft/clustering/moran.py | 51 +++++++---- .../crankshaft/test/test_clustering_moran.py | 87 +++++++++++-------- 2 files changed, 84 insertions(+), 54 deletions(-) diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 3b407c0..7f402a8 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -31,15 +31,15 @@ def moran(subquery, attr_name, try: result = plpy.execute(query) - ## if there are no neighbors, exit + # if there are no neighbors, exit if len(result) == 0: - return zip([None], [None]) + return empty_zipped_array(2) plpy.notice('** Query returned with %d rows' % len(result)) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - return zip([None], [None]) + return empty_zipped_array(2) ## collect attributes attr_vals = get_attributes(result) @@ -72,12 +72,13 @@ def moran_local(subquery, attr, try: result = plpy.execute(query) + # if there are no neighbors, exit if len(result) == 0: - return zip([None], [None], [None], [None], [None]) + return empty_zipped_array(5) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) - return zip([None], [None], [None], [None], [None]) + return empty_zipped_array(5) attr_vals = get_attributes(result) weight = get_weight(result, w_type) @@ -111,15 +112,15 @@ def moran_rate(subquery, numerator, denominator, try: result = plpy.execute(query) + # if there are no neighbors, exit if len(result) == 0: - ## if there are no values returned, exit - return zip([None], [None]) + return empty_zipped_array(2) plpy.notice('** Query returned with %d rows' % len(result)) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - return zip([None], [None]) + return empty_zipped_array(2) ## collect attributes numer = get_attributes(result, 1) @@ -152,14 +153,14 @@ def moran_local_rate(subquery, numerator, denominator, try: result = plpy.execute(query) - plpy.notice('** Query returned with %d rows' % len(result)) + # if there are no neighbors, exit if len(result) == 0: - return zip([None], [None], [None], [None], [None]) + return empty_zipped_array(5) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - return zip([None], [None], [None], [None], [None]) + return empty_zipped_array(5) ## collect attributes numer = get_attributes(result, 1) @@ -194,13 +195,14 @@ def moran_local_bv(subquery, attr1, attr2, try: result = plpy.execute(query) - plpy.notice('** Query returned with %d rows' % len(result)) + # if there are no neighbors, exit if len(result) == 0: - return zip([None], [None], [None], [None]) + return empty_zipped_array(4) except plpy.SPIError: - plpy.error('Error: areas of interest query failed, check input parameters') + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") plpy.notice('** Query failed: "%s"' % query) - return zip([None], [None], [None], [None]) + return empty_zipped_array(4) ## collect attributes attr1_vals = get_attributes(result, 1) @@ -222,7 +224,6 @@ def moran_local_bv(subquery, attr1, attr2, return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) - # Low level functions ---------------------------------------- def map_quads(coord): @@ -231,6 +232,8 @@ def map_quads(coord): HH=1, LH=2, LL=3, HL=4 Input: @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') """ if coord == 1: return 'HH' @@ -298,9 +301,10 @@ def knn(params): "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ "FROM ({subquery}) As j " \ - "WHERE %(attr_where_j)s " \ + "WHERE %(attr_where_j)s AND " \ + "i.\"{id_col}\" <> j.\"{id_col}\" " \ "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ - "LIMIT {num_ngbrs} OFFSET 1 ) " \ + "LIMIT {num_ngbrs}) " \ ") As neighbors " \ "FROM ({subquery}) As i " \ "WHERE " \ @@ -387,3 +391,14 @@ def quad_position(quads): lisa_sig = np.array([map_quads(q) for q in quads]) return lisa_sig + +def return_empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 95d959c..e2d2a50 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -16,7 +16,7 @@ from crankshaft import random_seeds import json class MoranTest(unittest.TestCase): - """Testing class for Moran's I functions.""" + """Testing class for Moran's I functions""" def setUp(self): plpy._reset() @@ -30,7 +30,7 @@ class MoranTest(unittest.TestCase): self.moran_data = json.loads(open(fixture_file('moran.json')).read()) def test_map_quads(self): - """Test map_quads.""" + """Test map_quads""" self.assertEqual(cc.map_quads(1), 'HH') self.assertEqual(cc.map_quads(2), 'LH') self.assertEqual(cc.map_quads(3), 'LL') @@ -39,7 +39,7 @@ class MoranTest(unittest.TestCase): self.assertEqual(cc.map_quads('andy'), None) def test_query_attr_select(self): - """Test query_attr_select.""" + """Test query_attr_select""" ans = "i.\"{attr1}\"::numeric As attr1, " \ "i.\"{attr2}\"::numeric As attr2, " @@ -47,10 +47,10 @@ class MoranTest(unittest.TestCase): self.assertEqual(cc.query_attr_select(self.params), ans) def test_query_attr_where(self): - """Test query_attr_where.""" + """Test query_attr_where""" - ans = "idx_replace.\"{attr1}\" IS NOT NULL AND "\ - "idx_replace.\"{attr2}\" IS NOT NULL AND "\ + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" IS NOT NULL AND " \ "idx_replace.\"{attr2}\" <> 0" self.assertEqual(cc.query_attr_where(self.params), ans) @@ -58,61 +58,76 @@ class MoranTest(unittest.TestCase): def test_knn(self): """Test knn neighbors constructor""" - ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 ORDER BY " \ - "j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 OFFSET 1 ) ) " \ - "As neighbors FROM (SELECT * FROM a_list) As i WHERE i.\"andy\" IS NOT " \ - "NULL AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER " \ - "BY i.\"cartodb_id\" ASC;" + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 AND " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321) ) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" self.assertEqual(cc.knn(self.params), ans) def test_queen(self): """Test queen neighbors constructor""" - ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE ST_Touches(" \ - "i.\"the_geom\", j.\"the_geom\") AND j.\"andy\" IS NOT NULL " \ - "AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0)) As " \ - "neighbors FROM (SELECT * FROM a_list) As i WHERE i.\"andy\" IS NOT NULL " \ - "AND i.\"jay_z\" IS NOT NULL AND i.\"jay_z\" <> 0 ORDER BY " \ - "i.\"cartodb_id\" ASC;" + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0) + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" self.assertEqual(cc.queen(self.params), ans) def test_construct_neighbor_query(self): """Test construct_neighbor_query""" - ans = "SELECT i.\"cartodb_id\" As id, i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, (SELECT ARRAY(SELECT " \ - "j.\"cartodb_id\" FROM (SELECT * FROM a_list) As j WHERE j.\"andy\" IS " \ - "NOT NULL AND j.\"jay_z\" IS NOT NULL AND j.\"jay_z\" <> 0 " \ - "ORDER BY j.\"the_geom\" <-> i.\"the_geom\" ASC LIMIT 321 " \ - "OFFSET 1 ) ) As neighbors FROM (SELECT * FROM a_list) As i " \ - "WHERE i.\"andy\" IS NOT NULL AND i.\"jay_z\" IS NOT NULL AND " \ - "i.\"jay_z\" <> 0 " \ - "ORDER BY i.\"cartodb_id\" ASC;" - - self.assertEqual(cc.construct_neighbor_query('knn', self.params), ans) + # Compare to raw knn query + self.assertEqual(cc.construct_neighbor_query('knn', self.params), + cc.knn(self.params)) def test_get_attributes(self): - """Test get_attributes.""" + """Test get_attributes""" ## need to add tests self.assertEqual(True, True) def test_get_weight(self): - """Test get_weight.""" + """Test get_weight""" self.assertEqual(True, True) + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(cc.empty_zipped_array(2), ans2) + self.assertEqual(cc.empty_zipped_array(4), ans4) def test_quad_position(self): - """Test lisa_sig_vals.""" + """Test lisa_sig_vals""" quads = np.array([1, 2, 3, 4], np.int) From 5a46f65e5916fb88ebf185fce240c49a4fdcca4b Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 08:09:48 -0400 Subject: [PATCH 37/81] update tests to remove plpy notices --- src/pg/test/expected/02_moran_test.out | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 20b92cd..eb2afa5 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -124,12 +124,6 @@ SELECT ppoints.code, m.quads JOIN cdb_crankshaft.CDB_AreasOfInterest_Local('SELECT * FROM ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; -NOTICE: ** Constructing query -CONTEXT: PL/Python function "cdb_areasofinterest_local" -NOTICE: ** Query returned with 52 rows -CONTEXT: PL/Python function "cdb_areasofinterest_local" -NOTICE: ** Finished calculations -CONTEXT: PL/Python function "cdb_areasofinterest_local" code | quads ------+------- 01 | HH @@ -197,12 +191,6 @@ SELECT ppoints2.code, m.quads JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; -NOTICE: ** Constructing query -CONTEXT: PL/Python function "cdb_areasofinterest_local_rate" -NOTICE: ** Query returned with 51 rows -CONTEXT: PL/Python function "cdb_areasofinterest_local_rate" -NOTICE: ** Finished calculations -CONTEXT: PL/Python function "cdb_areasofinterest_local_rate" code | quads ------+------- 01 | LL From 6bb4f36df5f18d94306cd508dd65639d894a06e1 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 08:10:35 -0400 Subject: [PATCH 38/81] extracting util code to new submodule --- .../crankshaft/crankshaft/clustering/moran.py | 214 +++--------------- .../crankshaft/pysal_utils/__init__.py | 1 + .../crankshaft/pysal_utils/pysal_utils.py | 149 ++++++++++++ .../crankshaft/test/test_clustering_moran.py | 21 +- 4 files changed, 196 insertions(+), 189 deletions(-) create mode 100644 src/py/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 7f402a8..2a043c3 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -5,10 +5,12 @@ Moran's I geostatistics (global clustering & outliers presence) # TODO: Fill in local neighbors which have null/NoneType values with the # average of the their neighborhood -import numpy as np import pysal as ps import plpy +# crankshaft module +import crankshaft.pysal_utils as pu + # High level interface --------------------------------------- def moran(subquery, attr_name, @@ -25,7 +27,7 @@ def moran(subquery, attr_name, "subquery": subquery, "num_ngbrs": num_ngbrs} - query = construct_neighbor_query(w_type, qvals) + query = pu.construct_neighbor_query(w_type, qvals) plpy.notice('** Query: %s' % query) @@ -33,22 +35,23 @@ def moran(subquery, attr_name, result = plpy.execute(query) # if there are no neighbors, exit if len(result) == 0: - return empty_zipped_array(2) + return pu.empty_zipped_array(2) plpy.notice('** Query returned with %d rows' % len(result)) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - return empty_zipped_array(2) + return pu.empty_zipped_array(2) ## collect attributes - attr_vals = get_attributes(result) + attr_vals = pu.get_attributes(result) ## calculate weights - weight = get_weight(result, w_type, num_ngbrs) + weight = pu.get_weight(result, w_type, num_ngbrs) ## calculate moran global - moran_global = ps.esda.moran.Moran(attr_vals, weight, permutations=permutations) + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) return zip([moran_global.I], [moran_global.EI]) @@ -68,20 +71,20 @@ def moran_local(subquery, attr, "subquery": subquery, "num_ngbrs": num_ngbrs} - query = construct_neighbor_query(w_type, qvals) + query = pu.construct_neighbor_query(w_type, qvals) try: result = plpy.execute(query) # if there are no neighbors, exit if len(result) == 0: - return empty_zipped_array(5) + return pu.empty_zipped_array(5) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) - return empty_zipped_array(5) + return pu.empty_zipped_array(5) - attr_vals = get_attributes(result) - weight = get_weight(result, w_type) + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) # calculate LISA values lisa = ps.esda.moran.Moran_Local(attr_vals, weight, @@ -90,7 +93,6 @@ def moran_local(subquery, attr, # find quadrants for each geometry quads = quad_position(lisa.q) - plpy.notice('** Finished calculations') return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) def moran_rate(subquery, numerator, denominator, @@ -106,7 +108,7 @@ def moran_rate(subquery, numerator, denominator, "subquery": subquery, "num_ngbrs": num_ngbrs} - query = construct_neighbor_query(w_type, qvals) + query = pu.construct_neighbor_query(w_type, qvals) plpy.notice('** Query: %s' % query) @@ -114,19 +116,19 @@ def moran_rate(subquery, numerator, denominator, result = plpy.execute(query) # if there are no neighbors, exit if len(result) == 0: - return empty_zipped_array(2) + return pu.empty_zipped_array(2) plpy.notice('** Query returned with %d rows' % len(result)) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - return empty_zipped_array(2) + return pu.empty_zipped_array(2) ## collect attributes - numer = get_attributes(result, 1) - denom = get_attributes(result, 2) + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) - weight = get_weight(result, w_type, num_ngbrs) + weight = pu.get_weight(result, w_type, num_ngbrs) ## calculate moran global rate lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, @@ -143,7 +145,7 @@ def moran_local_rate(subquery, numerator, denominator, # geometries with values that are null are ignored # resulting in a collection of not as near neighbors - query = construct_neighbor_query(w_type, + query = pu.construct_neighbor_query(w_type, {"id_col": id_col, "numerator": numerator, "denominator": denominator, @@ -155,18 +157,18 @@ def moran_local_rate(subquery, numerator, denominator, result = plpy.execute(query) # if there are no neighbors, exit if len(result) == 0: - return empty_zipped_array(5) + return pu.empty_zipped_array(5) except plpy.SPIError: plpy.error('Error: areas of interest query failed, check input parameters') plpy.notice('** Query failed: "%s"' % query) plpy.notice('** Error: %s' % plpy.SPIError) - return empty_zipped_array(5) + return pu.empty_zipped_array(5) ## collect attributes - numer = get_attributes(result, 1) - denom = get_attributes(result, 2) + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) - weight = get_weight(result, w_type, num_ngbrs) + weight = pu.get_weight(result, w_type, num_ngbrs) # calculate LISA values lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, @@ -191,25 +193,25 @@ def moran_local_bv(subquery, attr1, attr2, "geom_col": geom_col, "id_col": id_col} - query = construct_neighbor_query(w_type, qvals) + query = pu.construct_neighbor_query(w_type, qvals) try: result = plpy.execute(query) # if there are no neighbors, exit if len(result) == 0: - return empty_zipped_array(4) + return pu.empty_zipped_array(4) except plpy.SPIError: plpy.error("Error: areas of interest query failed, " \ "check input parameters") plpy.notice('** Query failed: "%s"' % query) - return empty_zipped_array(4) + return pu.empty_zipped_array(4) ## collect attributes - attr1_vals = get_attributes(result, 1) - attr2_vals = get_attributes(result, 2) + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) # create weights - weight = get_weight(result, w_type, num_ngbrs) + weight = pu.get_weight(result, w_type, num_ngbrs) # calculate LISA values lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, @@ -246,138 +248,6 @@ def map_quads(coord): else: return None -def query_attr_select(params): - """ - Create portion of SELECT statement for attributes inolved in query. - @param params: dict of information used in query (column names, - table name, etc.) - """ - - attrs = [k for k in params - if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] - - template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " - - attr_string = "" - - for idx, val in enumerate(sorted(attrs)): - attr_string += template % {"col": val, "alias_num": idx + 1} - - return attr_string - -def query_attr_where(params): - """ - Create portion of WHERE clauses for weeding out NULL-valued geometries - """ - attrs = sorted([k for k in params - if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) - - attr_string = [] - - for attr in attrs: - attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) - - if len(attrs) == 2: - attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) - - out = " AND ".join(attr_string) - - return out - -def knn(params): - """SQL query for k-nearest neighbors. - @param vars: dict of values to fill template - """ - - attr_select = query_attr_select(params) - attr_where = query_attr_where(params) - - replacements = {"attr_select": attr_select, - "attr_where_i": attr_where.replace("idx_replace", "i"), - "attr_where_j": attr_where.replace("idx_replace", "j")} - - query = "SELECT " \ - "i.\"{id_col}\" As id, " \ - "%(attr_select)s" \ - "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM ({subquery}) As j " \ - "WHERE %(attr_where_j)s AND " \ - "i.\"{id_col}\" <> j.\"{id_col}\" " \ - "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ - "LIMIT {num_ngbrs}) " \ - ") As neighbors " \ - "FROM ({subquery}) As i " \ - "WHERE " \ - "%(attr_where_i)s " \ - "ORDER BY i.\"{id_col}\" ASC;" % replacements - - return query.format(**params) - -## SQL query for finding queens neighbors (all contiguous polygons) -def queen(params): - """SQL query for queen neighbors. - @param params dict: information to fill query - """ - attr_select = query_attr_select(params) - attr_where = query_attr_where(params) - - replacements = {"attr_select": attr_select, - "attr_where_i": attr_where.replace("idx_replace", "i"), - "attr_where_j": attr_where.replace("idx_replace", "j")} - - query = "SELECT " \ - "i.\"{id_col}\" As id, " \ - "%(attr_select)s" \ - "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ - "FROM ({subquery}) As j " \ - "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ - "%(attr_where_j)s)" \ - ") As neighbors " \ - "FROM ({subquery}) As i " \ - "WHERE " \ - "%(attr_where_i)s " \ - "ORDER BY i.\"{id_col}\" ASC;" % replacements - - return query.format(**params) - -## to add more weight methods open a ticket or pull request - -def construct_neighbor_query(w_type, query_vals): - """Return requested query. - @param w_type text: type of neighbors to calculate ('knn' or 'queen') - @param query_vals dict: values used to construct the query - """ - - if w_type == 'knn': - return knn(query_vals) - else: - return queen(query_vals) - -def get_attributes(query_res, attr_num=1): - """ - @param query_res: query results with attributes and neighbors - @param attr_num: attribute number (1, 2, ...) - """ - return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) - -## Build weight object -def get_weight(query_res, w_type='queen', num_ngbrs=5): - """ - Construct PySAL weight from return value of query - @param query_res: query results with attributes and neighbors - """ - if w_type == 'knn': - row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs - weights = {x['id']: row_normed_weights for x in query_res} - else: - weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) - if len(x['neighbors']) > 0 - else [] for x in query_res} - - neighbors = {x['id']: x['neighbors'] for x in query_res} - - return ps.W(neighbors, weights) - def quad_position(quads): """ Produce Moran's I classification based of n @@ -385,20 +255,6 @@ def quad_position(quads): @param quads ndarray: an array of quads classified by 1-4 (PySAL default) Output: - @param ndarray: an array of quads classied by 'HH', 'LL', etc. + @param list: an array of quads classied by 'HH', 'LL', etc. """ - - lisa_sig = np.array([map_quads(q) for q in quads]) - - return lisa_sig - -def return_empty_zipped_array(num_nones): - """ - prepare return values for cases of empty weights objects (no neighbors) - Input: - @param num_nones int: number of columns (e.g., 4) - Output: - [(None, None, None, None)] - """ - - return [tuple([None] * num_nones)] + return [map_quads(q) for q in quads] diff --git a/src/py/crankshaft/crankshaft/pysal_utils/__init__.py b/src/py/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..835880d --- /dev/null +++ b/src/py/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1 @@ +from pysal_utils import * diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..5482cc7 --- /dev/null +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,149 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res: query results with attributes and neighbors + """ + if w_type == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + else: + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE %(attr_where_j)s AND " \ + "i.\"{id_col}\" <> j.\"{id_col}\" " \ + "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs}) " \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index e2d2a50..399e0cb 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -12,6 +12,7 @@ import unittest from helper import plpy, fixture_file import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu from crankshaft import random_seeds import json @@ -44,16 +45,16 @@ class MoranTest(unittest.TestCase): ans = "i.\"{attr1}\"::numeric As attr1, " \ "i.\"{attr2}\"::numeric As attr2, " - self.assertEqual(cc.query_attr_select(self.params), ans) + self.assertEqual(pu.query_attr_select(self.params), ans) def test_query_attr_where(self): - """Test query_attr_where""" + """Test pu.query_attr_where""" ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ "idx_replace.\"{attr2}\" IS NOT NULL AND " \ "idx_replace.\"{attr2}\" <> 0" - self.assertEqual(cc.query_attr_where(self.params), ans) + self.assertEqual(pu.query_attr_where(self.params), ans) def test_knn(self): """Test knn neighbors constructor""" @@ -76,7 +77,7 @@ class MoranTest(unittest.TestCase): "i.\"jay_z\" <> 0 " \ "ORDER BY i.\"cartodb_id\" ASC;" - self.assertEqual(cc.knn(self.params), ans) + self.assertEqual(pu.knn(self.params), ans) def test_queen(self): """Test queen neighbors constructor""" @@ -90,7 +91,7 @@ class MoranTest(unittest.TestCase): "j.\"the_geom\") AND " \ "j.\"andy\" IS NOT NULL AND " \ "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0) + "j.\"jay_z\" <> 0)" \ ") As neighbors " \ "FROM (SELECT * FROM a_list) As i " \ "WHERE i.\"andy\" IS NOT NULL AND " \ @@ -98,14 +99,14 @@ class MoranTest(unittest.TestCase): "i.\"jay_z\" <> 0 " \ "ORDER BY i.\"cartodb_id\" ASC;" - self.assertEqual(cc.queen(self.params), ans) + self.assertEqual(pu.queen(self.params), ans) def test_construct_neighbor_query(self): """Test construct_neighbor_query""" # Compare to raw knn query - self.assertEqual(cc.construct_neighbor_query('knn', self.params), - cc.knn(self.params)) + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) def test_get_attributes(self): """Test get_attributes""" @@ -123,8 +124,8 @@ class MoranTest(unittest.TestCase): """Test empty_zipped_array""" ans2 = [(None, None)] ans4 = [(None, None, None, None)] - self.assertEqual(cc.empty_zipped_array(2), ans2) - self.assertEqual(cc.empty_zipped_array(4), ans4) + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) def test_quad_position(self): """Test lisa_sig_vals""" From b0150d4fec612a8e6f92220c70f6b2c7454c636f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 08:27:14 -0400 Subject: [PATCH 39/81] adding tests for pysal_utils --- .../crankshaft/pysal_utils/pysal_utils.py | 2 +- .../crankshaft/test/test_clustering_moran.py | 90 --------------- src/py/crankshaft/test/test_pysal_utils.py | 104 ++++++++++++++++++ 3 files changed, 105 insertions(+), 91 deletions(-) create mode 100644 src/py/crankshaft/test/test_pysal_utils.py diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 5482cc7..9fdbfb3 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -92,7 +92,7 @@ def knn(params): "WHERE %(attr_where_j)s AND " \ "i.\"{id_col}\" <> j.\"{id_col}\" " \ "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ - "LIMIT {num_ngbrs}) " \ + "LIMIT {num_ngbrs})" \ ") As neighbors " \ "FROM ({subquery}) As i " \ "WHERE " \ diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 399e0cb..fbe11d6 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -1,8 +1,6 @@ import unittest import numpy as np -import unittest - # from mock_plpy import MockPlPy # plpy = MockPlPy() @@ -39,94 +37,6 @@ class MoranTest(unittest.TestCase): self.assertEqual(cc.map_quads(33), None) self.assertEqual(cc.map_quads('andy'), None) - def test_query_attr_select(self): - """Test query_attr_select""" - - ans = "i.\"{attr1}\"::numeric As attr1, " \ - "i.\"{attr2}\"::numeric As attr2, " - - self.assertEqual(pu.query_attr_select(self.params), ans) - - def test_query_attr_where(self): - """Test pu.query_attr_where""" - - ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ - "idx_replace.\"{attr2}\" IS NOT NULL AND " \ - "idx_replace.\"{attr2}\" <> 0" - - self.assertEqual(pu.query_attr_where(self.params), ans) - - def test_knn(self): - """Test knn neighbors constructor""" - - ans = "SELECT i.\"cartodb_id\" As id, " \ - "i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, " \ - "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM (SELECT * FROM a_list) As j " \ - "WHERE j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0 AND " \ - "i.\"cartodb_id\" <> j.\"cartodb_id\" " \ - "ORDER BY " \ - "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ - "LIMIT 321) ) As neighbors " \ - "FROM (SELECT * FROM a_list) As i " \ - "WHERE i.\"andy\" IS NOT NULL AND " \ - "i.\"jay_z\" IS NOT NULL AND " \ - "i.\"jay_z\" <> 0 " \ - "ORDER BY i.\"cartodb_id\" ASC;" - - self.assertEqual(pu.knn(self.params), ans) - - def test_queen(self): - """Test queen neighbors constructor""" - - ans = "SELECT i.\"cartodb_id\" As id, " \ - "i.\"andy\"::numeric As attr1, " \ - "i.\"jay_z\"::numeric As attr2, " \ - "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ - "FROM (SELECT * FROM a_list) As j " \ - "WHERE ST_Touches(i.\"the_geom\", " \ - "j.\"the_geom\") AND " \ - "j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0)" \ - ") As neighbors " \ - "FROM (SELECT * FROM a_list) As i " \ - "WHERE i.\"andy\" IS NOT NULL AND " \ - "i.\"jay_z\" IS NOT NULL AND " \ - "i.\"jay_z\" <> 0 " \ - "ORDER BY i.\"cartodb_id\" ASC;" - - self.assertEqual(pu.queen(self.params), ans) - - def test_construct_neighbor_query(self): - """Test construct_neighbor_query""" - - # Compare to raw knn query - self.assertEqual(pu.construct_neighbor_query('knn', self.params), - pu.knn(self.params)) - - def test_get_attributes(self): - """Test get_attributes""" - - ## need to add tests - - self.assertEqual(True, True) - - def test_get_weight(self): - """Test get_weight""" - - self.assertEqual(True, True) - - def test_empty_zipped_array(self): - """Test empty_zipped_array""" - ans2 = [(None, None)] - ans4 = [(None, None, None, None)] - self.assertEqual(pu.empty_zipped_array(2), ans2) - self.assertEqual(pu.empty_zipped_array(4), ans4) - def test_quad_position(self): """Test lisa_sig_vals""" diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..8e9b7da --- /dev/null +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,104 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(pu.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 AND " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) From 4c243bf1d35dcb802bee60b910dee1eae748eab9 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 11:44:44 -0400 Subject: [PATCH 40/81] correct func signatures --- src/pg/sql/10_moran.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index 3089fc6..85d69f0 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -8,7 +8,7 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id', w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +RETURNS TABLE (moran NUMERIC, significance NUMERIC) AS $$ from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary @@ -43,7 +43,7 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id', w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5) -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +RETURNS TABLE (moran FLOAT, significance FLOAT) AS $$ from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary From 02b74813aca7c7ebac7caae04a4d503b9357bca1 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 12:09:49 -0400 Subject: [PATCH 41/81] add test for global moran --- src/py/crankshaft/test/test_clustering_moran.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index fbe11d6..29c5bde 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -52,7 +52,7 @@ class MoranTest(unittest.TestCase): data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.moran_local('table', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) + result = cc.moran_local('subquery', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) result = [(row[0], row[1]) for row in result] expected = self.moran_data for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): @@ -70,3 +70,14 @@ class MoranTest(unittest.TestCase): expected = self.moran_data for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) From ea02f36235420e1ec4afc621b728f3dfd87fc7f5 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 30 Mar 2016 15:37:51 -0400 Subject: [PATCH 42/81] adding condition to avoid self-comparison in neighbor queries --- .../crankshaft/pysal_utils/pysal_utils.py | 13 +++++++---- src/py/crankshaft/test/test_pysal_utils.py | 23 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index 9fdbfb3..fa06e26 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -89,9 +89,11 @@ def knn(params): "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ "FROM ({subquery}) As j " \ - "WHERE %(attr_where_j)s AND " \ - "i.\"{id_col}\" <> j.\"{id_col}\" " \ - "ORDER BY j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ "LIMIT {num_ngbrs})" \ ") As neighbors " \ "FROM ({subquery}) As i " \ @@ -118,8 +120,9 @@ def queen(params): "%(attr_select)s" \ "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ "FROM ({subquery}) As j " \ - "WHERE ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ - "%(attr_where_j)s)" \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ ") As neighbors " \ "FROM ({subquery}) As i " \ "WHERE " \ diff --git a/src/py/crankshaft/test/test_pysal_utils.py b/src/py/crankshaft/test/test_pysal_utils.py index 8e9b7da..4ea0d9b 100644 --- a/src/py/crankshaft/test/test_pysal_utils.py +++ b/src/py/crankshaft/test/test_pysal_utils.py @@ -40,12 +40,13 @@ class PysalUtilsTest(unittest.TestCase): "i.\"jay_z\"::numeric As attr2, " \ "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ "FROM (SELECT * FROM a_list) As j " \ - "WHERE j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0 AND " \ - "i.\"cartodb_id\" <> j.\"cartodb_id\" " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ "ORDER BY " \ - "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ "LIMIT 321)) As neighbors " \ "FROM (SELECT * FROM a_list) As i " \ "WHERE i.\"andy\" IS NOT NULL AND " \ @@ -63,11 +64,13 @@ class PysalUtilsTest(unittest.TestCase): "i.\"jay_z\"::numeric As attr2, " \ "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ "FROM (SELECT * FROM a_list) As j " \ - "WHERE ST_Touches(i.\"the_geom\", " \ - "j.\"the_geom\") AND " \ - "j.\"andy\" IS NOT NULL AND " \ - "j.\"jay_z\" IS NOT NULL AND " \ - "j.\"jay_z\" <> 0)" \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ ") As neighbors " \ "FROM (SELECT * FROM a_list) As i " \ "WHERE i.\"andy\" IS NOT NULL AND " \ From cc4a35ebd9d6e2a8c08d930f62a0d8280f5e8069 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Fri, 20 May 2016 11:47:12 +0200 Subject: [PATCH 43/81] Fix instructions to update/install the extension --- CONTRIBUTING.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9cfb951..f642d45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,8 +45,8 @@ source envs/dev/bin/activate Update extension in a working database with: -* `ALTER EXTENSION crankshaft VERSION TO 'current';` - `ALTER EXTENSION crankshaft VERSION TO 'dev';` +* `ALTER EXTENSION crankshaft UPDATE TO 'current';` + `ALTER EXTENSION crankshaft UPDATE TO 'dev';` Note: we keep the current development version install as 'dev' always; we update through the 'current' alias to allow changing the extension @@ -58,7 +58,10 @@ should be dropped manually before the update. If the extension has not previously been installed in a database, it can be installed directly with: -* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` +* `CREATE EXTENSION IF NOT EXISTS plpythonu;` + `CREATE EXTENSION IF NOT EXISTS postgis;` + `CREATE EXTENSION IF NOT EXISTS cartodb;` + `CREATE EXTENSION crankshaft WITH VERSION 'dev';` Note: the development extension uses the development python virtual environment automatically. From 0400b1a88000c77fde34a731d60e3be04da96b01 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 13:23:56 -0400 Subject: [PATCH 44/81] adding template for code reviews --- .github/PULL_REQUEST_TEMPLATE.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..941542d --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,6 @@ + +- [ ] All declared geometries are `geometry(Geometry, 4326)` for general geoms, or `geometry(Point, 4326)` +- [ ] Include python is activated for new functions. Include this before importing modules: `plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')` +- [ ] Docs for public-facing functions are written +- [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore `_`. + From f571e59a95da014f676eb00200bb02042f934584 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 14:05:59 -0400 Subject: [PATCH 45/81] adding hot/cold/outlier getters --- src/pg/sql/10_moran.sql | 174 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 164 insertions(+), 10 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index 85d69f0..5b7c68f 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -10,6 +10,7 @@ CREATE OR REPLACE FUNCTION num_ngbrs INT DEFAULT 5) RETURNS TABLE (moran NUMERIC, significance NUMERIC) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran(subquery, attr, num_ngbrs, permutations, geom_col, id_col, w_type) @@ -17,21 +18,93 @@ $$ LANGUAGE plpythonu; -- Moran's I Local CREATE OR REPLACE FUNCTION - CDB_AreasOfInterest_Local( + _CDB_AreasOfInterest_Local( subquery TEXT, attr TEXT, - permutations INT DEFAULT 99, - geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + permutations INT, + geom_col TEXT, + id_col TEXT, + w_type TEXT, + num_ngbrs INT) RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterest_Local( + subquery TEXT, + attr TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs); + +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + attr TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + -- Moran's I Rate (global) CREATE OR REPLACE FUNCTION CDB_AreasOfInterest_Global_Rate( @@ -45,6 +118,7 @@ CREATE OR REPLACE FUNCTION num_ngbrs INT DEFAULT 5) RETURNS TABLE (moran FLOAT, significance FLOAT) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) @@ -52,6 +126,25 @@ $$ LANGUAGE plpythonu; -- Moran's I Local Rate +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterest_Local_Rate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + permutations INT, + geom_col TEXT, + id_col TEXT, + w_type TEXT, + num_ngbrs INT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) +$$ LANGUAGE plpythonu; + CREATE OR REPLACE FUNCTION CDB_AreasOfInterest_Local_Rate( subquery TEXT, @@ -65,10 +158,71 @@ CREATE OR REPLACE FUNCTION RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ - from crankshaft.clustering import moran_local_rate - # TODO: use named parameters or a dictionary - return moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) -$$ LANGUAGE plpythonu; + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs); + +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots_Rate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots_Rate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers_Rate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +AS $$ + + SELECT moran, quads, significance, ids, y + FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; -- -- Moran's I Local Bivariate -- CREATE OR REPLACE FUNCTION From c3913459d9a8a45d5b6c2af54ed1d21626e0f82f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 14:54:42 -0400 Subject: [PATCH 46/81] adding tests for hotspot, coldspot, and outlier functions --- src/pg/test/expected/02_moran_test.out | 432 +++++++++++-------------- src/pg/test/sql/02_moran_test.sql | 29 ++ 2 files changed, 220 insertions(+), 241 deletions(-) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index eb2afa5..0b9267a 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -1,248 +1,198 @@ +\pset format unaligned +\set ECHO all \i test/fixtures/ppoints.sql --- test table (spanish province centroids with some invented values) -CREATE TABLE ppoints (cartodb_id integer, the_geom geometry, the_geom_webmercator geometry, code text, region_code text, value float); -INSERT INTO ppoints VALUES -( 1,'0101000020E6100000A8306DC0CBC305C051D14B6CE56A4540'::geometry,ST_Transform('0101000020E6100000A8306DC0CBC305C051D14B6CE56A4540'::geometry, 3857),'01','16',0.5), -( 4,'0101000020E6100000E220A4362DC202C0FD8AFA5119994240'::geometry,ST_Transform('0101000020E6100000E220A4362DC202C0FD8AFA5119994240'::geometry, 3857),'04','01',0.1), -( 5,'0101000020E610000004377E573AC813C0CB5871BB17494440'::geometry,ST_Transform('0101000020E610000004377E573AC813C0CB5871BB17494440'::geometry, 3857),'05','07',0.3), -( 2,'0101000020E610000000F49BE19BAFFFBF639958FDA6694340'::geometry,ST_Transform('0101000020E610000000F49BE19BAFFFBF639958FDA6694340'::geometry, 3857),'02','08',0.7), -( 3,'0101000020E61000005D0B7E63C832E2BFDB63EB00443D4340'::geometry,ST_Transform('0101000020E61000005D0B7E63C832E2BFDB63EB00443D4340'::geometry, 3857),'03','10',0.2), -( 6,'0101000020E61000006F3742B7FB9018C0DD967DC4D95A4340'::geometry,ST_Transform('0101000020E61000006F3742B7FB9018C0DD967DC4D95A4340'::geometry, 3857),'06','11',0.05), -( 7,'0101000020E6100000E4BB36995F4C0740EAC0E5CA9FC94340'::geometry,ST_Transform('0101000020E6100000E4BB36995F4C0740EAC0E5CA9FC94340'::geometry, 3857),'07','04',0.4), -( 8,'0101000020E61000003D43CC6CAFBEFF3F6B52E66F91DD4440'::geometry,ST_Transform('0101000020E61000003D43CC6CAFBEFF3F6B52E66F91DD4440'::geometry, 3857),'08','09',0.7), -( 9,'0101000020E61000003CC797BD99AF0CC0495A87FA312F4540'::geometry,ST_Transform('0101000020E61000003CC797BD99AF0CC0495A87FA312F4540'::geometry, 3857),'09','07',0.5), -(13,'0101000020E61000001CAA00A9F19F0EC05DF9267B7A764340'::geometry,ST_Transform('0101000020E61000001CAA00A9F19F0EC05DF9267B7A764340'::geometry, 3857),'13','08',0.4), -(16,'0101000020E6100000D8208F3CBC9001C065638DC1B1F24340'::geometry,ST_Transform('0101000020E6100000D8208F3CBC9001C065638DC1B1F24340'::geometry, 3857),'16','08',0.4), -(17,'0101000020E6100000E9E6A94A71630540AD7A0CB062104540'::geometry,ST_Transform('0101000020E6100000E9E6A94A71630540AD7A0CB062104540'::geometry, 3857),'17','09',0.6), -(18,'0101000020E6100000719792D59E240AC098AC548E00A84240'::geometry,ST_Transform('0101000020E6100000719792D59E240AC098AC548E00A84240'::geometry, 3857),'18','01',0.3), -(19,'0101000020E6100000972C878B50FD04C0123C881D1F684440'::geometry,ST_Transform('0101000020E6100000972C878B50FD04C0123C881D1F684440'::geometry, 3857),'19','08',0.7), -(21,'0101000020E6100000F7893E9934511BC0EAA4BF03E1C94240'::geometry,ST_Transform('0101000020E6100000F7893E9934511BC0EAA4BF03E1C94240'::geometry, 3857),'21','01',0.1), -(22,'0101000020E6100000572C2123B2A8B2BF7ED7FABAFD194540'::geometry,ST_Transform('0101000020E6100000572C2123B2A8B2BF7ED7FABAFD194540'::geometry, 3857),'22','02',0.4), -(25,'0101000020E6100000461B67D688C4F03FD990EEC3A0054540'::geometry,ST_Transform('0101000020E6100000461B67D688C4F03FD990EEC3A0054540'::geometry, 3857),'25','09',0.4), -(26,'0101000020E6100000A139FB06E82204C0539D84F62E234540'::geometry,ST_Transform('0101000020E6100000A139FB06E82204C0539D84F62E234540'::geometry, 3857),'26','17',0.6), -(27,'0101000020E6100000A92E54E618C91DC00D3A947B81814540'::geometry,ST_Transform('0101000020E6100000A92E54E618C91DC00D3A947B81814540'::geometry, 3857),'27','12',0.3), -(28,'0101000020E6100000971DC8B682BC0DC016D0E8055F3F4440'::geometry,ST_Transform('0101000020E6100000971DC8B682BC0DC016D0E8055F3F4440'::geometry, 3857),'28','13',0.8), -(30,'0101000020E6100000A2DC1964A8C5F7BF19299C994D004340'::geometry,ST_Transform('0101000020E6100000A2DC1964A8C5F7BF19299C994D004340'::geometry, 3857),'30','14',0.1), -(31,'0101000020E6100000DCA1FCC87B56FABF9B88E9D866554540'::geometry,ST_Transform('0101000020E6100000DCA1FCC87B56FABF9B88E9D866554540'::geometry, 3857),'31','15',0.9), -(32,'0101000020E6100000E1517AFCD15E1EC0A18D8D4825194540'::geometry,ST_Transform('0101000020E6100000E1517AFCD15E1EC0A18D8D4825194540'::geometry, 3857),'32','12',0.3), -(33,'0101000020E6100000A7FF33825AF917C0FABE7DFB6BA54540'::geometry,ST_Transform('0101000020E6100000A7FF33825AF917C0FABE7DFB6BA54540'::geometry, 3857),'33','03',0.4), -(34,'0101000020E6100000FB4E4EBEB72412C0898E7240982F4540'::geometry,ST_Transform('0101000020E6100000FB4E4EBEB72412C0898E7240982F4540'::geometry, 3857),'34','07',0.3), -(35,'0101000020E6100000224682B01B1A2DC011091656CC5C3C40'::geometry,ST_Transform('0101000020E6100000224682B01B1A2DC011091656CC5C3C40'::geometry, 3857),'35','05',0.3), -(36,'0101000020E6100000F7C9447110EC20C04C5D4823C7374540'::geometry,ST_Transform('0101000020E6100000F7C9447110EC20C04C5D4823C7374540'::geometry, 3857),'36','12',0.2), -(37,'0101000020E610000053D6A26DFB4218C09D58FAE209674440'::geometry,ST_Transform('0101000020E610000053D6A26DFB4218C09D58FAE209674440'::geometry, 3857),'37','07',0.5), -(38,'0101000020E6100000B1D1B5FC910431C03C0C89BA03503C40'::geometry,ST_Transform('0101000020E6100000B1D1B5FC910431C03C0C89BA03503C40'::geometry, 3857),'38','05',0.4), -(39,'0101000020E610000086E6FEE1BD1E10C00417096748994540'::geometry,ST_Transform('0101000020E610000086E6FEE1BD1E10C00417096748994540'::geometry, 3857),'39','06',0.6), -(40,'0101000020E6100000FB51C33F733710C038D01729E4954440'::geometry,ST_Transform('0101000020E6100000FB51C33F733710C038D01729E4954440'::geometry, 3857),'40','07',0.5), -(41,'0101000020E6100000912D6FDA28BB16C031321F08C4B74240'::geometry,ST_Transform('0101000020E6100000912D6FDA28BB16C031321F08C4B74240'::geometry, 3857),'41','01',0.4), -(42,'0101000020E6100000554432EABEB504C069ECD78775CF4440'::geometry,ST_Transform('0101000020E6100000554432EABEB504C069ECD78775CF4440'::geometry, 3857),'42','07',0.2), -(43,'0101000020E6100000157F117C1A2EEA3F027CD1F2368B4440'::geometry,ST_Transform('0101000020E6100000157F117C1A2EEA3F027CD1F2368B4440'::geometry, 3857),'43','09',0.3), -(44,'0101000020E610000051AA5B1BD718EABFEE67613BA4544440'::geometry,ST_Transform('0101000020E610000051AA5B1BD718EABFEE67613BA4544440'::geometry, 3857),'44','02',0.2), -(45,'0101000020E610000022C5C01BB69710C08563BC1499E54340'::geometry,ST_Transform('0101000020E610000022C5C01BB69710C08563BC1499E54340'::geometry, 3857),'45','08',0.3), -(46,'0101000020E6100000D5FCF78A11A0E9BFDEA46F8E64AF4340'::geometry,ST_Transform('0101000020E6100000D5FCF78A11A0E9BFDEA46F8E64AF4340'::geometry, 3857),'46','10',0.2), -(47,'0101000020E61000003AE63525866313C02100050B2BD14440'::geometry,ST_Transform('0101000020E61000003AE63525866313C02100050B2BD14440'::geometry, 3857),'47','07',0.3), -(48,'0101000020E610000030F187FD1FD206C0C767E1496C9E4540'::geometry,ST_Transform('0101000020E610000030F187FD1FD206C0C767E1496C9E4540'::geometry, 3857),'48','16',0.5), -(49,'0101000020E61000009C22867B12EC17C006C5F40C14DD4440'::geometry,ST_Transform('0101000020E61000009C22867B12EC17C006C5F40C14DD4440'::geometry, 3857),'49','07',0.2), -(50,'0101000020E6100000F7D5EFC62D08F1BF69D1231D68CF4440'::geometry,ST_Transform('0101000020E6100000F7D5EFC62D08F1BF69D1231D68CF4440'::geometry, 3857),'50','02',0.6), -(51,'0101000020E61000005B0E1F8DAA5F15C0530BFE285BF24140'::geometry,ST_Transform('0101000020E61000005B0E1F8DAA5F15C0530BFE285BF24140'::geometry, 3857),'51','18',0.01), -(10,'0101000020E61000000FD65D82AEA418C06192D1351FDB4340'::geometry,ST_Transform('0101000020E61000000FD65D82AEA418C06192D1351FDB4340'::geometry, 3857),'10','11',0.04), -(11,'0101000020E6100000B305531DAB0A17C0DEAFCD4EE5464240'::geometry,ST_Transform('0101000020E6100000B305531DAB0A17C0DEAFCD4EE5464240'::geometry, 3857),'11','01',0.08), -(12,'0101000020E610000059721A7297C9C2BF9EBE383BE51E4440'::geometry,ST_Transform('0101000020E610000059721A7297C9C2BF9EBE383BE51E4440'::geometry, 3857),'12','10',0.2), -(14,'0101000020E610000000C86313AF3C13C0E530879C10FF4240'::geometry,ST_Transform('0101000020E610000000C86313AF3C13C0E530879C10FF4240'::geometry, 3857),'14','01',0.2), -(15,'0101000020E61000002A475497B6ED20C06643D4131A904540'::geometry,ST_Transform('0101000020E61000002A475497B6ED20C06643D4131A904540'::geometry, 3857),'15','12',0.3), -(20,'0101000020E6100000F975566FAD8D01C0E840C33F67924540'::geometry,ST_Transform('0101000020E6100000F975566FAD8D01C0E840C33F67924540'::geometry, 3857),'20','16',0.8), -(23,'0101000020E610000025FA13E595880BC022BB07131D024340'::geometry,ST_Transform('0101000020E610000025FA13E595880BC022BB07131D024340'::geometry, 3857),'23','01',0.1), -(24,'0101000020E61000009C5F91C5095C17C0C78784B15A4F4540'::geometry,ST_Transform('0101000020E61000009C5F91C5095C17C0C78784B15A4F4540'::geometry, 3857),'24','07',0.3), -(29,'0101000020E6100000C34D4A5B48E712C092E680892C684240'::geometry,ST_Transform('0101000020E6100000C34D4A5B48E712C092E680892C684240'::geometry, 3857),'29','01',0.3), -(52,'0101000020E6100000406A545EB29A07C04E5F0BDA39A54140'::geometry,ST_Transform('0101000020E6100000406A545EB29A07C04E5F0BDA39A54140'::geometry, 3857),'52','19',0.01) -\i test/fixtures/ppoints2.sql --- test table (spanish province centroids with some invented values) -CREATE TABLE ppoints2 (cartodb_id integer, the_geom geometry, code text, region_code text, numerator float, denominator float); -INSERT INTO ppoints2 VALUES -( 1,'0101000020E6100000A8306DC0CBC305C051D14B6CE56A4540'::geometry,'01','16',0.5, 1.0), -( 4,'0101000020E6100000E220A4362DC202C0FD8AFA5119994240'::geometry,'04','01',0.1, 1.0), -( 5,'0101000020E610000004377E573AC813C0CB5871BB17494440'::geometry,'05','07',0.3, 1.0), -( 2,'0101000020E610000000F49BE19BAFFFBF639958FDA6694340'::geometry,'02','08',0.7, 1.0), -( 3,'0101000020E61000005D0B7E63C832E2BFDB63EB00443D4340'::geometry,'03','10',0.2, 1.0), -( 6,'0101000020E61000006F3742B7FB9018C0DD967DC4D95A4340'::geometry,'06','11',0.05, 1.0), -( 7,'0101000020E6100000E4BB36995F4C0740EAC0E5CA9FC94340'::geometry,'07','04',0.4, 1.0), -( 8,'0101000020E61000003D43CC6CAFBEFF3F6B52E66F91DD4440'::geometry,'08','09',0.7, 1.0), -( 9,'0101000020E61000003CC797BD99AF0CC0495A87FA312F4540'::geometry,'09','07',0.5, 1.0), -(13,'0101000020E61000001CAA00A9F19F0EC05DF9267B7A764340'::geometry,'13','08',0.4, 1.0), -(16,'0101000020E6100000D8208F3CBC9001C065638DC1B1F24340'::geometry,'16','08',0.4, 1.0), -(17,'0101000020E6100000E9E6A94A71630540AD7A0CB062104540'::geometry,'17','09',0.6, 1.0), -(18,'0101000020E6100000719792D59E240AC098AC548E00A84240'::geometry,'18','01',0.3, 1.0), -(19,'0101000020E6100000972C878B50FD04C0123C881D1F684440'::geometry,'19','08',0.7, 1.0), -(21,'0101000020E6100000F7893E9934511BC0EAA4BF03E1C94240'::geometry,'21','01',0.1, 1.0), -(22,'0101000020E6100000572C2123B2A8B2BF7ED7FABAFD194540'::geometry,'22','02',0.4, 1.0), -(25,'0101000020E6100000461B67D688C4F03FD990EEC3A0054540'::geometry,'25','09',0.4, 1.0), -(26,'0101000020E6100000A139FB06E82204C0539D84F62E234540'::geometry,'26','17',0.6, 1.0), -(27,'0101000020E6100000A92E54E618C91DC00D3A947B81814540'::geometry,'27','12',0.3, 1.0), -(28,'0101000020E6100000971DC8B682BC0DC016D0E8055F3F4440'::geometry,'28','13',0.8, 1.0), -(30,'0101000020E6100000A2DC1964A8C5F7BF19299C994D004340'::geometry,'30','14',0.1, 1.0), -(31,'0101000020E6100000DCA1FCC87B56FABF9B88E9D866554540'::geometry,'31','15',0.9, 1.0), -(32,'0101000020E6100000E1517AFCD15E1EC0A18D8D4825194540'::geometry,'32','12',0.3, 1.0), -(33,'0101000020E6100000A7FF33825AF917C0FABE7DFB6BA54540'::geometry,'33','03',0.4, 1.0), -(34,'0101000020E6100000FB4E4EBEB72412C0898E7240982F4540'::geometry,'34','07',0.3, 1.0), -(35,'0101000020E6100000224682B01B1A2DC011091656CC5C3C40'::geometry,'35','05',0.3, 1.0), -(36,'0101000020E6100000F7C9447110EC20C04C5D4823C7374540'::geometry,'36','12',0.2, 1.0), -(37,'0101000020E610000053D6A26DFB4218C09D58FAE209674440'::geometry,'37','07',0.5, 1.0), -(38,'0101000020E6100000B1D1B5FC910431C03C0C89BA03503C40'::geometry,'38','05',0.4, 1.0), -(39,'0101000020E610000086E6FEE1BD1E10C00417096748994540'::geometry,'39','06',0.6, 1.0), -(40,'0101000020E6100000FB51C33F733710C038D01729E4954440'::geometry,'40','07',0.5, 1.0), -(41,'0101000020E6100000912D6FDA28BB16C031321F08C4B74240'::geometry,'41','01',0.4, 1.0), -(42,'0101000020E6100000554432EABEB504C069ECD78775CF4440'::geometry,'42','07',0.2, 1.0), -(43,'0101000020E6100000157F117C1A2EEA3F027CD1F2368B4440'::geometry,'43','09',0.3, 1.0), -(44,'0101000020E610000051AA5B1BD718EABFEE67613BA4544440'::geometry,'44','02',0.2, 1.0), -(45,'0101000020E610000022C5C01BB69710C08563BC1499E54340'::geometry,'45','08',0.3, 1.0), -(46,'0101000020E6100000D5FCF78A11A0E9BFDEA46F8E64AF4340'::geometry,'46','10',0.2, 1.0), -(47,'0101000020E61000003AE63525866313C02100050B2BD14440'::geometry,'47','07',0.3, 1.0), -(48,'0101000020E610000030F187FD1FD206C0C767E1496C9E4540'::geometry,'48','16',0.5, 1.0), -(49,'0101000020E61000009C22867B12EC17C006C5F40C14DD4440'::geometry,'49','07',0.2, 1.0), -(50,'0101000020E6100000F7D5EFC62D08F1BF69D1231D68CF4440'::geometry,'50','02',0.6, 1.0), -(51,'0101000020E61000005B0E1F8DAA5F15C0530BFE285BF24140'::geometry,'51','18',0.01, 1.0), -(10,'0101000020E61000000FD65D82AEA418C06192D1351FDB4340'::geometry,'10','11',0.04, 1.0), -(11,'0101000020E6100000B305531DAB0A17C0DEAFCD4EE5464240'::geometry,'11','01',0.08, 1.0), -(12,'0101000020E610000059721A7297C9C2BF9EBE383BE51E4440'::geometry,'12','10',0.2, 1.0), -(14,'0101000020E610000000C86313AF3C13C0E530879C10FF4240'::geometry,'14','01',0.2, 1.0), -(15,'0101000020E61000002A475497B6ED20C06643D4131A904540'::geometry,'15','12',0.3, 1.0), -(20,'0101000020E6100000F975566FAD8D01C0E840C33F67924540'::geometry,'20','16',0.8, 1.0), -(23,'0101000020E610000025FA13E595880BC022BB07131D024340'::geometry,'23','01',0.1, 1.0), -(24,'0101000020E61000009C5F91C5095C17C0C78784B15A4F4540'::geometry,'24','07',0.3, 1.0), -(29,'0101000020E6100000C34D4A5B48E712C092E680892C684240'::geometry,'29','01',0.3, 1.0), -(52,'0101000020E6100000406A545EB29A07C04E5F0BDA39A54140'::geometry,'52','19',0.0, 1.01) --- Areas of Interest functions perform some nondeterministic computations --- (to estimate the significance); we will set the seeds for the RNGs --- that affect those results to have repeateble results -SELECT cdb_crankshaft._cdb_random_seeds(1234); - _cdb_random_seeds -------------------- - -(1 row) +SET client_min_messages TO WARNING; +\set ECHO none +_cdb_random_seeds -SELECT ppoints.code, m.quads - FROM ppoints - JOIN cdb_crankshaft.CDB_AreasOfInterest_Local('SELECT * FROM ppoints', 'value') m - ON ppoints.cartodb_id = m.ids - ORDER BY ppoints.code; - code | quads -------+------- - 01 | HH - 02 | HL - 03 | LL - 04 | LL - 05 | LH - 06 | LL - 07 | HH - 08 | HH - 09 | HH - 10 | LL - 11 | LL - 12 | LL - 13 | HL - 14 | LL - 15 | LL - 16 | HH - 17 | HH - 18 | LL - 19 | HH - 20 | HH - 21 | LL - 22 | HH - 23 | LL - 24 | LL - 25 | HH - 26 | HH - 27 | LL - 28 | HH - 29 | LL - 30 | LL - 31 | HH - 32 | LL - 33 | HL - 34 | LH - 35 | LL - 36 | LL - 37 | HL - 38 | HL - 39 | HH - 40 | HH - 41 | HL - 42 | LH - 43 | LH - 44 | LL - 45 | LH - 46 | LL - 47 | LL - 48 | HH - 49 | LH - 50 | HH - 51 | LL - 52 | LL +(1 row) +code|quads +01|HH +02|HL +03|LL +04|LL +05|LH +06|LL +07|HH +08|HH +09|HH +10|LL +11|LL +12|LL +13|HL +14|LL +15|LL +16|HH +17|HH +18|LL +19|HH +20|HH +21|LL +22|HH +23|LL +24|LL +25|HH +26|HH +27|LL +28|HH +29|LL +30|LL +31|HH +32|LL +33|HL +34|LH +35|LL +36|LL +37|HL +38|HL +39|HH +40|HH +41|HL +42|LH +43|LH +44|LL +45|LH +46|LL +47|LL +48|HH +49|LH +50|HH +51|LL +52|LL (52 rows) +_cdb_random_seeds -SELECT cdb_crankshaft._cdb_random_seeds(1234); - _cdb_random_seeds -------------------- - (1 row) - -SELECT ppoints2.code, m.quads - FROM ppoints2 - JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m - ON ppoints2.cartodb_id = m.ids - ORDER BY ppoints2.code; - code | quads -------+------- - 01 | LL - 02 | LH - 03 | HH - 04 | HH - 05 | LL - 06 | HH - 07 | LL - 08 | LL - 09 | LL - 10 | HH - 11 | HH - 12 | HL - 13 | LL - 14 | HH - 15 | LL - 16 | LL - 17 | LL - 18 | LH - 19 | LL - 20 | LL - 21 | HH - 22 | LL - 23 | HL - 24 | LL - 25 | LL - 26 | LL - 27 | LL - 28 | LL - 29 | LH - 30 | HH - 31 | LL - 32 | LL - 33 | LL - 34 | LL - 35 | LH - 36 | HL - 37 | LH - 38 | LH - 39 | LL - 40 | LL - 41 | LH - 42 | HL - 43 | LL - 44 | HL - 45 | LL - 46 | HL - 47 | LL - 48 | LL - 49 | HL - 50 | LL - 51 | HH +code|quads +01|LL +02|LH +03|HH +04|HH +05|LL +06|HH +07|LL +08|LL +09|LL +10|HH +11|HH +12|HL +13|LL +14|HH +15|LL +16|LL +17|LL +18|LH +19|LL +20|LL +21|HH +22|LL +23|HL +24|LL +25|LL +26|LL +27|LL +28|LL +29|LH +30|HH +31|LL +32|LL +33|LL +34|LL +35|LH +36|HL +37|LH +38|LH +39|LL +40|LL +41|LH +42|HL +43|LL +44|HL +45|LL +46|HL +47|LL +48|LL +49|HL +50|LL +51|HH (51 rows) +_cdb_random_seeds +(1 row) +code|quads +03|HH +04|HH +06|HH +10|HH +11|HH +12|HL +14|HH +21|HH +23|HL +30|HH +36|HL +42|HL +44|HL +46|HL +49|HL +51|HH +(16 rows) +_cdb_random_seeds + +(1 row) +code|quads +01|LL +02|LH +05|LL +07|LL +08|LL +09|LL +13|LL +15|LL +16|LL +17|LL +18|LH +19|LL +20|LL +22|LL +24|LL +25|LL +26|LL +27|LL +28|LL +29|LH +31|LL +32|LL +33|LL +34|LL +35|LH +37|LH +38|LH +39|LL +40|LL +41|LH +43|LL +45|LL +47|LL +48|LL +50|LL +(35 rows) +_cdb_random_seeds + +(1 row) +code|quads +02|LH +12|HL +18|LH +23|HL +29|LH +35|LH +36|HL +37|LH +38|LH +41|LH +42|HL +44|HL +46|HL +49|HL +(14 rows) diff --git a/src/pg/test/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql index de9c6cf..fd8dd58 100644 --- a/src/pg/test/sql/02_moran_test.sql +++ b/src/pg/test/sql/02_moran_test.sql @@ -1,3 +1,5 @@ +\pset format unaligned +\set ECHO all \i test/fixtures/ppoints.sql \i test/fixtures/ppoints2.sql @@ -19,3 +21,30 @@ SELECT ppoints2.code, m.quads JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; + +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +-- Spatial Hotspots (rate) +SELECT ppoints2.code, m.quads + FROM ppoints2 + JOIN cdb_crankshaft.CDB_GetSpatialHotspots_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + ON ppoints2.cartodb_id = m.ids + ORDER BY ppoints2.code; + +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +-- Spatial Coldspots (rate) +SELECT ppoints2.code, m.quads + FROM ppoints2 + JOIN cdb_crankshaft.CDB_GetSpatialColdspots_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + ON ppoints2.cartodb_id = m.ids + ORDER BY ppoints2.code; + +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +-- Spatial Outliers (rate) +SELECT ppoints2.code, m.quads + FROM ppoints2 + JOIN cdb_crankshaft.CDB_GetSpatialOutliers_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + ON ppoints2.cartodb_id = m.ids + ORDER BY ppoints2.code; \ No newline at end of file From bc8055a12b50d8fea2025f5f77613a21f141e37f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 14:55:16 -0400 Subject: [PATCH 47/81] adds pset format unaligned to reduce output for tests --- src/pg/test/fixtures/polyg_values.sql | 2 ++ src/pg/test/fixtures/ppoints.sql | 2 ++ src/pg/test/fixtures/ppoints2.sql | 2 ++ 3 files changed, 6 insertions(+) diff --git a/src/pg/test/fixtures/polyg_values.sql b/src/pg/test/fixtures/polyg_values.sql index 7aaf4a0..a76ca7c 100644 --- a/src/pg/test/fixtures/polyg_values.sql +++ b/src/pg/test/fixtures/polyg_values.sql @@ -1,3 +1,5 @@ +SET client_min_messages TO WARNING; +\set ECHO none CREATE TABLE values (cartodb_id integer, value float, the_geom geometry); INSERT INTO values(cartodb_id, value, the_geom) VALUES (1,10,'0106000020E61000000100000001030000000100000005000000E5AF3500C03608C08068629111374440C7BC0A00C00F02C0AC0551523B414440C7BC0A00C0A700C0CAF23B6E74FB4340A7267FFFFF5206C0FBB7E41B7EE74340E5AF3500C03608C08068629111374440'::geometry), diff --git a/src/pg/test/fixtures/ppoints.sql b/src/pg/test/fixtures/ppoints.sql index 10bc0c1..d0749e0 100644 --- a/src/pg/test/fixtures/ppoints.sql +++ b/src/pg/test/fixtures/ppoints.sql @@ -1,3 +1,5 @@ +SET client_min_messages TO WARNING; +\set ECHO none -- test table (spanish province centroids with some invented values) CREATE TABLE ppoints (cartodb_id integer, the_geom geometry, the_geom_webmercator geometry, code text, region_code text, value float); INSERT INTO ppoints VALUES diff --git a/src/pg/test/fixtures/ppoints2.sql b/src/pg/test/fixtures/ppoints2.sql index 873fd3b..3363955 100644 --- a/src/pg/test/fixtures/ppoints2.sql +++ b/src/pg/test/fixtures/ppoints2.sql @@ -1,3 +1,5 @@ +SET client_min_messages TO WARNING; +\set ECHO none -- test table (spanish province centroids with some invented values) CREATE TABLE ppoints2 (cartodb_id integer, the_geom geometry, code text, region_code text, numerator float, denominator float); INSERT INTO ppoints2 VALUES From b05ad98ed95345d7410ad625ebd6748757f084f2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 15:15:19 -0400 Subject: [PATCH 48/81] adding tests for hot/cold/outlier for normal moran functions --- src/pg/test/expected/02_moran_test.out | 79 ++++++++++++++++++++++++++ src/pg/test/sql/02_moran_test.sql | 29 ++++++++++ 2 files changed, 108 insertions(+) diff --git a/src/pg/test/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out index 0b9267a..3ba5bfd 100644 --- a/src/pg/test/expected/02_moran_test.out +++ b/src/pg/test/expected/02_moran_test.out @@ -62,6 +62,85 @@ code|quads (52 rows) _cdb_random_seeds +(1 row) +code|quads +01|HH +02|HL +07|HH +08|HH +09|HH +13|HL +16|HH +17|HH +19|HH +20|HH +22|HH +25|HH +26|HH +28|HH +31|HH +33|HL +37|HL +38|HL +39|HH +40|HH +41|HL +48|HH +50|HH +(23 rows) +_cdb_random_seeds + +(1 row) +code|quads +03|LL +04|LL +05|LH +06|LL +10|LL +11|LL +12|LL +14|LL +15|LL +18|LL +21|LL +23|LL +24|LL +27|LL +29|LL +30|LL +32|LL +34|LH +35|LL +36|LL +42|LH +43|LH +44|LL +45|LH +46|LL +47|LL +49|LH +51|LL +52|LL +(29 rows) +_cdb_random_seeds + +(1 row) +code|quads +02|HL +05|LH +13|HL +33|HL +34|LH +37|HL +38|HL +41|HL +42|LH +43|LH +45|LH +49|LH +(12 rows) +_cdb_random_seeds + (1 row) code|quads 01|LL diff --git a/src/pg/test/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql index fd8dd58..e984b0b 100644 --- a/src/pg/test/sql/02_moran_test.sql +++ b/src/pg/test/sql/02_moran_test.sql @@ -16,6 +16,35 @@ SELECT ppoints.code, m.quads SELECT cdb_crankshaft._cdb_random_seeds(1234); +-- Spatial Hotspots +SELECT ppoints.code, m.quads + FROM ppoints + JOIN cdb_crankshaft.CDB_GetSpatialHotspots('SELECT * FROM ppoints', 'value') m + ON ppoints.cartodb_id = m.ids + ORDER BY ppoints.code; + +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +-- Spatial Coldspots +SELECT ppoints.code, m.quads + FROM ppoints + JOIN cdb_crankshaft.CDB_GetSpatialColdspots('SELECT * FROM ppoints', 'value') m + ON ppoints.cartodb_id = m.ids + ORDER BY ppoints.code; + +SELECT cdb_crankshaft._cdb_random_seeds(1234); + + -- Spatial Outliers +SELECT ppoints.code, m.quads + FROM ppoints + JOIN cdb_crankshaft.CDB_GetSpatialOutliers('SELECT * FROM ppoints', 'value') m + ON ppoints.cartodb_id = m.ids + ORDER BY ppoints.code; + + +SELECT cdb_crankshaft._cdb_random_seeds(1234); + +-- Areas of Interest (rate) SELECT ppoints2.code, m.quads FROM ppoints2 JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m From 4e870e4393d61d64ee52ecfa7aeec1a4f177459e Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 15:15:41 -0400 Subject: [PATCH 49/81] adapt other test for new settings in fixtures --- src/pg/test/expected/03_overlap_sum_test.out | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/pg/test/expected/03_overlap_sum_test.out b/src/pg/test/expected/03_overlap_sum_test.out index 58f02cb..7638301 100644 --- a/src/pg/test/expected/03_overlap_sum_test.out +++ b/src/pg/test/expected/03_overlap_sum_test.out @@ -1,21 +1,11 @@ \i test/fixtures/polyg_values.sql -CREATE TABLE values (cartodb_id integer, value float, the_geom geometry); -INSERT INTO values(cartodb_id, value, the_geom) VALUES -(1,10,'0106000020E61000000100000001030000000100000005000000E5AF3500C03608C08068629111374440C7BC0A00C00F02C0AC0551523B414440C7BC0A00C0A700C0CAF23B6E74FB4340A7267FFFFF5206C0FBB7E41B7EE74340E5AF3500C03608C08068629111374440'::geometry), -(2,20,'0106000020E610000001000000010300000001000000050000002439EC00804AF7BF07D6CCB5C3064440C7BC0A00C0A700C0CAF23B6E74FB4340C7BC0A00C00F02C0AC0551523B414440E20CD5FFFF30FABFBE4F76AFEA4B44402439EC00804AF7BF07D6CCB5C3064440'::geometry) -SELECT round(cdb_crankshaft.cdb_overlap_sum( - '0106000020E61000000100000001030000000100000004000000FFFFFFFFFF3604C09A0B9ECEC42E444000000000C060FBBF30C7FD70E01D44400000000040AD02C06481F1C8CD034440FFFFFFFFFF3604C09A0B9ECEC42E4440'::geometry, - 'values', 'value' -), 2); +SET client_min_messages TO WARNING; +\set ECHO none round ------- 4.42 (1 row) -SELECT round(cdb_crankshaft.cdb_overlap_sum( - '0106000020E61000000100000001030000000100000004000000FFFFFFFFFF3604C09A0B9ECEC42E444000000000C060FBBF30C7FD70E01D44400000000040AD02C06481F1C8CD034440FFFFFFFFFF3604C09A0B9ECEC42E4440'::geometry, - 'values', 'value', schema_name := 'public' -), 2); round ------- 4.42 From ca5175f15b5dc1a2ad5822056eb8e29f9db1716d Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Fri, 20 May 2016 16:26:43 -0400 Subject: [PATCH 50/81] adding reference to subquery argument requirement --- .github/PULL_REQUEST_TEMPLATE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 941542d..882cece 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,4 +3,5 @@ - [ ] Include python is activated for new functions. Include this before importing modules: `plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')` - [ ] Docs for public-facing functions are written - [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore `_`. +- [ ] If appropriate, new functions accepts an arbitrary query as an input (see [Crankshaft Issue #6](https://github.com/CartoDB/crankshaft/issues/6) for more information) From 1b0d1cc82c562cd821c04d79393f89bbeaeeb27f Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Tue, 24 May 2016 17:51:29 -0400 Subject: [PATCH 51/81] updating function names --- src/pg/sql/10_moran.sql | 52 +++++++++++++------------------ src/pg/test/sql/02_moran_test.sql | 10 +++--- 2 files changed, 26 insertions(+), 36 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index 5b7c68f..b6dfc17 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -1,6 +1,6 @@ --- Moran's I (global) +-- Moran's I Global Measure (public-facing) CREATE OR REPLACE FUNCTION - CDB_AreasOfInterest_Global ( + CDB_AreasOfInterestGlobal ( subquery TEXT, attr_name TEXT, permutations INT DEFAULT 99, @@ -16,9 +16,9 @@ AS $$ return moran(subquery, attr, num_ngbrs, permutations, geom_col, id_col, w_type) $$ LANGUAGE plpythonu; --- Moran's I Local +-- Moran's I Local (internal function) CREATE OR REPLACE FUNCTION - _CDB_AreasOfInterest_Local( + _CDB_AreasOfInterestLocal( subquery TEXT, attr TEXT, permutations INT, @@ -34,8 +34,9 @@ AS $$ return moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; +-- Moran's I Local (public-facing function) CREATE OR REPLACE FUNCTION - CDB_AreasOfInterest_Local( + CDB_AreasOfInterestLocal( subquery TEXT, attr TEXT, permutations INT DEFAULT 99, @@ -51,6 +52,7 @@ AS $$ $$ LANGUAGE SQL; +-- Moran's I only for HH and HL (public-facing function) CREATE OR REPLACE FUNCTION CDB_GetSpatialHotspots( subquery TEXT, @@ -69,6 +71,7 @@ AS $$ $$ LANGUAGE SQL; +-- Moran's I only for LL and LH (public-facing function) CREATE OR REPLACE FUNCTION CDB_GetSpatialColdspots( subquery TEXT, @@ -87,6 +90,7 @@ AS $$ $$ LANGUAGE SQL; +-- Moran's I only for LH and HL (public-facing function) CREATE OR REPLACE FUNCTION CDB_GetSpatialOutliers( subquery TEXT, @@ -105,9 +109,9 @@ AS $$ $$ LANGUAGE SQL; --- Moran's I Rate (global) +-- Moran's I Global Rate (public-facing function) CREATE OR REPLACE FUNCTION - CDB_AreasOfInterest_Global_Rate( + CDB_AreasOfInterestGlobalRate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -125,9 +129,9 @@ AS $$ $$ LANGUAGE plpythonu; --- Moran's I Local Rate +-- Moran's I Local Rate (internal function) CREATE OR REPLACE FUNCTION - _CDB_AreasOfInterest_Local_Rate( + _CDB_AreasOfInterestLocalRate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -145,8 +149,9 @@ AS $$ return moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) $$ LANGUAGE plpythonu; +-- Moran's I Local Rate (public-facing function) CREATE OR REPLACE FUNCTION - CDB_AreasOfInterest_Local_Rate( + CDB_AreasOfInterestLocalRate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -164,8 +169,9 @@ AS $$ $$ LANGUAGE SQL; +-- Moran's I Local Rate only for HH and HL (public-facing function) CREATE OR REPLACE FUNCTION - CDB_GetSpatialHotspots_Rate( + CDB_GetSpatialHotspotsRate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -184,8 +190,9 @@ AS $$ $$ LANGUAGE SQL; +-- Moran's I Local Rate only for LL and LH (public-facing function) CREATE OR REPLACE FUNCTION - CDB_GetSpatialColdspots_Rate( + CDB_GetSpatialColdspotsRate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -204,8 +211,9 @@ AS $$ $$ LANGUAGE SQL; +-- Moran's I Local Rate only for LH and HL (public-facing function) CREATE OR REPLACE FUNCTION - CDB_GetSpatialOutliers_Rate( + CDB_GetSpatialOutliersRate( subquery TEXT, numerator TEXT, denominator TEXT, @@ -223,21 +231,3 @@ AS $$ WHERE quads IN ('HL', 'LH'); $$ LANGUAGE SQL; - --- -- Moran's I Local Bivariate --- CREATE OR REPLACE FUNCTION --- cdb_moran_local_bv( --- subquery TEXT, --- attr1 TEXT, --- attr2 TEXT, --- permutations INT DEFAULT 99, --- geom_col TEXT DEFAULT 'the_geom', --- id_col TEXT DEFAULT 'cartodb_id', --- w_type TEXT DEFAULT 'knn', --- num_ngbrs INT DEFAULT 5) --- RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) --- AS $$ --- from crankshaft.clustering import moran_local_bv --- # TODO: use named parameters or a dictionary --- return moran_local_bv(t, attr1, attr2, permutations, geom_col, id_col, w_type, num_ngbrs) --- $$ LANGUAGE plpythonu; diff --git a/src/pg/test/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql index e984b0b..ca56186 100644 --- a/src/pg/test/sql/02_moran_test.sql +++ b/src/pg/test/sql/02_moran_test.sql @@ -10,7 +10,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints - JOIN cdb_crankshaft.CDB_AreasOfInterest_Local('SELECT * FROM ppoints', 'value') m + JOIN cdb_crankshaft.CDB_AreasOfInterestLocal('SELECT * FROM ppoints', 'value') m ON ppoints.cartodb_id = m.ids ORDER BY ppoints.code; @@ -47,7 +47,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); -- Areas of Interest (rate) SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.CDB_AreasOfInterest_Local_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.CDB_AreasOfInterestLocalRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; @@ -56,7 +56,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); -- Spatial Hotspots (rate) SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.CDB_GetSpatialHotspots_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.CDB_GetSpatialHotspotsRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; @@ -65,7 +65,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); -- Spatial Coldspots (rate) SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.CDB_GetSpatialColdspots_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.CDB_GetSpatialColdspotsRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; @@ -74,6 +74,6 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); -- Spatial Outliers (rate) SELECT ppoints2.code, m.quads FROM ppoints2 - JOIN cdb_crankshaft.CDB_GetSpatialOutliers_Rate('SELECT * FROM ppoints2', 'numerator', 'denominator') m + JOIN cdb_crankshaft.CDB_GetSpatialOutliersRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m ON ppoints2.cartodb_id = m.ids ORDER BY ppoints2.code; \ No newline at end of file From b5c6b42081060c8cfc89f643620efdb8aced53d3 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 11:33:02 -0400 Subject: [PATCH 52/81] update names to align with CamelCase convention --- src/pg/sql/10_moran.sql | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index b6dfc17..a6e9ec6 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -1,6 +1,6 @@ -- Moran's I Global Measure (public-facing) CREATE OR REPLACE FUNCTION - CDB_AreasOfInterestGlobal ( + CDB_AreasOfInterestGlobal( subquery TEXT, attr_name TEXT, permutations INT DEFAULT 99, @@ -48,7 +48,7 @@ RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMER AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs); + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs); $$ LANGUAGE SQL; @@ -66,7 +66,7 @@ CREATE OR REPLACE FUNCTION AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) WHERE quads IN ('HH', 'HL'); $$ LANGUAGE SQL; @@ -85,7 +85,7 @@ CREATE OR REPLACE FUNCTION AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) WHERE quads IN ('LL', 'LH'); $$ LANGUAGE SQL; @@ -104,7 +104,7 @@ CREATE OR REPLACE FUNCTION AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) WHERE quads IN ('HL', 'LH'); $$ LANGUAGE SQL; @@ -165,7 +165,7 @@ TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs); + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs); $$ LANGUAGE SQL; @@ -185,7 +185,7 @@ TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) WHERE quads IN ('HH', 'HL'); $$ LANGUAGE SQL; @@ -206,7 +206,7 @@ TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) WHERE quads IN ('LL', 'LH'); $$ LANGUAGE SQL; @@ -227,7 +227,7 @@ TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterest_Local_Rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) WHERE quads IN ('HL', 'LH'); $$ LANGUAGE SQL; From 2b8adb744d24d4f28a202730b19cca06ecea0319 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 12:09:43 -0400 Subject: [PATCH 53/81] switched signature to put more common options in the front --- src/pg/sql/10_moran.sql | 96 +++++++++---------- .../crankshaft/crankshaft/clustering/moran.py | 8 +- .../crankshaft/test/test_clustering_moran.py | 6 +- 3 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index a6e9ec6..fdc2d0d 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -3,17 +3,17 @@ CREATE OR REPLACE FUNCTION CDB_AreasOfInterestGlobal( subquery TEXT, attr_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, significance NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary - return moran(subquery, attr, num_ngbrs, permutations, geom_col, id_col, w_type) + return moran(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- Moran's I Local (internal function) @@ -21,17 +21,17 @@ CREATE OR REPLACE FUNCTION _CDB_AreasOfInterestLocal( subquery TEXT, attr TEXT, + w_type TEXT, + num_ngbrs INT, permutations INT, geom_col TEXT, - id_col TEXT, - w_type TEXT, - num_ngbrs INT) + id_col TEXT) RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary - return moran_local(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + return moran_local(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- Moran's I Local (public-facing function) @@ -39,16 +39,16 @@ CREATE OR REPLACE FUNCTION CDB_AreasOfInterestLocal( subquery TEXT, attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs); + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -57,16 +57,16 @@ CREATE OR REPLACE FUNCTION CDB_GetSpatialHotspots( subquery TEXT, attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); $$ LANGUAGE SQL; @@ -76,16 +76,16 @@ CREATE OR REPLACE FUNCTION CDB_GetSpatialColdspots( subquery TEXT, attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('LL', 'LH'); $$ LANGUAGE SQL; @@ -95,16 +95,16 @@ CREATE OR REPLACE FUNCTION CDB_GetSpatialOutliers( subquery TEXT, attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HL', 'LH'); $$ LANGUAGE SQL; @@ -115,17 +115,17 @@ CREATE OR REPLACE FUNCTION subquery TEXT, numerator TEXT, denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran FLOAT, significance FLOAT) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary - return moran_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; @@ -135,18 +135,18 @@ CREATE OR REPLACE FUNCTION subquery TEXT, numerator TEXT, denominator TEXT, + w_type TEXT, + num_ngbrs INT, permutations INT, geom_col TEXT, - id_col TEXT, - w_type TEXT, - num_ngbrs INT) + id_col TEXT) RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate # TODO: use named parameters or a dictionary - return moran_local_rate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- Moran's I Local Rate (public-facing function) @@ -155,17 +155,17 @@ CREATE OR REPLACE FUNCTION subquery TEXT, numerator TEXT, denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs); + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -175,17 +175,17 @@ CREATE OR REPLACE FUNCTION subquery TEXT, numerator TEXT, denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); $$ LANGUAGE SQL; @@ -196,17 +196,17 @@ CREATE OR REPLACE FUNCTION subquery TEXT, numerator TEXT, denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('LL', 'LH'); $$ LANGUAGE SQL; @@ -217,17 +217,17 @@ CREATE OR REPLACE FUNCTION subquery TEXT, numerator TEXT, denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn', - num_ngbrs INT DEFAULT 5) + id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) AS $$ SELECT moran, quads, significance, ids, y - FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, permutations, geom_col, id_col, w_type, num_ngbrs) + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HL', 'LH'); $$ LANGUAGE SQL; diff --git a/src/py/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py index 2a043c3..39b3ff6 100644 --- a/src/py/crankshaft/crankshaft/clustering/moran.py +++ b/src/py/crankshaft/crankshaft/clustering/moran.py @@ -14,7 +14,7 @@ import crankshaft.pysal_utils as pu # High level interface --------------------------------------- def moran(subquery, attr_name, - permutations, geom_col, id_col, w_type, num_ngbrs): + w_type, num_ngbrs, permutations, geom_col, id_col): """ Moran's I (global) Implementation building neighbors with a PostGIS database and Moran's I @@ -56,7 +56,7 @@ def moran(subquery, attr_name, return zip([moran_global.I], [moran_global.EI]) def moran_local(subquery, attr, - permutations, geom_col, id_col, w_type, num_ngbrs): + w_type, num_ngbrs, permutations, geom_col, id_col): """ Moran's I implementation for PL/Python Andy Eschbacher @@ -96,7 +96,7 @@ def moran_local(subquery, attr, return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) def moran_rate(subquery, numerator, denominator, - permutations, geom_col, id_col, w_type, num_ngbrs): + w_type, num_ngbrs, permutations, geom_col, id_col): """ Moran's I Rate (global) Andy Eschbacher @@ -137,7 +137,7 @@ def moran_rate(subquery, numerator, denominator, return zip([lisa_rate.I], [lisa_rate.EI]) def moran_local_rate(subquery, numerator, denominator, - permutations, geom_col, id_col, w_type, num_ngbrs): + w_type, num_ngbrs, permutations, geom_col, id_col): """ Moran's I Local Rate Andy Eschbacher diff --git a/src/py/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py index 29c5bde..393e93b 100644 --- a/src/py/crankshaft/test/test_clustering_moran.py +++ b/src/py/crankshaft/test/test_clustering_moran.py @@ -52,7 +52,7 @@ class MoranTest(unittest.TestCase): data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.moran_local('subquery', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') result = [(row[0], row[1]) for row in result] expected = self.moran_data for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): @@ -64,7 +64,7 @@ class MoranTest(unittest.TestCase): data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1234) - result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 99, 'the_geom', 'cartodb_id', 'knn', 5) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') print 'result == None? ', result == None result = [(row[0], row[1]) for row in result] expected = self.moran_data @@ -76,7 +76,7 @@ class MoranTest(unittest.TestCase): data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] plpy._define_result('select', data) random_seeds.set_random_seeds(1235) - result = cc.moran('table', 'value', 99, 'the_geom', 'cartodb_id', 'knn', 5) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') print 'result == None?', result == None result_moran = result[0][0] expected_moran = np.array([row[0] for row in self.moran_data]).mean() From 3013998e1b64d4688bd3fec8a4cf23980e50efe5 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 12:19:08 -0400 Subject: [PATCH 54/81] make functions more flexible to case of weight type --- src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py index fa06e26..02b5e35 100644 --- a/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py +++ b/src/py/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -11,7 +11,7 @@ def construct_neighbor_query(w_type, query_vals): @param query_vals dict: values used to construct the query """ - if w_type == 'knn': + if w_type.lower() == 'knn': return knn(query_vals) else: return queen(query_vals) @@ -22,7 +22,7 @@ def get_weight(query_res, w_type='knn', num_ngbrs=5): Construct PySAL weight from return value of query @param query_res: query results with attributes and neighbors """ - if w_type == 'knn': + if w_type.lower() == 'knn': row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs weights = {x['id']: row_normed_weights for x in query_res} else: From 90c3e21c0dce970b9781ebc9c4dd0251dccefb73 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 12:43:33 -0400 Subject: [PATCH 55/81] renaming output id to rowid --- src/pg/sql/10_moran.sql | 36 +++++++++++++++---------------- src/pg/test/sql/02_moran_test.sql | 20 ++++++++--------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index fdc2d0d..d0730e9 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -26,7 +26,7 @@ CREATE OR REPLACE FUNCTION permutations INT, geom_col TEXT, id_col TEXT) -RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local @@ -44,10 +44,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') -RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -62,10 +62,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') - RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); @@ -81,10 +81,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') - RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('LL', 'LH'); @@ -100,10 +100,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') - RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HL', 'LH'); @@ -141,7 +141,7 @@ CREATE OR REPLACE FUNCTION geom_col TEXT, id_col TEXT) RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate @@ -161,10 +161,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -181,10 +181,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); @@ -202,10 +202,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('LL', 'LH'); @@ -223,10 +223,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, ids INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) AS $$ - SELECT moran, quads, significance, ids, y + SELECT moran, quads, significance, rowid, y FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HL', 'LH'); diff --git a/src/pg/test/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql index ca56186..5545a4a 100644 --- a/src/pg/test/sql/02_moran_test.sql +++ b/src/pg/test/sql/02_moran_test.sql @@ -11,7 +11,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints JOIN cdb_crankshaft.CDB_AreasOfInterestLocal('SELECT * FROM ppoints', 'value') m - ON ppoints.cartodb_id = m.ids + ON ppoints.cartodb_id = m.rowid ORDER BY ppoints.code; SELECT cdb_crankshaft._cdb_random_seeds(1234); @@ -20,7 +20,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints JOIN cdb_crankshaft.CDB_GetSpatialHotspots('SELECT * FROM ppoints', 'value') m - ON ppoints.cartodb_id = m.ids + ON ppoints.cartodb_id = m.rowid ORDER BY ppoints.code; SELECT cdb_crankshaft._cdb_random_seeds(1234); @@ -29,7 +29,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints JOIN cdb_crankshaft.CDB_GetSpatialColdspots('SELECT * FROM ppoints', 'value') m - ON ppoints.cartodb_id = m.ids + ON ppoints.cartodb_id = m.rowid ORDER BY ppoints.code; SELECT cdb_crankshaft._cdb_random_seeds(1234); @@ -38,7 +38,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints.code, m.quads FROM ppoints JOIN cdb_crankshaft.CDB_GetSpatialOutliers('SELECT * FROM ppoints', 'value') m - ON ppoints.cartodb_id = m.ids + ON ppoints.cartodb_id = m.rowid ORDER BY ppoints.code; @@ -48,16 +48,16 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 JOIN cdb_crankshaft.CDB_AreasOfInterestLocalRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m - ON ppoints2.cartodb_id = m.ids + ON ppoints2.cartodb_id = m.rowid ORDER BY ppoints2.code; - + SELECT cdb_crankshaft._cdb_random_seeds(1234); -- Spatial Hotspots (rate) SELECT ppoints2.code, m.quads FROM ppoints2 JOIN cdb_crankshaft.CDB_GetSpatialHotspotsRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m - ON ppoints2.cartodb_id = m.ids + ON ppoints2.cartodb_id = m.rowid ORDER BY ppoints2.code; SELECT cdb_crankshaft._cdb_random_seeds(1234); @@ -66,7 +66,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 JOIN cdb_crankshaft.CDB_GetSpatialColdspotsRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m - ON ppoints2.cartodb_id = m.ids + ON ppoints2.cartodb_id = m.rowid ORDER BY ppoints2.code; SELECT cdb_crankshaft._cdb_random_seeds(1234); @@ -75,5 +75,5 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SELECT ppoints2.code, m.quads FROM ppoints2 JOIN cdb_crankshaft.CDB_GetSpatialOutliersRate('SELECT * FROM ppoints2', 'numerator', 'denominator') m - ON ppoints2.cartodb_id = m.ids - ORDER BY ppoints2.code; \ No newline at end of file + ON ppoints2.cartodb_id = m.rowid + ORDER BY ppoints2.code; From 59c520da165d8a232a470452c4eab6105aff557c Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 13:40:32 -0400 Subject: [PATCH 56/81] renaming output value to vals --- src/pg/sql/10_moran.sql | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index d0730e9..9e7adf2 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -26,7 +26,7 @@ CREATE OR REPLACE FUNCTION permutations INT, geom_col TEXT, id_col TEXT) -RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local @@ -44,10 +44,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') -RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -62,10 +62,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') - RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); @@ -81,10 +81,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') - RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('LL', 'LH'); @@ -100,10 +100,10 @@ CREATE OR REPLACE FUNCTION permutations INT DEFAULT 99, geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') - RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HL', 'LH'); @@ -141,7 +141,7 @@ CREATE OR REPLACE FUNCTION geom_col TEXT, id_col TEXT) RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate @@ -161,10 +161,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -181,10 +181,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); @@ -202,10 +202,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('LL', 'LH'); @@ -223,10 +223,10 @@ CREATE OR REPLACE FUNCTION geom_col TEXT DEFAULT 'the_geom', id_col TEXT DEFAULT 'cartodb_id') RETURNS -TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, y NUMERIC) +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - SELECT moran, quads, significance, rowid, y + SELECT moran, quads, significance, rowid, vals FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HL', 'LH'); From 0e3970f52c12009492e47322ce9b70ba5e9c027a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 13:46:12 -0400 Subject: [PATCH 57/81] adds docs for areasofinterestlocal --- doc/02_moran.md | 127 ++++++++++++++++++------------------------------ 1 file changed, 46 insertions(+), 81 deletions(-) diff --git a/doc/02_moran.md b/doc/02_moran.md index c91eb3f..095c44d 100644 --- a/doc/02_moran.md +++ b/doc/02_moran.md @@ -1,8 +1,50 @@ -## Name +## Areas of Interest Functions -CDB_AreasOfInterest -- returns a table with a cluster/outlier classification, the significance of a classification, an autocorrelation statistic (Local Moran's I), and the geometry id for each geometry in the original dataset. -## Synopsis + +### CDB_AreasOfInterestLocal(subquery text, column_name text) + +This function classifies your data as being part of a cluster, as an outlier, or or not part of a pattern based the significance of a classification. The classification happens through an autocorrelation statistic called Local Moran's I. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| column_name | TEXT | Name of column (e.g., should `'interesting_value'` instead of `interesting_value` without single quotes) used for the analysis. | +| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySal's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | +| num_ngbrs (optional) | INT | Number of neighbors if using k-nearest neighbors weight type. Defaults to 5. | +| permutations (optional) | INT | Number of permutations to check against a random arrangement of the values in `column_name`. This influences the accuracy of the output field `significance`. Defaults to 99. | +| geom_col | TEXT | The column name for the geometries. Defaults to `'the_geom'` | +| id_col | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| moran | NUMERIC | Value of Moran's I (spatial autocorrelation measure) for the geometry with id of `rowid` | +| quads | TEXT | Classification of geometry. Result is one of 'HH' (a high value with neighbors high on average), 'LL' (opposite of 'HH'), 'HL' (a high value surrounded by lows on average), and 'LH' (opposite of 'HL'). Null values are returned when nulls exist in the original data. | +| significance | NUMERIC | The statistical significance (from 0 to 1) of a cluster or outlier classification. Lower numbers are more significant. | +| rowid | INT | Row id of the values which correspond to the input rows. | +| vals | NUMERIC | Values from `'column_name'`. | + + +#### Example Usage + +```sql +SELECT + c.the_geom, + aoi.quads, + aoi.significance, + c.cyclists_per_total_population +FROM CDB_GetAreasOfInterestLocal('SELECT * FROM commute_data' + 'cyclists_per_total_population') As aoi +JOIN commute_data As c +ON c.cartodb_id = aoi.rowid; +``` + ```sql table(numeric moran_val, text quadrant, numeric significance, int ids, numeric column_values) CDB_AreasOfInterest(text query, text column_name) @@ -89,81 +131,4 @@ crankshaft's areas of interest functions: * [CDB_AreasOfInterest_Rate_Local]() * [CDB_AreasOfInterest_Rate_Global]() - -PostGIS clustering functions: - -* [ST_ClusterIntersecting](http://postgis.net/docs/manual-2.2/ST_ClusterIntersecting.html) -* [ST_ClusterWithin](http://postgis.net/docs/manual-2.2/ST_ClusterWithin.html) - - --- removing below, working into above - -#### What is Moran's I and why is it significant for CartoDB? - -Moran's I is a geostatistical calculation which gives a measure of the global -clustering and presence of outliers within the geographies in a map. Here global -means over all of the geographies in a dataset. Imagine mapping the incidence -rates of cancer in neighborhoods of a city. If there were areas covering several -neighborhoods with abnormally low rates of cancer, those areas are positively -spatially correlated with one another and would be considered a cluster. If -there was a single neighborhood with a high rate but with all neighbors on -average having a low rate, it would be considered a spatial outlier. - -While Moran's I gives a global snapshot, there are local indicators for -clustering called Local Indicators of Spatial Autocorrelation. Clustering is a -process related to autocorrelation -- i.e., a process that compares a -geography's attribute to the attribute in neighbor geographies. - -For the example of cancer rates in neighborhoods, since these neighborhoods have -a high value for rate of cancer, and all of their neighbors do as well, they are -designated as "High High" or simply **HH**. For areas with multiple neighborhoods -with low rates of cancer, they are designated as "Low Low" or **LL**. HH and LL -naturally fit into the concept of clustering and are in the correlated -variables. - -"Anticorrelated" geogs are in **LH** and **HL** regions -- that is, regions -where a geog has a high value and it's neighbors, on average, have a low value -(or vice versa). An example of this is a "gated community" or placement of a -city housing project in a rich region. These deliberate developments have -opposite median income as compared to the neighbors around them. They have a -high (or low) value while their neighbors have a low (or high) value. They exist -typically as islands, and in rare circumstances can extend as chains dividing -**LL** or **HH**. - -Strong policies such as rent stabilization (probably) tend to prevent the -clustering of high rent areas as they integrate middle class incomes. Luxury -apartment buildings, which are a kind of gated community, probably tend to skew -an area's median income upwards while housing projects have the opposite effect. -What are the nuggets in the analysis? - -Two functions are available to compute Moran I statistics: - -* `cdb_moran_local` computes Moran I measures, quad classification and - significance values from numerial values associated to geometry entities - in an input table. The geometries should be contiguous polygons When - then `queen` `w_type` is used. -* `cdb_moran_local_rate` computes the same statistics using a ratio between - numerator and denominator columns of a table. - -The parameters for `cdb_moran_local` are: - -* `table` name of the table that contains the data values -* `attr` name of the column -* `signficance` significance threshold for the quads values -* `num_ngbrs` number of neighbors to consider (default: 5) -* `permutations` number of random permutations for calculation of - pseudo-p values (default: 99) -* `geom_column` number of the geometry column (default: "the_geom") -* `id_col` PK column of the table (default: "cartodb_id") -* `w_type` Weight types: can be "knn" for k-nearest neighbor weights - or "queen" for contiguity based weights. - -The function returns a table with the following columns: - -* `moran` Moran's value -* `quads` quad classification ('HH', 'LL', 'HL', 'LH' or 'Not significant') -* `significance` significance value -* `ids` id of the corresponding record in the input table - -Function `cdb_moran_local_rate` only differs in that the `attr` input -parameter is substituted by `numerator` and `denominator`. +## Hotspot, Coldspot, and Outlier Functions From ae1bb703a74ff29700d87c2a6041405ff151540a Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 15:06:58 -0400 Subject: [PATCH 58/81] descriptions for all functions --- doc/02_moran.md | 237 +++++++++++++++++++++++++++++------------------- 1 file changed, 144 insertions(+), 93 deletions(-) diff --git a/doc/02_moran.md b/doc/02_moran.md index 095c44d..2d9ae99 100644 --- a/doc/02_moran.md +++ b/doc/02_moran.md @@ -1,22 +1,20 @@ ## Areas of Interest Functions - - ### CDB_AreasOfInterestLocal(subquery text, column_name text) -This function classifies your data as being part of a cluster, as an outlier, or or not part of a pattern based the significance of a classification. The classification happens through an autocorrelation statistic called Local Moran's I. +This function classifies your data as being part of a cluster, as an outlier, or not part of a pattern based the significance of a classification. The classification happens through an autocorrelation statistic called Local Moran's I. #### Arguments | Name | Type | Description | |------|------|-------------| | subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | -| column_name | TEXT | Name of column (e.g., should `'interesting_value'` instead of `interesting_value` without single quotes) used for the analysis. | -| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySal's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | +| column_name | TEXT | Name of column (e.g., should be `'interesting_value'` instead of `interesting_value` without single quotes) used for the analysis. | +| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySAL's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | | num_ngbrs (optional) | INT | Number of neighbors if using k-nearest neighbors weight type. Defaults to 5. | | permutations (optional) | INT | Number of permutations to check against a random arrangement of the values in `column_name`. This influences the accuracy of the output field `significance`. Defaults to 99. | -| geom_col | TEXT | The column name for the geometries. Defaults to `'the_geom'` | -| id_col | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | +| geom_col (optional) | TEXT | The column name for the geometries. Defaults to `'the_geom'` | +| id_col (optional) | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | #### Returns @@ -24,8 +22,84 @@ A table with the following columns. | Column Name | Type | Description | |-------------|------|-------------| -| moran | NUMERIC | Value of Moran's I (spatial autocorrelation measure) for the geometry with id of `rowid` | -| quads | TEXT | Classification of geometry. Result is one of 'HH' (a high value with neighbors high on average), 'LL' (opposite of 'HH'), 'HL' (a high value surrounded by lows on average), and 'LH' (opposite of 'HL'). Null values are returned when nulls exist in the original data. | +| moran | NUMERIC | Value of Moran's I (spatial autocorrelation measure) for the geometry with id of `rowid` | +| quads | TEXT | Classification of geometry. Result is one of 'HH' (a high value with neighbors high on average), 'LL' (opposite of 'HH'), 'HL' (a high value surrounded by lows on average), and 'LH' (opposite of 'HL'). Null values are returned when nulls exist in the original data. | +| significance | NUMERIC | The statistical significance (from 0 to 1) of a cluster or outlier classification. Lower numbers are more significant. | +| rowid | INT | Row id of the values which correspond to the input rows. | +| vals | NUMERIC | Values from `'column_name'`. | + + +#### Example Usage + +```sql +SELECT + c.the_geom, + aoi.quads, + aoi.significance, + c.num_cyclists_per_total_population +FROM CDB_GetAreasOfInterestLocal('SELECT * FROM commute_data' + 'num_cyclists_per_total_population') As aoi +JOIN commute_data As c +ON c.cartodb_id = aoi.rowid; +``` + +### CDB_AreasOfInterestGlobal(subquery text, column_name text) + +This function identifies the extent to which geometries cluster (the groupings of geometries with similarly high or low values relative to the mean) or form outliers (areas where geometries have values opposite of their neighbors). The output of this function gives values between -1 and 1 as well as a significance of that classification. Values close to 0 mean that there is little to no distribution of values as compared to what one would see in a randomly distributed collection of geometries and values. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| column_name | TEXT | Name of column (e.g., should be `'interesting_value'` instead of `interesting_value` without single quotes) used for the analysis. | +| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySAL's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | +| num_ngbrs (optional) | INT | Number of neighbors if using k-nearest neighbors weight type. Defaults to 5. | +| permutations (optional) | INT | Number of permutations to check against a random arrangement of the values in `column_name`. This influences the accuracy of the output field `significance`. Defaults to 99. | +| geom_col (optional) | TEXT | The column name for the geometries. Defaults to `'the_geom'` | +| id_col (optional) | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| moran | NUMERIC | Value of Moran's I (spatial autocorrelation measure) for the entire dataset. Values closer to one indicate cluster, closer to -1 mean more outliers, and near zero indicates a random distribution of data. | +| significance | NUMERIC | The statistical significance of the `moran` measure. | + +#### Examples + +```sql +SELECT * +FROM CDB_AreasOfInterestGlobal('SELECT * FROM commute_data', 'num_cyclists_per_total_population') +``` + +### CDB_AreasOfInterestLocalRate(subquery text, numerator_column text, denominator_column text) + +Just like `CDB_AreasOfInterestLocal`, this function classifies your data as being part of a cluster, as an outlier, or not part of a pattern based the significance of a classification. This function differs in that it calculates the classifications based on input `numerator` and `denominator` columns for finding the areas where there are clusters and outliers for the resulting rate of those two values. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| numerator | TEXT | Name of the numerator for forming a rate to be used in analysis. | +| denominator | TEXT | Name of the denominator for forming a rate to be used in analysis. | +| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySAL's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | +| num_ngbrs (optional) | INT | Number of neighbors if using k-nearest neighbors weight type. Defaults to 5. | +| permutations (optional) | INT | Number of permutations to check against a random arrangement of the values in `column_name`. This influences the accuracy of the output field `significance`. Defaults to 99. | +| geom_col (optional) | TEXT | The column name for the geometries. Defaults to `'the_geom'` | +| id_col (optional) | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| moran | NUMERIC | Value of Moran's I (spatial autocorrelation measure) for the geometry with id of `rowid` | +| quads | TEXT | Classification of geometry. Result is one of 'HH' (a high value with neighbors high on average), 'LL' (opposite of 'HH'), 'HL' (a high value surrounded by lows on average), and 'LH' (opposite of 'HL'). Null values are returned when nulls exist in the original data. | | significance | NUMERIC | The statistical significance (from 0 to 1) of a cluster or outlier classification. Lower numbers are more significant. | | rowid | INT | Row id of the values which correspond to the input rows. | | vals | NUMERIC | Values from `'column_name'`. | @@ -39,96 +113,73 @@ SELECT aoi.quads, aoi.significance, c.cyclists_per_total_population -FROM CDB_GetAreasOfInterestLocal('SELECT * FROM commute_data' - 'cyclists_per_total_population') As aoi +FROM CDB_GetAreasOfInterestLocalRate('SELECT * FROM commute_data' + 'num_cyclists', + 'total_population') As aoi JOIN commute_data As c ON c.cartodb_id = aoi.rowid; ``` +### CDB_AreasOfInterestGlobalRate(subquery text, column_name text) + +This function identifies the extent to which geometries cluster (the groupings of geometries with similarly high or low values relative to the mean) or form outliers (areas where geometries have values opposite of their neighbors). The output of this function gives values between -1 and 1 as well as a significance of that classification. Values close to 0 mean that there is little to no distribution of values as compared to what one would see in a randomly distributed collection of geometries and values. + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| numerator | TEXT | Name of the numerator for forming a rate to be used in analysis. | +| denominator | TEXT | Name of the denominator for forming a rate to be used in analysis. | +| weight type (optional) | TEXT | Type of weight to use when finding neighbors. Currently available options are 'knn' (default) and 'queen'. Read more about weight types in [PySAL's weights documentation](https://pysal.readthedocs.io/en/v1.11.0/users/tutorials/weights.html). | +| num_ngbrs (optional) | INT | Number of neighbors if using k-nearest neighbors weight type. Defaults to 5. | +| permutations (optional) | INT | Number of permutations to check against a random arrangement of the values in `column_name`. This influences the accuracy of the output field `significance`. Defaults to 99. | +| geom_col (optional) | TEXT | The column name for the geometries. Defaults to `'the_geom'` | +| id_col (optional) | TEXT | The column name for the unique ID of each geometry/value pair. Defaults to `'cartodb_id'`. | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| moran | NUMERIC | Value of Moran's I (spatial autocorrelation measure) for the entire dataset. Values closer to one indicate cluster, closer to -1 mean more outliers, and near zero indicates a random distribution of data. | +| significance | NUMERIC | The statistical significance of the `moran` measure. | + +#### Examples ```sql -table(numeric moran_val, text quadrant, numeric significance, int ids, numeric column_values) CDB_AreasOfInterest(text query, text column_name) - -table(numeric moran_val, text quadrant, numeric significance, int ids, numeric column_values) CDB_AreasOfInterest(text query, text column_name, int permutations, text geom_column, text id_column, text weight_type, int num_ngbrs) +SELECT * +FROM CDB_AreasOfInterestGlobalRate('SELECT * FROM commute_data', + 'num_cyclists', + 'total_population') ``` -## Description - -CDB_AreasOfInterest is a table-returning function that classifies the geometries in a table by an attribute and gives a significance for that classification. This information can be used to find "Areas of Interest" by using the correlation of a geometry's attribute with that of its neighbors. Areas can be clusters, outliers, or neither (depending on which significance value is used). - -Inputs: - -* `query` (required): an arbitrary query against tables you have access to (e.g., in your account, shared in your organization, or through the Data Observatory). This string must contain the following columns: an id `INT` (e.g., `cartodb_id`), geometry (e.g., `the_geom`), and the numeric attribute which is specified in `column_name` -* `column_name` (required): column to perform the area of interest analysis tool on. The data must be numeric (e.g., `float`, `int`, etc.) -* `permutations` (optional): used to calculate the significance of a classification. Defaults to 99, which is sufficient in most situations. -* `geom_column` (optional): the name of the geometry column. Data must be of type `geometry`. -* `id_column` (optional): the name of the id column (e.g., `cartodb_id`). Data must be of type `int` or `bigint` and have a unique condition on the data. -* `weight_type` (optional): the type of weight used for determining what defines a neighborhood. Options are `knn` or `queen`. -* `num_ngbrs` (optional): the number of neighbors in a neighborhood around a geometry. Only used if `knn` is chosen above. - -Outputs: - -* `moran_val`: underlying correlation statistic used in analysis -* `quadrant`: human-readable interpretation of classification -* `significance`: significance of classification (closer to 0 is more significant) -* `ids`: id of original geometry (used for joining against original table if desired -- see examples) -* `column_values`: original column values from `column_name` - -Availability: crankshaft v0.0.1 and above - -## Examples - -```sql -SELECT - t.the_geom_webmercator, - t.cartodb_id, - aoi.significance, - aoi.quadrant As aoi_quadrant -FROM - observatory.acs2013 As t -JOIN - crankshaft.CDB_AreasOfInterest('SELECT * FROM observatory.acs2013', - 'gini_index') -``` - -## API Usage - -Example - -```text -http://eschbacher.cartodb.com/api/v2/sql?q=SELECT * FROM crankshaft.CDB_AreasOfInterest('SELECT * FROM observatory.acs2013','gini_index') -``` - -Result -```json -{ - time: 0.120, - total_rows: 100, - rows: [{ - moran_vals: 0.7213, - quadrant: 'High area', - significance: 0.03, - ids: 1, - column_value: 0.22 - }, - { - moran_vals: -0.7213, - quadrant: 'Low outlier', - significance: 0.13, - ids: 2, - column_value: 0.03 - }, - ... - ] -} -``` - -## See Also - -crankshaft's areas of interest functions: - -* [CDB_AreasOfInterest_Global]() -* [CDB_AreasOfInterest_Rate_Local]() -* [CDB_AreasOfInterest_Rate_Global]() - ## Hotspot, Coldspot, and Outlier Functions + +These functions are convenience functions for extracting only information that you are interested in exposing based on the outputs of the `CDB_AreasOfInterest` functions. For instance, you can use `CDB_GetSpatialHotspots` to output only the classifications of `HH` and `HL`. + +### Non-rate functions + +#### CDB_GetSpatialHotspots +This function's inputs and outputs exactly mirror `CDB_AreasOfInterestLocal` except that the outputs are filtered to be only 'HH' and 'HL' (areas of high values). For more information about this function's use, see `CDB_AreasOfInterestLocal`. + +#### CDB_GetSpatialColdspots +This function's inputs and outputs exactly mirror `CDB_AreasOfInterestLocal` except that the outputs are filtered to be only 'LL' and 'LH' (areas of low values). For more information about this function's use, see `CDB_AreasOfInterestLocal`. + +#### CDB_GetSpatialOutliers +This function's inputs and outputs exactly mirror `CDB_AreasOfInterestLocal` except that the outputs are filtered to be only 'HL' and 'LH' (areas where highs or lows are surrounded by opposite values on average). For more information about this function's use, see `CDB_AreasOfInterestLocal`. + +### Rate functions + +#### CDB_GetSpatialHotspotsRate + +This function's inputs and outputs exactly mirror `CDB_AreasOfInterestLocalRate` except that the outputs are filtered to be only 'HH' and 'HL' (areas of high values). For more information about this function's use, see `CDB_AreasOfInterestLocalRate`. + +#### CDB_GetSpatialColdspotsRate + +This function's inputs and outputs exactly mirror `CDB_AreasOfInterestLocalRate` except that the outputs are filtered to be only 'LL' and 'LH' (areas of low values). For more information about this function's use, see `CDB_AreasOfInterestLocalRate`. + +#### CDB_GetSpatialOutliersRate + +This function's inputs and outputs exactly mirror `CDB_AreasOfInterestLocalRate` except that the outputs are filtered to be only 'HL' and 'LH' (areas where highs or lows are surrounded by opposite values on average). For more information about this function's use, see `CDB_AreasOfInterestLocalRate`. From c04e15ef810c8fadc151dffe277597670e5ce1f2 Mon Sep 17 00:00:00 2001 From: Andy Eschbacher Date: Wed, 1 Jun 2016 15:07:16 -0400 Subject: [PATCH 59/81] rename some variables --- src/pg/sql/10_moran.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index 9e7adf2..a336867 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -2,7 +2,7 @@ CREATE OR REPLACE FUNCTION CDB_AreasOfInterestGlobal( subquery TEXT, - attr_name TEXT, + column_name TEXT, w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, @@ -13,14 +13,14 @@ AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary - return moran(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- Moran's I Local (internal function) CREATE OR REPLACE FUNCTION _CDB_AreasOfInterestLocal( subquery TEXT, - attr TEXT, + column_name TEXT, w_type TEXT, num_ngbrs INT, permutations INT, @@ -31,14 +31,14 @@ AS $$ plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary - return moran_local(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) $$ LANGUAGE plpythonu; -- Moran's I Local (public-facing function) CREATE OR REPLACE FUNCTION CDB_AreasOfInterestLocal( subquery TEXT, - attr TEXT, + column_name TEXT, w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, @@ -48,7 +48,7 @@ RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals AS $$ SELECT moran, quads, significance, rowid, vals - FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col); + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); $$ LANGUAGE SQL; @@ -56,7 +56,7 @@ $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION CDB_GetSpatialHotspots( subquery TEXT, - attr TEXT, + column_name TEXT, w_type TEXT DEFAULT 'knn', num_ngbrs INT DEFAULT 5, permutations INT DEFAULT 99, @@ -66,7 +66,7 @@ CREATE OR REPLACE FUNCTION AS $$ SELECT moran, quads, significance, rowid, vals - FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) WHERE quads IN ('HH', 'HL'); $$ LANGUAGE SQL; From 4e86965f033f14252ded35534bbd2c911e104da7 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 7 Jun 2016 19:58:32 +0000 Subject: [PATCH 60/81] KMeans clustering and weighted centroid analysis --- doc/11_kmeans.md | 62 +++++++++++++++++++ src/pg/sql/11_kmeans.sql | 31 ++++++++++ src/pg/test/expected/05_kmeans_test.out | 10 +++ src/pg/test/sql/05_kmeans_test.sql | 6 ++ .../crankshaft/clustering/__init__.py | 1 + .../crankshaft/clustering/kmeans.py | 17 +++++ src/py/crankshaft/test/fixtures/kmeans.json | 1 + src/py/crankshaft/test/test_cluster_kmeans.py | 38 ++++++++++++ 8 files changed, 166 insertions(+) create mode 100644 doc/11_kmeans.md create mode 100644 src/pg/sql/11_kmeans.sql create mode 100644 src/pg/test/expected/05_kmeans_test.out create mode 100644 src/pg/test/sql/05_kmeans_test.sql create mode 100644 src/py/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 src/py/crankshaft/test/fixtures/kmeans.json create mode 100644 src/py/crankshaft/test/test_cluster_kmeans.py diff --git a/doc/11_kmeans.md b/doc/11_kmeans.md new file mode 100644 index 0000000..6153010 --- /dev/null +++ b/doc/11_kmeans.md @@ -0,0 +1,62 @@ +## K-Means Functions + +### CDB_KMeans(subquery text, no_clusters INTEGER) + +This function attempts to find n clusters within the input data. It will return a table to CartoDB ids and +the number of the cluster each point in the input was assigend to. + + +#### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column name `the_geom` and id column name `cartodb_id` unless otherwise specified in the input arguments | +| no\_clusters | INTEGER | The number of clusters to try and find | + +#### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| cartodb\_id | INTEGER | The CartoDB id of the row in the input table.| +| cluster\_no | INTEGER | The cluster that this point belongs to. | + + +#### Example Usage + +```sql +SELECT + customers.*, + km.cluster_no + FROM cdb_crankshaft.CDB_Kmeans('SELECT * from customers' , 6) km, customers_3 + WHERE customers.cartodb_id = km.cartodb_id +``` + +### CDB_WeightedMean(subquery text, weight_column text, category_column text) + +Function that computes the weighted centroid of a number of clusters by some weight column. + +### Arguments + +| Name | Type | Description | +|------|------|-------------| +| subquery | TEXT | SQL query that exposes the data to be analyzed (e.g., `SELECT * FROM interesting_table`). This query must have the geometry column and the columns specified as the weight and category columns| +| weight\_column | TEXT | The name of the column to use as a weight | +| category\_column | TEXT | The name of the column to use as a category | + +### Returns + +A table with the following columns. + +| Column Name | Type | Description | +|-------------|------|-------------| +| the\_geom | GEOMETRY | A point for the weighted cluster center | +| class | INTEGER | The cluster class | + +### Example Usage + +```sql +SELECT ST_TRANSFORM(the_geom, 3857) as the_geom_webmercator, class +FROM cdb_weighted_mean('SELECT *, customer_value FROM customers','customer_value','cluster_no') +``` diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql new file mode 100644 index 0000000..73e2f1d --- /dev/null +++ b/src/pg/sql/11_kmeans.sql @@ -0,0 +1,31 @@ +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + import plpy + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + +CREATE OR REPLACE FUNCTION CDB_WeightedMean(query text, weight_column text, category_column text default null ) +RETURNS table (the_geom geometry,class integer ) as $$ +BEGIN + +RETURN QUERY + EXECUTE format( $string$ + select ST_SETSRID(st_makepoint(cx, cy),4326) the_geom, class from ( + select + %I as class, + sum(st_x(the_geom)*%I)/sum(%I) cx, + sum(st_y(the_geom)*%I)/sum(%I) cy + from (%s) a + group by %I + ) q + + $string$, category_column, weight_column,weight_column,weight_column,weight_column,query, category_column + ) + using the_geom + RETURN; +END +$$ LANGUAGE plpgsql; diff --git a/src/pg/test/expected/05_kmeans_test.out b/src/pg/test/expected/05_kmeans_test.out new file mode 100644 index 0000000..4e6db09 --- /dev/null +++ b/src/pg/test/expected/05_kmeans_test.out @@ -0,0 +1,10 @@ +\pset format unaligned +\set ECHO all +SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); +clusters +2 +(1 row) +SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); +clusters +52 +(1 row) diff --git a/src/pg/test/sql/05_kmeans_test.sql b/src/pg/test/sql/05_kmeans_test.sql new file mode 100644 index 0000000..a400e5e --- /dev/null +++ b/src/pg/test/sql/05_kmeans_test.sql @@ -0,0 +1,6 @@ +\pset format unaligned +\set ECHO all + +SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); + +SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); diff --git a/src/py/crankshaft/crankshaft/clustering/__init__.py b/src/py/crankshaft/crankshaft/clustering/__init__.py index 0df080f..338e8ea 100644 --- a/src/py/crankshaft/crankshaft/clustering/__init__.py +++ b/src/py/crankshaft/crankshaft/clustering/__init__.py @@ -1 +1,2 @@ from moran import * +from kmeans import * diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..3d9ed58 --- /dev/null +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,17 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/src/py/crankshaft/test/fixtures/kmeans.json b/src/py/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/src/py/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/src/py/crankshaft/test/test_cluster_kmeans.py b/src/py/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/src/py/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + From e95c40c2f9bea57156449276d80603646cd4317d Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 9 Jun 2016 11:27:15 +0200 Subject: [PATCH 61/81] Ignore idea based configurations --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 1161ea2..a09b4fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ envs/ *.pyc .DS_Store +.idea/ From 7f3b23f67a958faa9162efef2362cc13a1e665ff Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 10 Jun 2016 13:06:49 +0000 Subject: [PATCH 62/81] reworking CDB_WeightedMean to be an aggregate function --- src/pg/sql/11_kmeans.sql | 60 +++++++++++++++++-------- src/pg/test/expected/05_kmeans_test.out | 2 +- src/pg/test/sql/05_kmeans_test.sql | 2 +- 3 files changed, 43 insertions(+), 21 deletions(-) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 73e2f1d..87f07ea 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -8,24 +8,46 @@ RETURNS table (cartodb_id integer, cluster_no integer) as $$ $$ language plpythonu; -CREATE OR REPLACE FUNCTION CDB_WeightedMean(query text, weight_column text, category_column text default null ) -RETURNS table (the_geom geometry,class integer ) as $$ -BEGIN -RETURN QUERY - EXECUTE format( $string$ - select ST_SETSRID(st_makepoint(cx, cy),4326) the_geom, class from ( - select - %I as class, - sum(st_x(the_geom)*%I)/sum(%I) cx, - sum(st_y(the_geom)*%I)/sum(%I) cy - from (%s) a - group by %I - ) q - - $string$, category_column, weight_column,weight_column,weight_column,weight_column,query, category_column - ) - using the_geom - RETURN; -END +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END $$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(the_geom geometry(Point, 4326), weight NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); + + diff --git a/src/pg/test/expected/05_kmeans_test.out b/src/pg/test/expected/05_kmeans_test.out index 4e6db09..8c6ffa1 100644 --- a/src/pg/test/expected/05_kmeans_test.out +++ b/src/pg/test/expected/05_kmeans_test.out @@ -4,7 +4,7 @@ SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('se clusters 2 (1 row) -SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); +SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p; clusters 52 (1 row) diff --git a/src/pg/test/sql/05_kmeans_test.sql b/src/pg/test/sql/05_kmeans_test.sql index a400e5e..2298b85 100644 --- a/src/pg/test/sql/05_kmeans_test.sql +++ b/src/pg/test/sql/05_kmeans_test.sql @@ -3,4 +3,4 @@ SELECT count(DISTINCT cluster_no) as clusters from cdb_crankshaft.cdb_kmeans('select * from ppoints', 2); -SELECT count(*) clusters from cdb_crankshaft.cdb_WeightedMean( 'select *, code::INTEGER as cluster from ppoints' , 'value', 'cluster' ); +SELECT count(*) clusters from (select cdb_crankshaft.CDB_WeightedMean(the_geom, value::NUMERIC), code from ppoints group by code) p; From 9d3de5a8ef13be63539248f3e4d82d7b4b68df9d Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 10 Jun 2016 13:12:55 +0000 Subject: [PATCH 63/81] adding not null filter for geom on kmeans --- src/py/crankshaft/crankshaft/clustering/kmeans.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/py/crankshaft/crankshaft/clustering/kmeans.py b/src/py/crankshaft/crankshaft/clustering/kmeans.py index 3d9ed58..4134062 100644 --- a/src/py/crankshaft/crankshaft/clustering/kmeans.py +++ b/src/py/crankshaft/crankshaft/clustering/kmeans.py @@ -5,6 +5,7 @@ def kmeans(query, no_clusters, no_init=20): data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, array_agg(ST_X(the_geom) order by cartodb_id) xs, array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null '''.format(query=query)) xs = data[0]['xs'] From 1a4944b9600250a972458bfe1952f79ffce76ff2 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Fri, 10 Jun 2016 13:16:16 +0000 Subject: [PATCH 64/81] adding sklearn as a dep --- src/py/crankshaft/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 8d5e622..baa88e3 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1'], + install_requires=['pysal==1.9.1', 'sklearn==0.17.1'], - requires=['pysal', 'numpy' ], + requires=['pysal', 'numpy', 'sklearn' ], test_suite='test' ) From 889cd5c5791d2f87e35b3e510b7c7ac14eac9fcf Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 10 Jun 2016 17:47:46 +0200 Subject: [PATCH 65/81] Fix scikit-learn dep name --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index baa88e3..68f9e17 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1', 'sklearn==0.17.1'], + install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], requires=['pysal', 'numpy', 'sklearn' ], From b33ba2d2949ab0bef092f25acf82d2308775a2a5 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 10 Jun 2016 18:24:43 +0200 Subject: [PATCH 66/81] Do not use names for the aggregate params --- src/pg/sql/11_kmeans.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 87f07ea..a27f803 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -43,11 +43,9 @@ BEGIN END $$ LANGUAGE plpgsql; -CREATE AGGREGATE CDB_WeightedMean(the_geom geometry(Point, 4326), weight NUMERIC)( +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( SFUNC = CDB_WeightedMeanS, FINALFUNC = CDB_WeightedMeanF, STYPE = Numeric[], INITCOND = "{0.0,0.0,0.0}" ); - - From 1e8bc12e0a6ea2ffefe580b63133b88f4db045a7 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jun 2016 12:17:46 +0200 Subject: [PATCH 67/81] Declare scipy as dep --- src/py/crankshaft/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 68f9e17..e787d32 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], - requires=['pysal', 'numpy', 'sklearn' ], + requires=['scipy', 'pysal', 'numpy', 'sklearn'], test_suite='test' ) From c870f68c77652a11f8401bbbb981797694174288 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jun 2016 13:05:50 +0200 Subject: [PATCH 68/81] Revert "Declare scipy as dep" This reverts commit 1e8bc12e0a6ea2ffefe580b63133b88f4db045a7. --- src/py/crankshaft/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index e787d32..68f9e17 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], - requires=['scipy', 'pysal', 'numpy', 'sklearn'], + requires=['pysal', 'numpy', 'sklearn' ], test_suite='test' ) From fd1862167c123ad7e59906801027e06c88fbf90e Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Mon, 13 Jun 2016 13:06:21 +0200 Subject: [PATCH 69/81] Remove trailing space --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 68f9e17..04822dd 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -42,7 +42,7 @@ setup( # provisioned in the production servers. install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], - requires=['pysal', 'numpy', 'sklearn' ], + requires=['pysal', 'numpy', 'sklearn'], test_suite='test' ) From 7b98415da318e5dd5119e7c10b5b0b2ca54f3c8d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:06:23 +0200 Subject: [PATCH 70/81] Remove virtualenv activation #60 --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- src/pg/sql/02_py.sql | 23 ----------------------- src/pg/sql/03_random_seeds.sql | 1 - src/pg/sql/10_moran.sql | 4 ---- 4 files changed, 1 insertion(+), 29 deletions(-) delete mode 100644 src/pg/sql/02_py.sql diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 882cece..9bb2e75 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ - [ ] All declared geometries are `geometry(Geometry, 4326)` for general geoms, or `geometry(Point, 4326)` -- [ ] Include python is activated for new functions. Include this before importing modules: `plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()')` +- [ ] Existing functions in crankshaft python library called from the extension are kept at least from version N to version N+1 (to avoid breakage during upgrades). - [ ] Docs for public-facing functions are written - [ ] New functions follow the naming conventions: `CDB_NameOfFunction`. Where internal functions begin with an underscore `_`. - [ ] If appropriate, new functions accepts an arbitrary query as an input (see [Crankshaft Issue #6](https://github.com/CartoDB/crankshaft/issues/6) for more information) diff --git a/src/pg/sql/02_py.sql b/src/pg/sql/02_py.sql deleted file mode 100644 index 7da5f47..0000000 --- a/src/pg/sql/02_py.sql +++ /dev/null @@ -1,23 +0,0 @@ -CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() -RETURNS text -AS $$ - BEGIN - -- RETURN '/opt/virtualenvs/crankshaft'; - RETURN '@@VIRTUALENV_PATH@@'; - END; -$$ language plpgsql IMMUTABLE STRICT; - --- Use the crankshaft python module -CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() -RETURNS VOID -AS $$ - import os - # plpy.notice('%',str(os.environ)) - # activate virtualenv - crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] - base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] - default_venv_path = os.path.join(base_path, crankshaft_version) - venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) - activate_path = venv_path + '/bin/activate_this.py' - exec(open(activate_path).read(), dict(__file__=activate_path)) -$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/03_random_seeds.sql b/src/pg/sql/03_random_seeds.sql index 9a0cca6..2b62be3 100644 --- a/src/pg/sql/03_random_seeds.sql +++ b/src/pg/sql/03_random_seeds.sql @@ -4,7 +4,6 @@ CREATE OR REPLACE FUNCTION _cdb_random_seeds (seed_value INTEGER) RETURNS VOID AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft import random_seeds random_seeds.set_random_seeds(seed_value) $$ LANGUAGE plpythonu; diff --git a/src/pg/sql/10_moran.sql b/src/pg/sql/10_moran.sql index a336867..3be31a2 100644 --- a/src/pg/sql/10_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -10,7 +10,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran NUMERIC, significance NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) @@ -28,7 +27,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT) RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) @@ -122,7 +120,6 @@ CREATE OR REPLACE FUNCTION id_col TEXT DEFAULT 'cartodb_id') RETURNS TABLE (moran FLOAT, significance FLOAT) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) @@ -143,7 +140,6 @@ CREATE OR REPLACE FUNCTION RETURNS TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) AS $$ - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate # TODO: use named parameters or a dictionary return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) From 0acae8240f777e042f59dfcf3f0a3e1430dcb984 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:23:30 +0200 Subject: [PATCH 71/81] Remove virtualenv stuff from Makefiles #60 --- Makefile | 6 ------ src/pg/Makefile | 7 +------ src/py/Makefile | 11 +++-------- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 6c3e219..ef9415b 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,6 @@ PYP_DIR = src/py # Generate and install developmet versions of the extension # and python package. # The extension is named 'dev' with a 'current' alias for easily upgrading. -# The Python package is installed in a virtual environment envs/dev/ # Requires sudo. install: ## Generate and install development version of the extension; requires sudo. $(MAKE) -C $(PYP_DIR) install @@ -29,7 +28,6 @@ release: ## Generate a new release of the extension. Only for telease manager $(MAKE) -C $(PYP_DIR) release # Install the current release. -# The Python package is installed in a virtual environment envs/X.Y.Z/ # Requires sudo. # Use the RELEASE_VERSION environment variable to deploy a specific version: # sudo make deploy RELEASE_VERSION=1.0.0 @@ -52,10 +50,6 @@ clean-release: ## clean up current release rm -rf release/python/$(RELEASE_VERSION) rm -f release/$(RELEASE_VERSION)--*.sql -# Cleanup all virtual environments -clean-environments: ## clean up all virtual environments - rm -rf envs/* - clean-all: clean-dev clean-release clean-environments help: diff --git a/src/pg/Makefile b/src/pg/Makefile index 8a745c4..178ed08 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -7,7 +7,6 @@ include ../../Makefile.global # requires sudo. In additionof the current development version # named 'dev', an alias 'current' is generating for ease of # update (upgrade to 'current', then to 'dev'). -# the python module is installed in a virtualenv in envs/dev/ # * test runs the tests for the currently generated Development # extension. @@ -18,11 +17,8 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) -VIRTUALENV_PATH = $(realpath ../../envs) -ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) -REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ - -e 's/@@VIRTUALENV_PATH@@/$(ESC_VIRVIRTUALENV_PATH)/g' +REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' $(DATA): $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@ @@ -54,7 +50,6 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql # Install the current relese into the PostgreSQL extensions directory -# and the Python package in a virtual environment envs/X.Y.Z deploy: $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' diff --git a/src/py/Makefile b/src/py/Makefile index 90b22b8..403c5a1 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -2,14 +2,11 @@ include ../../Makefile.global # Install the package locally for development install: - virtualenv --system-site-packages ../../envs/dev - # source ../../envs/dev/bin/activate - ../../envs/dev/bin/pip install -I ./crankshaft - ../../envs/dev/bin/pip install -I nose + pip install ./crankshaft # Test develpment install test: - ../../envs/dev/bin/nosetests crankshaft/test/ + nosetests crankshaft/test/ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) mkdir -p ../../release/python/$(EXTVERSION) @@ -17,6 +14,4 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py deploy: - virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose + pip install --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) From 75531b671e247b507d0a11d6f2fdced5ef3a8084 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:24:43 +0200 Subject: [PATCH 72/81] Remove virtualenv references from READMEs #60 --- README.md | 3 +-- src/py/README.md | 17 +---------------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 68a64fb..0ff9090 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,10 @@ CartoDB Spatial Analysis extension for PostgreSQL. * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code * *release* reseleased versions -* *env* base directory for Python virtual environments ## Requirements -* pip, virtualenv, PostgreSQL +* pip, PostgreSQL * python-scipy system package (see [src/py/README.md](https://github.com/CartoDB/crankshaft/blob/master/src/py/README.md)) # Working Process -- Quickstart Guide diff --git a/src/py/README.md b/src/py/README.md index 29a3145..8fcfcb7 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -10,7 +10,6 @@ nosetests test/ ## Notes about Python dependencies * This extension is targeted at production databases. Therefore certain restrictions must be assumed about the production environment vs other experimental environments. -* We're using `pip` and `virtualenv` to generate a suitable isolated environment for python code that has all the dependencies * Every dependency should be: - Added to the `setup.py` file - Installed through it @@ -30,21 +29,7 @@ PySAL 1.10 or later, so we'll stick to 1.9.1. apt-get install -y python-scipy ``` -We'll use virtual environments to install our packages, -but configued to use also system modules so that the -mentioned scipy and numpy are used. - - # Create a virtual environment for python - $ virtualenv --system-site-packages dev - - # Activate the virtualenv - $ source dev/bin/activate - - # Install all the requirements - # expect this to take a while, as it will trigger a few compilations - (dev) $ pip install -I ./crankshaft - -#### Test the libraries with that virtual env +#### Test the libraries ##### Test numpy library dependency: From a8943bae985acc4d960d7cb614c5e6ad4bb68ed1 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Tue, 14 Jun 2016 18:27:35 +0200 Subject: [PATCH 73/81] Remove reference to clean-environments #60 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ef9415b..50f690c 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ clean-release: ## clean up current release rm -rf release/python/$(RELEASE_VERSION) rm -f release/$(RELEASE_VERSION)--*.sql -clean-all: clean-dev clean-release clean-environments +clean-all: clean-dev clean-release help: @IFS=$$'\n' ; \ From d08a2b6d2d756be58a16e80bf4ded3d134dfb97a Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 14:12:28 +0200 Subject: [PATCH 74/81] Remove _cdb_crankshaft_activate_py activation call from kmeans function --- src/pg/sql/11_kmeans.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index a27f803..125aac3 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -1,8 +1,6 @@ CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) RETURNS table (cartodb_id integer, cluster_no integer) as $$ - import plpy - plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import kmeans return kmeans(query,no_clusters,no_init) From 8b5e9102345fc2a7218961ef26c033715a441d6b Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 14:16:32 +0200 Subject: [PATCH 75/81] Release 0.0.3 --- NEWS.md | 5 + release/crankshaft--0.0.2--0.0.3.sql | 413 ++++++++++++++++++ release/crankshaft--0.0.3--0.0.2.sql | 209 +++++++++ release/crankshaft--0.0.3.sql | 403 +++++++++++++++++ release/crankshaft.control | 2 +- .../0.0.3/crankshaft/crankshaft/__init__.py | 2 + .../crankshaft/clustering/__init__.py | 2 + .../crankshaft/clustering/kmeans.py | 18 + .../crankshaft/crankshaft/clustering/moran.py | 260 +++++++++++ .../crankshaft/pysal_utils/__init__.py | 1 + .../crankshaft/pysal_utils/pysal_utils.py | 152 +++++++ .../crankshaft/crankshaft/random_seeds.py | 10 + release/python/0.0.3/crankshaft/setup.py | 48 ++ .../crankshaft/test/fixtures/kmeans.json | 1 + .../0.0.3/crankshaft/test/fixtures/moran.json | 52 +++ .../crankshaft/test/fixtures/neighbors.json | 54 +++ .../python/0.0.3/crankshaft/test/helper.py | 13 + .../python/0.0.3/crankshaft/test/mock_plpy.py | 34 ++ .../crankshaft/test/test_cluster_kmeans.py | 38 ++ .../crankshaft/test/test_clustering_moran.py | 83 ++++ .../0.0.3/crankshaft/test/test_pysal_utils.py | 107 +++++ src/pg/crankshaft.control | 2 +- 22 files changed, 1907 insertions(+), 2 deletions(-) create mode 100644 release/crankshaft--0.0.2--0.0.3.sql create mode 100644 release/crankshaft--0.0.3--0.0.2.sql create mode 100644 release/crankshaft--0.0.3.sql create mode 100644 release/python/0.0.3/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py create mode 100644 release/python/0.0.3/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.0.3/crankshaft/setup.py create mode 100644 release/python/0.0.3/crankshaft/test/fixtures/kmeans.json create mode 100644 release/python/0.0.3/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.0.3/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.0.3/crankshaft/test/helper.py create mode 100644 release/python/0.0.3/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py create mode 100644 release/python/0.0.3/crankshaft/test/test_clustering_moran.py create mode 100644 release/python/0.0.3/crankshaft/test/test_pysal_utils.py diff --git a/NEWS.md b/NEWS.md index 0b8c2da..ed66fd9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.0.3 (2016-06-16) +------------------ +* Adds new functions: kmeans, weighted centroids. +* Replaces moran functions with new areas of interest naming. + 0.0.2 (2016-03-16) ------------------ * New versioning approach using per-version Python virtual environments diff --git a/release/crankshaft--0.0.2--0.0.3.sql b/release/crankshaft--0.0.2--0.0.3.sql new file mode 100644 index 0000000..8a865d5 --- /dev/null +++ b/release/crankshaft--0.0.2--0.0.3.sql @@ -0,0 +1,413 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit + +-- [MANUALLY] DROP FUNCTIONS REMOVED SINCE 0.0.2 version + +DROP FUNCTION IF EXISTS cdb_moran_local(TEXT, TEXT, float, INT, INT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS cdb_moran_local_rate(TEXT, TEXT, TEXT, FLOAT, INT, INT, TEXT, TEXT, TEXT); +DROP FUNCTION IF EXISTS _cdb_crankshaft_virtualenvs_path(); +DROP FUNCTION IF EXISTS _cdb_crankshaft_activate_py(); + +-- [END MANUALLY] DROP FUNCTIONS REMOVED SINCE 0.0.2 version + +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() + RETURNS text AS $$ + SELECT '0.0.3'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() + RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION + _cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS + TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) + RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) + RETURNS Numeric[] AS + $$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) + RETURNS GEOMETRY AS + $$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( +SFUNC = CDB_WeightedMeanS, +FINALFUNC = CDB_WeightedMeanF, +STYPE = Numeric[], +INITCOND = "{0.0,0.0,0.0}" +); +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS + $$ + DECLARE + result numeric; + qualified_name text; + BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; + END; + $$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) + RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft--0.0.3--0.0.2.sql b/release/crankshaft--0.0.3--0.0.2.sql new file mode 100644 index 0000000..a2ccd2f --- /dev/null +++ b/release/crankshaft--0.0.3--0.0.2.sql @@ -0,0 +1,209 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit + +-- [MANUALLY] DROP FUNCTIONS INTRODUCED IN 0.0.3 version + +DROP FUNCTION IF EXISTS CDB_AreasOfInterestGlobal(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS _CDB_AreasOfInterestLocal(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_AreasOfInterestLocal(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialHotspots(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialColdspots(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialOutliers(TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_AreasOfInterestGlobalRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_AreasOfInterestLocalRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS _CDB_AreasOfInterestLocalRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialHotspotsRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialColdspotsRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_GetSpatialOutliersRate(TEXT,TEXT,TEXT,TEXT,INT,INT,TEXT,TEXT); +DROP FUNCTION IF EXISTS CDB_KMeans(text,integer,integer); +DROP AGGREGATE IF EXISTS CDB_WeightedMean(geometry(Point, 4326), NUMERIC); +DROP FUNCTION IF EXISTS CDB_WeightedMeanS(Numeric[], GEOMETRY(Point, 4326), NUMERIC); +DROP FUNCTION IF EXISTS CDB_WeightedMeanF(Numeric[]); + + +-- [END MANUALLY] DROP FUNCTIONS INTRODUCED IN 0.0.3 version + +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.2'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() +RETURNS text +AS $$ + BEGIN + -- RETURN '/opt/virtualenvs/crankshaft'; + RETURN '/home/ubuntu/crankshaft/envs'; + END; +$$ language plpgsql IMMUTABLE STRICT; + +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + import os + # plpy.notice('%',str(os.environ)) + # activate virtualenv + crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] + base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] + default_venv_path = os.path.join(base_path, crankshaft_version) + venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), dict(__file__=activate_path)) +$$ LANGUAGE plpythonu; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I +CREATE OR REPLACE FUNCTION + cdb_moran_local ( + t TEXT, + attr TEXT, + significance float DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate +CREATE OR REPLACE FUNCTION + cdb_moran_local_rate(t TEXT, + numerator TEXT, + denominator TEXT, + significance FLOAT DEFAULT 0.05, + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_column TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id', + w_type TEXT DEFAULT 'knn') +RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) +AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) +$$ LANGUAGE plpythonu; +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft--0.0.3.sql b/release/crankshaft--0.0.3.sql new file mode 100644 index 0000000..caacd75 --- /dev/null +++ b/release/crankshaft--0.0.3.sql @@ -0,0 +1,403 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.3'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 49c0d22..2029b7e 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.2' +default_version = '0.0.3' requires = 'plpythonu, postgis, cartodb' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.0.3/crankshaft/crankshaft/__init__.py b/release/python/0.0.3/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..d07e330 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/__init__.py @@ -0,0 +1,2 @@ +import random_seeds +import clustering diff --git a/release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..338e8ea --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1,2 @@ +from moran import * +from kmeans import * diff --git a/release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py b/release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..4134062 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,18 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py b/release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..39b3ff6 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,260 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def moran(subquery, attr_name, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I (global) + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": attr_name, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + attr_vals = pu.get_attributes(result) + + ## calculate weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) + + return zip([moran_global.I], [moran_global.EI]) + +def moran_local(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "attr1": attr, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(5) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": numerator, + "attr2": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global rate + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) + + return zip([lisa_rate.I], [lisa_rate.EI]) + +def moran_local_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Local Rate + Andy Eschbacher + """ + # geometries with values that are null are ignored + # resulting in a collection of not as near neighbors + + query = pu.construct_neighbor_query(w_type, + {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs}) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(5) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) + + # find units of significance + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ + plpy.notice('** Constructing query') + + qvals = {"num_ngbrs": num_ngbrs, + "attr1": attr1, + "attr2": attr2, + "subquery": subquery, + "geom_col": geom_col, + "id_col": id_col} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + except plpy.SPIError: + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(4) + + ## collect attributes + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) + + # create weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def quad_position(quads): + """ + Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param list: an array of quads classied by 'HH', 'LL', etc. + """ + return [map_quads(q) for q in quads] diff --git a/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..835880d --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1 @@ +from pysal_utils import * diff --git a/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..02b5e35 --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,152 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type.lower() == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res: query results with attributes and neighbors + """ + if w_type.lower() == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + else: + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs})" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/release/python/0.0.3/crankshaft/crankshaft/random_seeds.py b/release/python/0.0.3/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..b7c8eed --- /dev/null +++ b/release/python/0.0.3/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,10 @@ +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.0.3/crankshaft/setup.py b/release/python/0.0.3/crankshaft/setup.py new file mode 100644 index 0000000..33a3b62 --- /dev/null +++ b/release/python/0.0.3/crankshaft/setup.py @@ -0,0 +1,48 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.0.3', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], + + requires=['pysal', 'numpy', 'sklearn'], + + test_suite='test' +) diff --git a/release/python/0.0.3/crankshaft/test/fixtures/kmeans.json b/release/python/0.0.3/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/release/python/0.0.3/crankshaft/test/fixtures/moran.json b/release/python/0.0.3/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..2f75cf1 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], +[0.18562939195219, "LL"], +[0.3010757406693439, "LL"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], +[0.18562939195219, "LL"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/release/python/0.0.3/crankshaft/test/fixtures/neighbors.json b/release/python/0.0.3/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.0.3/crankshaft/test/helper.py b/release/python/0.0.3/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.0.3/crankshaft/test/mock_plpy.py b/release/python/0.0.3/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..63c88f6 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/mock_plpy.py @@ -0,0 +1,34 @@ +import re + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py b/release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + diff --git a/release/python/0.0.3/crankshaft/test/test_clustering_moran.py b/release/python/0.0.3/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..393e93b --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,83 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_quad_position(self): + """Test lisa_sig_vals""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) diff --git a/release/python/0.0.3/crankshaft/test/test_pysal_utils.py b/release/python/0.0.3/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..4ea0d9b --- /dev/null +++ b/release/python/0.0.3/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,107 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(pu.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 49c0d22..2029b7e 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.2' +default_version = '0.0.3' requires = 'plpythonu, postgis, cartodb' superuser = true schema = cdb_crankshaft From 1e19f468ebfc0626c37b85fd88ad0d1da1531771 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 16:23:43 +0200 Subject: [PATCH 76/81] Declare numpy dep --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 04822dd..f072f17 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['numpy==1.11.0', 'pysal==1.9.1', 'scikit-learn==0.17.1'], requires=['pysal', 'numpy', 'sklearn'], From 237aa1c5818f003ffb459817ea5e72392c765c5c Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 16:34:45 +0200 Subject: [PATCH 77/81] Declare scipy as dep --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index f072f17..266b6f1 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['numpy==1.11.0', 'pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['numpy==1.11.0', 'scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], requires=['pysal', 'numpy', 'sklearn'], From 3480a0d252b1b7f9e79397b126b08f65837d3036 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Thu, 16 Jun 2016 16:56:16 +0200 Subject: [PATCH 78/81] Allow passing options to pip install --- src/py/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py/Makefile b/src/py/Makefile index 403c5a1..b584645 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -13,5 +13,5 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) cp -r ./$(PACKAGE) ../../release/python/$(EXTVERSION)/ $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py -deploy: - pip install --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) +deploy: + pip install $(RUN_OPTIONS) --upgrade ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) From 1db938c450634532133190b3c28425e7313acc72 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 16 Jun 2016 19:07:42 +0200 Subject: [PATCH 79/81] Removes cartodb-extension-dep --- CONTRIBUTING.md | 1 - src/pg/crankshaft.control | 2 +- src/pg/test/expected/01_install_test.out | 1 - src/pg/test/sql/01_install_test.sql | 1 - src/pg/test/sql/90_permissions.sql | 2 +- 5 files changed, 2 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f642d45..42385dc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,7 +60,6 @@ it can be installed directly with: * `CREATE EXTENSION IF NOT EXISTS plpythonu;` `CREATE EXTENSION IF NOT EXISTS postgis;` - `CREATE EXTENSION IF NOT EXISTS cartodb;` `CREATE EXTENSION crankshaft WITH VERSION 'dev';` Note: the development extension uses the development python virtual diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index 2029b7e..e71321f 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' default_version = '0.0.3' -requires = 'plpythonu, postgis, cartodb' +requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index e40d267..e84a48a 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,6 +1,5 @@ -- Install dependencies CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; -CREATE EXTENSION cartodb; -- Install the extension CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index fc3ea80..bbce805 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -1,7 +1,6 @@ -- Install dependencies CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; -CREATE EXTENSION cartodb; -- Install the extension CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/src/pg/test/sql/90_permissions.sql b/src/pg/test/sql/90_permissions.sql index 187f795..1e9ea99 100644 --- a/src/pg/test/sql/90_permissions.sql +++ b/src/pg/test/sql/90_permissions.sql @@ -4,7 +4,7 @@ SELECT cdb_crankshaft._cdb_random_seeds(1234); SET ROLE test_regular_user; -- Add to the search path the schema -SET search_path TO public,cartodb,cdb_crankshaft; +SET search_path TO public,cdb_crankshaft; -- Exercise public functions SELECT ppoints.code, m.quads From f5fb4499db226521adb952b7524449ae15ddcc3a Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 20 Jun 2016 09:44:52 +0200 Subject: [PATCH 80/81] Set final dependencies versions --- src/py/crankshaft/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index 266b6f1..abd4dae 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,7 +40,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['numpy==1.11.0', 'scipy==0.17.1', 'pysal==1.9.1', 'scikit-learn==0.17.1'], + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], requires=['pysal', 'numpy', 'sklearn'], From 01fc2c1dd1087e58679cc2728ee88a0df5702ea6 Mon Sep 17 00:00:00 2001 From: Luis Bosque Date: Mon, 20 Jun 2016 10:04:22 +0200 Subject: [PATCH 81/81] Release 0.0.4 --- NEWS.md | 5 + release/crankshaft--0.0.3--0.0.4.sql | 8 + release/crankshaft--0.0.4--0.0.3.sql | 8 + release/crankshaft--0.0.4.sql | 403 ++++++++++++++++++ release/crankshaft.control | 4 +- .../0.0.4/crankshaft/crankshaft/__init__.py | 2 + .../crankshaft/clustering/__init__.py | 2 + .../crankshaft/clustering/kmeans.py | 18 + .../crankshaft/crankshaft/clustering/moran.py | 260 +++++++++++ .../crankshaft/pysal_utils/__init__.py | 1 + .../crankshaft/pysal_utils/pysal_utils.py | 152 +++++++ .../crankshaft/crankshaft/random_seeds.py | 10 + release/python/0.0.4/crankshaft/setup.py | 48 +++ .../crankshaft/test/fixtures/kmeans.json | 1 + .../0.0.4/crankshaft/test/fixtures/moran.json | 52 +++ .../crankshaft/test/fixtures/neighbors.json | 54 +++ .../python/0.0.4/crankshaft/test/helper.py | 13 + .../python/0.0.4/crankshaft/test/mock_plpy.py | 34 ++ .../crankshaft/test/test_cluster_kmeans.py | 38 ++ .../crankshaft/test/test_clustering_moran.py | 83 ++++ .../0.0.4/crankshaft/test/test_pysal_utils.py | 107 +++++ src/pg/crankshaft.control | 2 +- 22 files changed, 1302 insertions(+), 3 deletions(-) create mode 100644 release/crankshaft--0.0.3--0.0.4.sql create mode 100644 release/crankshaft--0.0.4--0.0.3.sql create mode 100644 release/crankshaft--0.0.4.sql create mode 100644 release/python/0.0.4/crankshaft/crankshaft/__init__.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py create mode 100644 release/python/0.0.4/crankshaft/crankshaft/random_seeds.py create mode 100644 release/python/0.0.4/crankshaft/setup.py create mode 100644 release/python/0.0.4/crankshaft/test/fixtures/kmeans.json create mode 100644 release/python/0.0.4/crankshaft/test/fixtures/moran.json create mode 100644 release/python/0.0.4/crankshaft/test/fixtures/neighbors.json create mode 100644 release/python/0.0.4/crankshaft/test/helper.py create mode 100644 release/python/0.0.4/crankshaft/test/mock_plpy.py create mode 100644 release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py create mode 100644 release/python/0.0.4/crankshaft/test/test_clustering_moran.py create mode 100644 release/python/0.0.4/crankshaft/test/test_pysal_utils.py diff --git a/NEWS.md b/NEWS.md index ed66fd9..c011a0d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.0.4 (2016-06-20) +------------------ +* Remove cartodb extension dependency from tests +* Declare all correct dependencies with correct versions in setup.py + 0.0.3 (2016-06-16) ------------------ * Adds new functions: kmeans, weighted centroids. diff --git a/release/crankshaft--0.0.3--0.0.4.sql b/release/crankshaft--0.0.3--0.0.4.sql new file mode 100644 index 0000000..69038a3 --- /dev/null +++ b/release/crankshaft--0.0.3--0.0.4.sql @@ -0,0 +1,8 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.4'::text; +$$ language 'sql' STABLE STRICT; diff --git a/release/crankshaft--0.0.4--0.0.3.sql b/release/crankshaft--0.0.4--0.0.3.sql new file mode 100644 index 0000000..bd8ed82 --- /dev/null +++ b/release/crankshaft--0.0.4--0.0.3.sql @@ -0,0 +1,8 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.3'::text; +$$ language 'sql' STABLE STRICT; diff --git a/release/crankshaft--0.0.4.sql b/release/crankshaft--0.0.4.sql new file mode 100644 index 0000000..c855958 --- /dev/null +++ b/release/crankshaft--0.0.4.sql @@ -0,0 +1,403 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '0.0.4'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; +-- Internal function. +-- Set the seeds of the RNGs (Random Number Generators) +-- used internally. +CREATE OR REPLACE FUNCTION +_cdb_random_seeds (seed_value INTEGER) RETURNS VOID +AS $$ + from crankshaft import random_seeds + random_seeds.set_random_seeds(seed_value) +$$ LANGUAGE plpythonu; +-- Moran's I Global Measure (public-facing) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, significance NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_local(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocal( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspots( + subquery TEXT, + column_name TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, column_name, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspots( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliers( + subquery TEXT, + attr TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') + RETURNS TABLE (moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocal(subquery, attr, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Global Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestGlobalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS TABLE (moran FLOAT, significance FLOAT) +AS $$ + from crankshaft.clustering import moran_local + # TODO: use named parameters or a dictionary + return moran_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + + +-- Moran's I Local Rate (internal function) +CREATE OR REPLACE FUNCTION + _CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT, + num_ngbrs INT, + permutations INT, + geom_col TEXT, + id_col TEXT) +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + from crankshaft.clustering import moran_local_rate + # TODO: use named parameters or a dictionary + return moran_local_rate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) +$$ LANGUAGE plpythonu; + +-- Moran's I Local Rate (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_AreasOfInterestLocalRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for HH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialHotspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HH', 'HL'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LL and LH (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialColdspotsRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('LL', 'LH'); + +$$ LANGUAGE SQL; + +-- Moran's I Local Rate only for LH and HL (public-facing function) +CREATE OR REPLACE FUNCTION + CDB_GetSpatialOutliersRate( + subquery TEXT, + numerator TEXT, + denominator TEXT, + w_type TEXT DEFAULT 'knn', + num_ngbrs INT DEFAULT 5, + permutations INT DEFAULT 99, + geom_col TEXT DEFAULT 'the_geom', + id_col TEXT DEFAULT 'cartodb_id') +RETURNS +TABLE(moran NUMERIC, quads TEXT, significance NUMERIC, rowid INT, vals NUMERIC) +AS $$ + + SELECT moran, quads, significance, rowid, vals + FROM cdb_crankshaft._CDB_AreasOfInterestLocalRate(subquery, numerator, denominator, w_type, num_ngbrs, permutations, geom_col, id_col) + WHERE quads IN ('HL', 'LH'); + +$$ LANGUAGE SQL; +CREATE OR REPLACE FUNCTION CDB_KMeans(query text, no_clusters integer,no_init integer default 20) +RETURNS table (cartodb_id integer, cluster_no integer) as $$ + + from crankshaft.clustering import kmeans + return kmeans(query,no_clusters,no_init) + +$$ language plpythonu; + + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanS(state Numeric[],the_geom GEOMETRY(Point, 4326), weight NUMERIC) +RETURNS Numeric[] AS +$$ +DECLARE + newX NUMERIC; + newY NUMERIC; + newW NUMERIC; +BEGIN + IF weight IS NULL OR the_geom IS NULL THEN + newX = state[1]; + newY = state[2]; + newW = state[3]; + ELSE + newX = state[1] + ST_X(the_geom)*weight; + newY = state[2] + ST_Y(the_geom)*weight; + newW = state[3] + weight; + END IF; + RETURN Array[newX,newY,newW]; + +END +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION CDB_WeightedMeanF(state Numeric[]) +RETURNS GEOMETRY AS +$$ +BEGIN + IF state[3] = 0 THEN + RETURN ST_SetSRID(ST_MakePoint(state[1],state[2]), 4326); + ELSE + RETURN ST_SETSRID(ST_MakePoint(state[1]/state[3], state[2]/state[3]),4326); + END IF; +END +$$ LANGUAGE plpgsql; + +CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" +); +-- Function by Stuart Lynn for a simple interpolation of a value +-- from a polygon table over an arbitrary polygon +-- (weighted by the area proportion overlapped) +-- Aereal weighting is a very simple form of aereal interpolation. +-- +-- Parameters: +-- * geom a Polygon geometry which defines the area where a value will be +-- estimated as the area-weighted sum of a given table/column +-- * target_table_name table name of the table that provides the values +-- * target_column column name of the column that provides the values +-- * schema_name optional parameter to defina the schema the target table +-- belongs to, which is necessary if its not in the search_path. +-- Note that target_table_name should never include the schema in it. +-- Return value: +-- Aereal-weighted interpolation of the column values over the geometry +CREATE OR REPLACE +FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) + RETURNS numeric AS +$$ +DECLARE + result numeric; + qualified_name text; +BEGIN + IF schema_name IS NULL THEN + qualified_name := Format('%I', target_table_name); + ELSE + qualified_name := Format('%I.%s', schema_name, target_table_name); + END IF; + EXECUTE Format(' + SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) + FROM %s AS a + WHERE $1 && a.the_geom + ', target_column, qualified_name) + USING geom + INTO result; + RETURN result; +END; +$$ LANGUAGE plpgsql; +-- +-- Creates N points randomly distributed arround the polygon +-- +-- @param g - the geometry to be turned in to points +-- +-- @param no_points - the number of points to generate +-- +-- @params max_iter_per_point - the function generates points in the polygon's bounding box +-- and discards points which don't lie in the polygon. max_iter_per_point specifies how many +-- misses per point the funciton accepts before giving up. +-- +-- Returns: Multipoint with the requested points +CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) +RETURNS GEOMETRY AS $$ +DECLARE + extent GEOMETRY; + test_point Geometry; + width NUMERIC; + height NUMERIC; + x0 NUMERIC; + y0 NUMERIC; + xp NUMERIC; + yp NUMERIC; + no_left INTEGER; + remaining_iterations INTEGER; + points GEOMETRY[]; + bbox_line GEOMETRY; + intersection_line GEOMETRY; +BEGIN + extent := ST_Envelope(geom); + width := ST_XMax(extent) - ST_XMIN(extent); + height := ST_YMax(extent) - ST_YMIN(extent); + x0 := ST_XMin(extent); + y0 := ST_YMin(extent); + no_left := no_points; + + LOOP + if(no_left=0) THEN + EXIT; + END IF; + yp = y0 + height*random(); + bbox_line = ST_MakeLine( + ST_SetSRID(ST_MakePoint(yp, x0),4326), + ST_SetSRID(ST_MakePoint(yp, x0+width),4326) + ); + intersection_line = ST_Intersection(bbox_line,geom); + test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); + points := points || test_point; + no_left = no_left - 1 ; + END LOOP; + RETURN ST_Collect(points); +END; +$$ +LANGUAGE plpgsql VOLATILE; +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/release/crankshaft.control b/release/crankshaft.control index 2029b7e..01088b1 100644 --- a/release/crankshaft.control +++ b/release/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.3' -requires = 'plpythonu, postgis, cartodb' +default_version = '0.0.4' +requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/release/python/0.0.4/crankshaft/crankshaft/__init__.py b/release/python/0.0.4/crankshaft/crankshaft/__init__.py new file mode 100644 index 0000000..d07e330 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/__init__.py @@ -0,0 +1,2 @@ +import random_seeds +import clustering diff --git a/release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py b/release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py new file mode 100644 index 0000000..338e8ea --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/clustering/__init__.py @@ -0,0 +1,2 @@ +from moran import * +from kmeans import * diff --git a/release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py b/release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py new file mode 100644 index 0000000..4134062 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/clustering/kmeans.py @@ -0,0 +1,18 @@ +from sklearn.cluster import KMeans +import plpy + +def kmeans(query, no_clusters, no_init=20): + data = plpy.execute('''select array_agg(cartodb_id order by cartodb_id) as ids, + array_agg(ST_X(the_geom) order by cartodb_id) xs, + array_agg(ST_Y(the_geom) order by cartodb_id) ys from ({query}) a + where the_geom is not null + '''.format(query=query)) + + xs = data[0]['xs'] + ys = data[0]['ys'] + ids = data[0]['ids'] + + km = KMeans(n_clusters= no_clusters, n_init=no_init) + labels = km.fit_predict(zip(xs,ys)) + return zip(ids,labels) + diff --git a/release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py b/release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py new file mode 100644 index 0000000..39b3ff6 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/clustering/moran.py @@ -0,0 +1,260 @@ +""" +Moran's I geostatistics (global clustering & outliers presence) +""" + +# TODO: Fill in local neighbors which have null/NoneType values with the +# average of the their neighborhood + +import pysal as ps +import plpy + +# crankshaft module +import crankshaft.pysal_utils as pu + +# High level interface --------------------------------------- + +def moran(subquery, attr_name, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I (global) + Implementation building neighbors with a PostGIS database and Moran's I + core clusters with PySAL. + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": attr_name, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + attr_vals = pu.get_attributes(result) + + ## calculate weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global + moran_global = ps.esda.moran.Moran(attr_vals, weight, + permutations=permutations) + + return zip([moran_global.I], [moran_global.EI]) + +def moran_local(subquery, attr, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I implementation for PL/Python + Andy Eschbacher + """ + + # geometries with attributes that are null are ignored + # resulting in a collection of not as near neighbors + + qvals = {"id_col": id_col, + "attr1": attr, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(5) + + attr_vals = pu.get_attributes(result) + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local(attr_vals, weight, + permutations=permutations) + + # find quadrants for each geometry + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Rate (global) + Andy Eschbacher + """ + qvals = {"id_col": id_col, + "attr1": numerator, + "attr2": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs} + + query = pu.construct_neighbor_query(w_type, qvals) + + plpy.notice('** Query: %s' % query) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(2) + plpy.notice('** Query returned with %d rows' % len(result)) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(2) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + ## calculate moran global rate + lisa_rate = ps.esda.moran.Moran_Rate(numer, denom, weight, + permutations=permutations) + + return zip([lisa_rate.I], [lisa_rate.EI]) + +def moran_local_rate(subquery, numerator, denominator, + w_type, num_ngbrs, permutations, geom_col, id_col): + """ + Moran's I Local Rate + Andy Eschbacher + """ + # geometries with values that are null are ignored + # resulting in a collection of not as near neighbors + + query = pu.construct_neighbor_query(w_type, + {"id_col": id_col, + "numerator": numerator, + "denominator": denominator, + "geom_col": geom_col, + "subquery": subquery, + "num_ngbrs": num_ngbrs}) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(5) + except plpy.SPIError: + plpy.error('Error: areas of interest query failed, check input parameters') + plpy.notice('** Query failed: "%s"' % query) + plpy.notice('** Error: %s' % plpy.SPIError) + return pu.empty_zipped_array(5) + + ## collect attributes + numer = pu.get_attributes(result, 1) + denom = pu.get_attributes(result, 2) + + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_Rate(numer, denom, weight, + permutations=permutations) + + # find units of significance + quads = quad_position(lisa.q) + + return zip(lisa.Is, quads, lisa.p_sim, weight.id_order, lisa.y) + +def moran_local_bv(subquery, attr1, attr2, + permutations, geom_col, id_col, w_type, num_ngbrs): + """ + Moran's I (local) Bivariate (untested) + """ + plpy.notice('** Constructing query') + + qvals = {"num_ngbrs": num_ngbrs, + "attr1": attr1, + "attr2": attr2, + "subquery": subquery, + "geom_col": geom_col, + "id_col": id_col} + + query = pu.construct_neighbor_query(w_type, qvals) + + try: + result = plpy.execute(query) + # if there are no neighbors, exit + if len(result) == 0: + return pu.empty_zipped_array(4) + except plpy.SPIError: + plpy.error("Error: areas of interest query failed, " \ + "check input parameters") + plpy.notice('** Query failed: "%s"' % query) + return pu.empty_zipped_array(4) + + ## collect attributes + attr1_vals = pu.get_attributes(result, 1) + attr2_vals = pu.get_attributes(result, 2) + + # create weights + weight = pu.get_weight(result, w_type, num_ngbrs) + + # calculate LISA values + lisa = ps.esda.moran.Moran_Local_BV(attr1_vals, attr2_vals, weight, + permutations=permutations) + + plpy.notice("len of Is: %d" % len(lisa.Is)) + + # find clustering of significance + lisa_sig = quad_position(lisa.q) + + plpy.notice('** Finished calculations') + + return zip(lisa.Is, lisa_sig, lisa.p_sim, weight.id_order) + +# Low level functions ---------------------------------------- + +def map_quads(coord): + """ + Map a quadrant number to Moran's I designation + HH=1, LH=2, LL=3, HL=4 + Input: + @param coord (int): quadrant of a specific measurement + Output: + classification (one of 'HH', 'LH', 'LL', or 'HL') + """ + if coord == 1: + return 'HH' + elif coord == 2: + return 'LH' + elif coord == 3: + return 'LL' + elif coord == 4: + return 'HL' + else: + return None + +def quad_position(quads): + """ + Produce Moran's I classification based of n + Input: + @param quads ndarray: an array of quads classified by + 1-4 (PySAL default) + Output: + @param list: an array of quads classied by 'HH', 'LL', etc. + """ + return [map_quads(q) for q in quads] diff --git a/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py new file mode 100644 index 0000000..835880d --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/__init__.py @@ -0,0 +1 @@ +from pysal_utils import * diff --git a/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py new file mode 100644 index 0000000..02b5e35 --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/pysal_utils/pysal_utils.py @@ -0,0 +1,152 @@ +""" + Utilities module for generic PySAL functionality, mainly centered on translating queries into numpy arrays or PySAL weights objects +""" + +import numpy as np +import pysal as ps + +def construct_neighbor_query(w_type, query_vals): + """Return query (a string) used for finding neighbors + @param w_type text: type of neighbors to calculate ('knn' or 'queen') + @param query_vals dict: values used to construct the query + """ + + if w_type.lower() == 'knn': + return knn(query_vals) + else: + return queen(query_vals) + +## Build weight object +def get_weight(query_res, w_type='knn', num_ngbrs=5): + """ + Construct PySAL weight from return value of query + @param query_res: query results with attributes and neighbors + """ + if w_type.lower() == 'knn': + row_normed_weights = [1.0 / float(num_ngbrs)] * num_ngbrs + weights = {x['id']: row_normed_weights for x in query_res} + else: + weights = {x['id']: [1.0 / len(x['neighbors'])] * len(x['neighbors']) + if len(x['neighbors']) > 0 + else [] for x in query_res} + + neighbors = {x['id']: x['neighbors'] for x in query_res} + + return ps.W(neighbors, weights) + +def query_attr_select(params): + """ + Create portion of SELECT statement for attributes inolved in query. + @param params: dict of information used in query (column names, + table name, etc.) + """ + + attrs = [k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')] + + template = "i.\"{%(col)s}\"::numeric As attr%(alias_num)s, " + + attr_string = "" + + for idx, val in enumerate(sorted(attrs)): + attr_string += template % {"col": val, "alias_num": idx + 1} + + return attr_string + +def query_attr_where(params): + """ + Create portion of WHERE clauses for weeding out NULL-valued geometries + """ + attrs = sorted([k for k in params + if k not in ('id_col', 'geom_col', 'subquery', 'num_ngbrs')]) + + attr_string = [] + + for attr in attrs: + attr_string.append("idx_replace.\"{%s}\" IS NOT NULL" % attr) + + if len(attrs) == 2: + attr_string.append("idx_replace.\"{%s}\" <> 0" % attrs[1]) + + out = " AND ".join(attr_string) + + return out + +def knn(params): + """SQL query for k-nearest neighbors. + @param vars: dict of values to fill template + """ + + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE " \ + "i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "%(attr_where_j)s " \ + "ORDER BY " \ + "j.\"{geom_col}\" <-> i.\"{geom_col}\" ASC " \ + "LIMIT {num_ngbrs})" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## SQL query for finding queens neighbors (all contiguous polygons) +def queen(params): + """SQL query for queen neighbors. + @param params dict: information to fill query + """ + attr_select = query_attr_select(params) + attr_where = query_attr_where(params) + + replacements = {"attr_select": attr_select, + "attr_where_i": attr_where.replace("idx_replace", "i"), + "attr_where_j": attr_where.replace("idx_replace", "j")} + + query = "SELECT " \ + "i.\"{id_col}\" As id, " \ + "%(attr_select)s" \ + "(SELECT ARRAY(SELECT j.\"{id_col}\" " \ + "FROM ({subquery}) As j " \ + "WHERE i.\"{id_col}\" <> j.\"{id_col}\" AND " \ + "ST_Touches(i.\"{geom_col}\", j.\"{geom_col}\") AND " \ + "%(attr_where_j)s)" \ + ") As neighbors " \ + "FROM ({subquery}) As i " \ + "WHERE " \ + "%(attr_where_i)s " \ + "ORDER BY i.\"{id_col}\" ASC;" % replacements + + return query.format(**params) + +## to add more weight methods open a ticket or pull request + +def get_attributes(query_res, attr_num=1): + """ + @param query_res: query results with attributes and neighbors + @param attr_num: attribute number (1, 2, ...) + """ + return np.array([x['attr' + str(attr_num)] for x in query_res], dtype=np.float) + +def empty_zipped_array(num_nones): + """ + prepare return values for cases of empty weights objects (no neighbors) + Input: + @param num_nones int: number of columns (e.g., 4) + Output: + [(None, None, None, None)] + """ + + return [tuple([None] * num_nones)] diff --git a/release/python/0.0.4/crankshaft/crankshaft/random_seeds.py b/release/python/0.0.4/crankshaft/crankshaft/random_seeds.py new file mode 100644 index 0000000..b7c8eed --- /dev/null +++ b/release/python/0.0.4/crankshaft/crankshaft/random_seeds.py @@ -0,0 +1,10 @@ +import random +import numpy + +def set_random_seeds(value): + """ + Set the seeds of the RNGs (Random Number Generators) + used internally. + """ + random.seed(value) + numpy.random.seed(value) diff --git a/release/python/0.0.4/crankshaft/setup.py b/release/python/0.0.4/crankshaft/setup.py new file mode 100644 index 0000000..32d1ead --- /dev/null +++ b/release/python/0.0.4/crankshaft/setup.py @@ -0,0 +1,48 @@ + +""" +CartoDB Spatial Analysis Python Library +See: +https://github.com/CartoDB/crankshaft +""" + +from setuptools import setup, find_packages + +setup( + name='crankshaft', + + version='0.0.4', + + description='CartoDB Spatial Analysis Python Library', + + url='https://github.com/CartoDB/crankshaft', + + author='Data Services Team - CartoDB', + author_email='dataservices@cartodb.com', + + license='MIT', + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Mapping comunity', + 'Topic :: Maps :: Mapping Tools', + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 2.7', + ], + + keywords='maps mapping tools spatial analysis geostatistics', + + packages=find_packages(exclude=['contrib', 'docs', 'tests']), + + extras_require={ + 'dev': ['unittest'], + 'test': ['unittest', 'nose', 'mock'], + }, + + # The choice of component versions is dictated by what's + # provisioned in the production servers. + install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.11.2', 'scikit-learn==0.14.1'], + + requires=['pysal', 'numpy', 'sklearn'], + + test_suite='test' +) diff --git a/release/python/0.0.4/crankshaft/test/fixtures/kmeans.json b/release/python/0.0.4/crankshaft/test/fixtures/kmeans.json new file mode 100644 index 0000000..8f31c79 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/fixtures/kmeans.json @@ -0,0 +1 @@ +[{"xs": [9.917239463463458, 9.042767302696836, 10.798929825304187, 8.763751051762995, 11.383882954810852, 11.018206993460897, 8.939526075734316, 9.636159342565252, 10.136336896960058, 11.480610059427342, 12.115011910725082, 9.173267848893428, 10.239300931201738, 8.00012512174072, 8.979962292282131, 9.318376124429575, 10.82259513754284, 10.391747171927115, 10.04904588886165, 9.96007160443463, -0.78825626804569, -0.3511819898577426, -1.2796410003764271, -0.3977049391203402, 2.4792311265774667, 1.3670311632092624, 1.2963504112955613, 2.0404844103073025, -1.6439708506073223, 0.39122885445645805, 1.026031821452462, -0.04044477160482201, -0.7442346929085072, -0.34687120826243034, -0.23420359971379054, -0.5919629143336708, -0.202903054395391, -0.1893399644841902, 1.9331834251176807, -0.12321054392851609], "ys": [8.735627063679981, 9.857615954045011, 10.81439096759407, 10.586727233537191, 9.232919976568622, 11.54281262696508, 8.392787912674466, 9.355119689665944, 9.22380703532752, 10.542142541823122, 10.111980619367035, 10.760836265570738, 8.819773453269804, 10.25325722424816, 9.802077905695608, 8.955420161552611, 9.833801181904477, 10.491684241001613, 12.076108669877556, 11.74289693140474, -0.5685725015474191, -0.5715728344759778, -0.20180907868635137, 0.38431336480089595, -0.3402202083684184, -2.4652736827783586, 0.08295159401756182, 0.8503818775816505, 0.6488691600321166, 0.5794762568230527, -0.6770063922144103, -0.6557616416449478, -1.2834289177624947, 0.1096318195532717, -0.38986922166834853, -1.6224497706950238, 0.09429787743230483, 0.4005097316394031, -0.508002811195673, -1.2473463371366507], "ids": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39]}] \ No newline at end of file diff --git a/release/python/0.0.4/crankshaft/test/fixtures/moran.json b/release/python/0.0.4/crankshaft/test/fixtures/moran.json new file mode 100644 index 0000000..2f75cf1 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/fixtures/moran.json @@ -0,0 +1,52 @@ +[[0.9319096128346788, "HH"], +[-1.135787401862846, "HL"], +[0.11732030672508517, "LL"], +[0.6152779669180425, "LL"], +[-0.14657336660125297, "LH"], +[0.6967858120189607, "LL"], +[0.07949310115714454, "HH"], +[0.4703198759258987, "HH"], +[0.4421125200498064, "HH"], +[0.5724288737143592, "LL"], +[0.8970743435692062, "LL"], +[0.18327334401918674, "LL"], +[-0.01466729201304962, "HL"], +[0.3481559372544409, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329988, "HH"], +[0.4373841193538136, "HH"], +[0.15971286468915544, "LL"], +[1.0543588860308968, "HH"], +[1.7372866900020818, "HH"], +[1.091998586053999, "LL"], +[0.1171572584252222, "HH"], +[0.08438455015300014, "LL"], +[0.06547094736902978, "LL"], +[0.15482141569329985, "HH"], +[1.1627044812890683, "HH"], +[0.06547094736902978, "LL"], +[0.795275137550483, "HH"], +[0.18562939195219, "LL"], +[0.3010757406693439, "LL"], +[2.8205795942839376, "HH"], +[0.11259190602909264, "LL"], +[-0.07116352791516614, "HL"], +[-0.09945240794119009, "LH"], +[0.18562939195219, "LL"], +[0.1832733440191868, "LL"], +[-0.39054253768447705, "HL"], +[-0.1672071289487642, "HL"], +[0.3337669247916343, "HH"], +[0.2584386102554792, "HH"], +[-0.19733845476322634, "HL"], +[-0.9379282899805409, "LH"], +[-0.028770969951095866, "LH"], +[0.051367269430983485, "LL"], +[-0.2172548045913472, "LH"], +[0.05136726943098351, "LL"], +[0.04191046803899837, "LL"], +[0.7482357030403517, "HH"], +[-0.014585767863118111, "LH"], +[0.5410013139159929, "HH"], +[1.0223932668429925, "LL"], +[1.4179402898927476, "LL"]] \ No newline at end of file diff --git a/release/python/0.0.4/crankshaft/test/fixtures/neighbors.json b/release/python/0.0.4/crankshaft/test/fixtures/neighbors.json new file mode 100644 index 0000000..055b359 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/fixtures/neighbors.json @@ -0,0 +1,54 @@ +[ + {"neighbors": [48, 26, 20, 9, 31], "id": 1, "value": 0.5}, + {"neighbors": [30, 16, 46, 3, 4], "id": 2, "value": 0.7}, + {"neighbors": [46, 30, 2, 12, 16], "id": 3, "value": 0.2}, + {"neighbors": [18, 30, 23, 2, 52], "id": 4, "value": 0.1}, + {"neighbors": [47, 40, 45, 37, 28], "id": 5, "value": 0.3}, + {"neighbors": [10, 21, 41, 14, 37], "id": 6, "value": 0.05}, + {"neighbors": [8, 17, 43, 25, 12], "id": 7, "value": 0.4}, + {"neighbors": [17, 25, 43, 22, 7], "id": 8, "value": 0.7}, + {"neighbors": [39, 34, 1, 26, 48], "id": 9, "value": 0.5}, + {"neighbors": [6, 37, 5, 45, 49], "id": 10, "value": 0.04}, + {"neighbors": [51, 41, 29, 21, 14], "id": 11, "value": 0.08}, + {"neighbors": [44, 46, 43, 50, 3], "id": 12, "value": 0.2}, + {"neighbors": [45, 23, 14, 28, 18], "id": 13, "value": 0.4}, + {"neighbors": [41, 29, 13, 23, 6], "id": 14, "value": 0.2}, + {"neighbors": [36, 27, 32, 33, 24], "id": 15, "value": 0.3}, + {"neighbors": [19, 2, 46, 44, 28], "id": 16, "value": 0.4}, + {"neighbors": [8, 25, 43, 7, 22], "id": 17, "value": 0.6}, + {"neighbors": [23, 4, 29, 14, 13], "id": 18, "value": 0.3}, + {"neighbors": [42, 16, 28, 26, 40], "id": 19, "value": 0.7}, + {"neighbors": [1, 48, 31, 26, 42], "id": 20, "value": 0.8}, + {"neighbors": [41, 6, 11, 14, 10], "id": 21, "value": 0.1}, + {"neighbors": [25, 50, 43, 31, 44], "id": 22, "value": 0.4}, + {"neighbors": [18, 13, 14, 4, 2], "id": 23, "value": 0.1}, + {"neighbors": [33, 49, 34, 47, 27], "id": 24, "value": 0.3}, + {"neighbors": [43, 8, 22, 17, 50], "id": 25, "value": 0.4}, + {"neighbors": [1, 42, 20, 31, 48], "id": 26, "value": 0.6}, + {"neighbors": [32, 15, 36, 33, 24], "id": 27, "value": 0.3}, + {"neighbors": [40, 45, 19, 5, 13], "id": 28, "value": 0.8}, + {"neighbors": [11, 51, 41, 14, 18], "id": 29, "value": 0.3}, + {"neighbors": [2, 3, 4, 46, 18], "id": 30, "value": 0.1}, + {"neighbors": [20, 26, 1, 50, 48], "id": 31, "value": 0.9}, + {"neighbors": [27, 36, 15, 49, 24], "id": 32, "value": 0.3}, + {"neighbors": [24, 27, 49, 34, 32], "id": 33, "value": 0.4}, + {"neighbors": [47, 9, 39, 40, 24], "id": 34, "value": 0.3}, + {"neighbors": [38, 51, 11, 21, 41], "id": 35, "value": 0.3}, + {"neighbors": [15, 32, 27, 49, 33], "id": 36, "value": 0.2}, + {"neighbors": [49, 10, 5, 47, 24], "id": 37, "value": 0.5}, + {"neighbors": [35, 21, 51, 11, 41], "id": 38, "value": 0.4}, + {"neighbors": [9, 34, 48, 1, 47], "id": 39, "value": 0.6}, + {"neighbors": [28, 47, 5, 9, 34], "id": 40, "value": 0.5}, + {"neighbors": [11, 14, 29, 21, 6], "id": 41, "value": 0.4}, + {"neighbors": [26, 19, 1, 9, 31], "id": 42, "value": 0.2}, + {"neighbors": [25, 12, 8, 22, 44], "id": 43, "value": 0.3}, + {"neighbors": [12, 50, 46, 16, 43], "id": 44, "value": 0.2}, + {"neighbors": [28, 13, 5, 40, 19], "id": 45, "value": 0.3}, + {"neighbors": [3, 12, 44, 2, 16], "id": 46, "value": 0.2}, + {"neighbors": [34, 40, 5, 49, 24], "id": 47, "value": 0.3}, + {"neighbors": [1, 20, 26, 9, 39], "id": 48, "value": 0.5}, + {"neighbors": [24, 37, 47, 5, 33], "id": 49, "value": 0.2}, + {"neighbors": [44, 22, 31, 42, 26], "id": 50, "value": 0.6}, + {"neighbors": [11, 29, 41, 14, 21], "id": 51, "value": 0.01}, + {"neighbors": [4, 18, 29, 51, 23], "id": 52, "value": 0.01} + ] diff --git a/release/python/0.0.4/crankshaft/test/helper.py b/release/python/0.0.4/crankshaft/test/helper.py new file mode 100644 index 0000000..7d28b94 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/helper.py @@ -0,0 +1,13 @@ +import unittest + +from mock_plpy import MockPlPy +plpy = MockPlPy() + +import sys +sys.modules['plpy'] = plpy + +import os + +def fixture_file(name): + dir = os.path.dirname(os.path.realpath(__file__)) + return os.path.join(dir, 'fixtures', name) diff --git a/release/python/0.0.4/crankshaft/test/mock_plpy.py b/release/python/0.0.4/crankshaft/test/mock_plpy.py new file mode 100644 index 0000000..63c88f6 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/mock_plpy.py @@ -0,0 +1,34 @@ +import re + +class MockPlPy: + def __init__(self): + self._reset() + + def _reset(self): + self.infos = [] + self.notices = [] + self.debugs = [] + self.logs = [] + self.warnings = [] + self.errors = [] + self.fatals = [] + self.executes = [] + self.results = [] + self.prepares = [] + self.results = [] + + def _define_result(self, query, result): + pattern = re.compile(query, re.IGNORECASE | re.MULTILINE) + self.results.append([pattern, result]) + + def notice(self, msg): + self.notices.append(msg) + + def info(self, msg): + self.infos.append(msg) + + def execute(self, query): # TODO: additional arguments + for result in self.results: + if result[0].match(query): + return result[1] + return [] diff --git a/release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py b/release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py new file mode 100644 index 0000000..aba8e07 --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/test_cluster_kmeans.py @@ -0,0 +1,38 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file +import numpy as np +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class KMeansTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.cluster_data = json.loads(open(fixture_file('kmeans.json')).read()) + self.params = {"subquery": "select * from table", + "no_clusters": "10" + } + + def test_kmeans(self): + data = self.cluster_data + plpy._define_result('select' ,data) + clusters = cc.kmeans('subquery', 2) + labels = [a[1] for a in clusters] + c1 = [a for a in clusters if a[1]==0] + c2 = [a for a in clusters if a[1]==1] + + self.assertEqual(len(np.unique(labels)),2) + self.assertEqual(len(c1),20) + self.assertEqual(len(c2),20) + diff --git a/release/python/0.0.4/crankshaft/test/test_clustering_moran.py b/release/python/0.0.4/crankshaft/test/test_clustering_moran.py new file mode 100644 index 0000000..393e93b --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/test_clustering_moran.py @@ -0,0 +1,83 @@ +import unittest +import numpy as np + + +# from mock_plpy import MockPlPy +# plpy = MockPlPy() +# +# import sys +# sys.modules['plpy'] = plpy +from helper import plpy, fixture_file + +import crankshaft.clustering as cc +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds +import json + +class MoranTest(unittest.TestCase): + """Testing class for Moran's I functions""" + + def setUp(self): + plpy._reset() + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + self.neighbors_data = json.loads(open(fixture_file('neighbors.json')).read()) + self.moran_data = json.loads(open(fixture_file('moran.json')).read()) + + def test_map_quads(self): + """Test map_quads""" + self.assertEqual(cc.map_quads(1), 'HH') + self.assertEqual(cc.map_quads(2), 'LH') + self.assertEqual(cc.map_quads(3), 'LL') + self.assertEqual(cc.map_quads(4), 'HL') + self.assertEqual(cc.map_quads(33), None) + self.assertEqual(cc.map_quads('andy'), None) + + def test_quad_position(self): + """Test lisa_sig_vals""" + + quads = np.array([1, 2, 3, 4], np.int) + + ans = np.array(['HH', 'LH', 'LL', 'HL']) + test_ans = cc.quad_position(quads) + + self.assertTrue((test_ans == ans).all()) + + def test_moran_local(self): + """Test Moran's I local""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local('subquery', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + self.assertEqual(res_quad, exp_quad) + + def test_moran_local_rate(self): + """Test Moran's I rate""" + data = [ { 'id': d['id'], 'attr1': d['value'], 'attr2': 1, 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1234) + result = cc.moran_local_rate('subquery', 'numerator', 'denominator', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None? ', result == None + result = [(row[0], row[1]) for row in result] + expected = self.moran_data + for ([res_val, res_quad], [exp_val, exp_quad]) in zip(result, expected): + self.assertAlmostEqual(res_val, exp_val) + + def test_moran(self): + """Test Moran's I global""" + data = [{ 'id': d['id'], 'attr1': d['value'], 'neighbors': d['neighbors'] } for d in self.neighbors_data] + plpy._define_result('select', data) + random_seeds.set_random_seeds(1235) + result = cc.moran('table', 'value', 'knn', 5, 99, 'the_geom', 'cartodb_id') + print 'result == None?', result == None + result_moran = result[0][0] + expected_moran = np.array([row[0] for row in self.moran_data]).mean() + self.assertAlmostEqual(expected_moran, result_moran, delta=10e-2) diff --git a/release/python/0.0.4/crankshaft/test/test_pysal_utils.py b/release/python/0.0.4/crankshaft/test/test_pysal_utils.py new file mode 100644 index 0000000..4ea0d9b --- /dev/null +++ b/release/python/0.0.4/crankshaft/test/test_pysal_utils.py @@ -0,0 +1,107 @@ +import unittest + +import crankshaft.pysal_utils as pu +from crankshaft import random_seeds + + +class PysalUtilsTest(unittest.TestCase): + """Testing class for utility functions related to PySAL integrations""" + + def setUp(self): + self.params = {"id_col": "cartodb_id", + "attr1": "andy", + "attr2": "jay_z", + "subquery": "SELECT * FROM a_list", + "geom_col": "the_geom", + "num_ngbrs": 321} + + def test_query_attr_select(self): + """Test query_attr_select""" + + ans = "i.\"{attr1}\"::numeric As attr1, " \ + "i.\"{attr2}\"::numeric As attr2, " + + self.assertEqual(pu.query_attr_select(self.params), ans) + + def test_query_attr_where(self): + """Test pu.query_attr_where""" + + ans = "idx_replace.\"{attr1}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" IS NOT NULL AND " \ + "idx_replace.\"{attr2}\" <> 0" + + self.assertEqual(pu.query_attr_where(self.params), ans) + + def test_knn(self): + """Test knn neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0 " \ + "ORDER BY " \ + "j.\"the_geom\" <-> i.\"the_geom\" ASC " \ + "LIMIT 321)) As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.knn(self.params), ans) + + def test_queen(self): + """Test queen neighbors constructor""" + + ans = "SELECT i.\"cartodb_id\" As id, " \ + "i.\"andy\"::numeric As attr1, " \ + "i.\"jay_z\"::numeric As attr2, " \ + "(SELECT ARRAY(SELECT j.\"cartodb_id\" " \ + "FROM (SELECT * FROM a_list) As j " \ + "WHERE " \ + "i.\"cartodb_id\" <> j.\"cartodb_id\" AND " \ + "ST_Touches(i.\"the_geom\", " \ + "j.\"the_geom\") AND " \ + "j.\"andy\" IS NOT NULL AND " \ + "j.\"jay_z\" IS NOT NULL AND " \ + "j.\"jay_z\" <> 0)" \ + ") As neighbors " \ + "FROM (SELECT * FROM a_list) As i " \ + "WHERE i.\"andy\" IS NOT NULL AND " \ + "i.\"jay_z\" IS NOT NULL AND " \ + "i.\"jay_z\" <> 0 " \ + "ORDER BY i.\"cartodb_id\" ASC;" + + self.assertEqual(pu.queen(self.params), ans) + + def test_construct_neighbor_query(self): + """Test construct_neighbor_query""" + + # Compare to raw knn query + self.assertEqual(pu.construct_neighbor_query('knn', self.params), + pu.knn(self.params)) + + def test_get_attributes(self): + """Test get_attributes""" + + ## need to add tests + + self.assertEqual(True, True) + + def test_get_weight(self): + """Test get_weight""" + + self.assertEqual(True, True) + + def test_empty_zipped_array(self): + """Test empty_zipped_array""" + ans2 = [(None, None)] + ans4 = [(None, None, None, None)] + self.assertEqual(pu.empty_zipped_array(2), ans2) + self.assertEqual(pu.empty_zipped_array(4), ans4) diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index e71321f..01088b1 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.0.3' +default_version = '0.0.4' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft