From cdd2d9e722fe2e93d458bb647216b7871aeea0b6 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 8 Mar 2016 19:35:02 +0100 Subject: [PATCH 01/22] Directory reorganization and sketch of new versioning procedure --- Makefile | 4 +- README.md | 84 +++++++++- {pg/doc => doc}/02_moran.md | 0 {pg/doc => doc}/03_overlap_sum.md | 0 pg/.gitignore | 3 - pg/Makefile | 33 ---- pg/crankshaft--0.0.1.sql | 148 ------------------ pg/test/0.0.1/expected/01_install_test.out | 6 - python/Makefile | 11 -- src/pg/.gitignore | 6 + src/pg/Makefile | 41 +++++ {pg => src/pg}/README.md | 0 {pg => src/pg}/crankshaft.control | 0 {pg/sql/0.0.1 => src/pg/sql}/00_header.sql | 0 .../0.0.1 => src/pg/sql}/01_random_seeds.sql | 0 {pg/sql/0.0.1 => src/pg/sql}/02_moran.sql | 0 .../0.0.1 => src/pg/sql}/03_overlap_sum.sql | 0 .../0.0.1 => src/pg/sql}/04_dot_density.sql | 0 .../0.0.1 => src/pg/sql}/90_permissions.sql | 0 .../pg/test/expected}/01_install_test.out | 2 +- .../pg/test}/expected/02_moran_test.out | 0 .../pg/test}/expected/03_overlap_sum_test.out | 0 .../pg/test}/expected/04_dot_density_test.out | 0 {pg => src/pg}/test/fixtures/polyg_values.sql | 0 {pg => src/pg}/test/fixtures/ppoints.sql | 0 {pg => src/pg}/test/fixtures/ppoints2.sql | 0 .../pg/test}/sql/01_install_test.sql | 2 +- .../pg/test}/sql/02_moran_test.sql | 0 .../pg/test}/sql/03_overlap_sum_test.sql | 0 .../pg/test}/sql/04_dot_density_test.sql | 0 .../pg/test}/sql/90_permissions.sql | 0 {python => src/py}/.gitignore | 1 + src/py/Makefile | 9 ++ {python => src/py}/README.md | 0 .../py}/crankshaft/crankshaft/__init__.py | 0 .../crankshaft/clustering/__init__.py | 0 .../crankshaft/crankshaft/clustering/moran.py | 0 .../py}/crankshaft/crankshaft/random_seeds.py | 0 {python => src/py}/crankshaft/setup.py | 0 .../py}/crankshaft/test/fixtures/moran.json | 0 .../crankshaft/test/fixtures/neighbors.json | 0 {python => src/py}/crankshaft/test/helper.py | 0 .../py}/crankshaft/test/mock_plpy.py | 0 .../crankshaft/test/test_clustering_moran.py | 0 44 files changed, 142 insertions(+), 208 deletions(-) rename {pg/doc => doc}/02_moran.md (100%) rename {pg/doc => doc}/03_overlap_sum.md (100%) delete mode 100644 pg/.gitignore delete mode 100644 pg/Makefile delete mode 100644 pg/crankshaft--0.0.1.sql delete mode 100644 pg/test/0.0.1/expected/01_install_test.out delete mode 100644 python/Makefile create mode 100644 src/pg/.gitignore create mode 100644 src/pg/Makefile rename {pg => src/pg}/README.md (100%) rename {pg => src/pg}/crankshaft.control (100%) rename {pg/sql/0.0.1 => src/pg/sql}/00_header.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/01_random_seeds.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/02_moran.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/03_overlap_sum.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/04_dot_density.sql (100%) rename {pg/sql/0.0.1 => src/pg/sql}/90_permissions.sql (100%) rename {pg/test/0.0.1/results => src/pg/test/expected}/01_install_test.out (75%) rename {pg/test/0.0.1 => src/pg/test}/expected/02_moran_test.out (100%) rename {pg/test/0.0.1 => src/pg/test}/expected/03_overlap_sum_test.out (100%) rename {pg/test/0.0.1 => src/pg/test}/expected/04_dot_density_test.out (100%) rename {pg => src/pg}/test/fixtures/polyg_values.sql (100%) rename {pg => src/pg}/test/fixtures/ppoints.sql (100%) rename {pg => src/pg}/test/fixtures/ppoints2.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/01_install_test.sql (75%) rename {pg/test/0.0.1 => src/pg/test}/sql/02_moran_test.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/03_overlap_sum_test.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/04_dot_density_test.sql (100%) rename {pg/test/0.0.1 => src/pg/test}/sql/90_permissions.sql (100%) rename {python => src/py}/.gitignore (54%) create mode 100644 src/py/Makefile rename {python => src/py}/README.md (100%) rename {python => src/py}/crankshaft/crankshaft/__init__.py (100%) rename {python => src/py}/crankshaft/crankshaft/clustering/__init__.py (100%) rename {python => src/py}/crankshaft/crankshaft/clustering/moran.py (100%) rename {python => src/py}/crankshaft/crankshaft/random_seeds.py (100%) rename {python => src/py}/crankshaft/setup.py (100%) rename {python => src/py}/crankshaft/test/fixtures/moran.json (100%) rename {python => src/py}/crankshaft/test/fixtures/neighbors.json (100%) rename {python => src/py}/crankshaft/test/helper.py (100%) rename {python => src/py}/crankshaft/test/mock_plpy.py (100%) rename {python => src/py}/crankshaft/test/test_clustering_moran.py (100%) diff --git a/Makefile b/Makefile index d1d9734..5cae2cb 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ -EXT_DIR = pg -PYP_DIR = python +EXT_DIR = src/pg +PYP_DIR = src/py .PHONY: install .PHONY: run_tests diff --git a/README.md b/README.md index 61e8738..5a12c2b 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,87 @@ CartoDB Spatial Analysis extension for PostgreSQL. ## Code organization -* *pg* contains the PostgreSQL extension source code -* *python* Python module +* *doc* documentation +* *src* source code +* - *src/pg* contains the PostgreSQL extension source code +* - *src/py* Python module source code +* *release* reselesed versions ## Requirements -* pip +* pip, virtualenv, PostgreSQL + +# Working Process + +## Development + +Work in `src/pg/sql`, `src/py/crankshaft`; +use topic branch. + +Update local installation with `sudo make install` +(this will update the 'dev' version of the extension in 'src/pg/') + +Run the tests with `PGUSER=postgres make test` + +Update extension in working database with + +* `ALTER EXTENSION crankshaft VERSION TO 'current';` + `ALTER EXTENSION crankshaft VERSION TO 'dev';` + +Note: we keep the current development version install as 'dev' always; +we update through the 'current' alias to allow changing the extension +contents but not the version identifier. This will fail if the +changes involve incompatible function changes such as a different +return type; in that case the offending function (or the whole extension) +should be dropped manually before the update. + +If the extension has not previously been installed in a database +we can: + +Add tests... + +* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` + +Test + +Commit, push, create PR, wait for CI tests, CR, ... + +## Release + +To release current development version +(working directory should be clean in dev branch) + +(process to be gradually automated) + +For backwards compatible changes (no return value, num of arguments, etc. changes...) +new version number increasing either patch level (no new functionality) +or minor level (new functionality) => 'X.Y.Z'. +Update version in src/pg/crankshaft.control +Copy release/crankshaft--current.sql to release/crankshaft--X.Y.Z.sql +Prepare incremental downgrade, upgrade scripts.... + +Python: ... + +Install the new release + +`make install-release` + +Test the new release + +`make test-release` + +Push the release + +Wait for CI tests + +Merge into master + +Deploy: install extension and python to production hosts, +update extension in databases (limited to team users, data observatory, ...) + +Release manager role: ... + +.sql release scripts +commit +tests: staging.... +merge, tag, deploy... diff --git a/pg/doc/02_moran.md b/doc/02_moran.md similarity index 100% rename from pg/doc/02_moran.md rename to doc/02_moran.md diff --git a/pg/doc/03_overlap_sum.md b/doc/03_overlap_sum.md similarity index 100% rename from pg/doc/03_overlap_sum.md rename to doc/03_overlap_sum.md diff --git a/pg/.gitignore b/pg/.gitignore deleted file mode 100644 index 820df46..0000000 --- a/pg/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -regression.diffs -regression.out -results/ diff --git a/pg/Makefile b/pg/Makefile deleted file mode 100644 index 99605f5..0000000 --- a/pg/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -# Makefile to generate the extension out of separate sql source files. -# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. - -EXTENSION = crankshaft -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") - -# The new version to be generated from templates -NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql - -# DATA is a special variable used by postgres build infrastructure -# These are the files to be installed in the server shared dir, -# for installation from scratch, upgrades and downgrades. -# @see http://www.postgresql.org/docs/current/static/extend-pgxs.html -DATA = $(NEW_EXTENSION_ARTIFACT) - -SOURCES_DATA_DIR = sql/$(EXTVERSION) -SOURCES_DATA = $(wildcard sql/$(EXTVERSION)/*.sql) - -# The extension installation artifacts are stored in the base subdirectory -$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) - rm -f $@ - cat $(SOURCES_DATA_DIR)/*.sql >> $@ - -REGRESS = $(notdir $(basename $(wildcard test/$(EXTVERSION)/sql/*test.sql))) -TEST_DIR = test/$(EXTVERSION) -REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' - -PG_CONFIG = pg_config -PGXS := $(shell $(PG_CONFIG) --pgxs) -include $(PGXS) - -# This seems to be needed at least for PG 9.3.11 -all: $(DATA) diff --git a/pg/crankshaft--0.0.1.sql b/pg/crankshaft--0.0.1.sql deleted file mode 100644 index 436beea..0000000 --- a/pg/crankshaft--0.0.1.sql +++ /dev/null @@ -1,148 +0,0 @@ ---DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES --- Complain if script is sourced in psql, rather than via CREATE EXTENSION -\echo Use "CREATE EXTENSION crankshaft" to load this file. \quit --- Internal function. --- Set the seeds of the RNGs (Random Number Generators) --- used internally. -CREATE OR REPLACE FUNCTION -_cdb_random_seeds (seed_value INTEGER) RETURNS VOID -AS $$ - from crankshaft import random_seeds - random_seeds.set_random_seeds(seed_value) -$$ LANGUAGE plpythonu; --- Moran's I -CREATE OR REPLACE FUNCTION - cdb_moran_local ( - t TEXT, - attr TEXT, - significance float DEFAULT 0.05, - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn') -RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) -AS $$ - from crankshaft.clustering import moran_local - # TODO: use named parameters or a dictionary - return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) -$$ LANGUAGE plpythonu; - --- Moran's I Local Rate -CREATE OR REPLACE FUNCTION - cdb_moran_local_rate(t TEXT, - numerator TEXT, - denominator TEXT, - significance FLOAT DEFAULT 0.05, - num_ngbrs INT DEFAULT 5, - permutations INT DEFAULT 99, - geom_column TEXT DEFAULT 'the_geom', - id_col TEXT DEFAULT 'cartodb_id', - w_type TEXT DEFAULT 'knn') -RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) -AS $$ - from crankshaft.clustering import moran_local_rate - # TODO: use named parameters or a dictionary - return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) -$$ LANGUAGE plpythonu; --- Function by Stuart Lynn for a simple interpolation of a value --- from a polygon table over an arbitrary polygon --- (weighted by the area proportion overlapped) --- Aereal weighting is a very simple form of aereal interpolation. --- --- Parameters: --- * geom a Polygon geometry which defines the area where a value will be --- estimated as the area-weighted sum of a given table/column --- * target_table_name table name of the table that provides the values --- * target_column column name of the column that provides the values --- * schema_name optional parameter to defina the schema the target table --- belongs to, which is necessary if its not in the search_path. --- Note that target_table_name should never include the schema in it. --- Return value: --- Aereal-weighted interpolation of the column values over the geometry -CREATE OR REPLACE -FUNCTION cdb_overlap_sum(geom geometry, target_table_name text, target_column text, schema_name text DEFAULT NULL) - RETURNS numeric AS -$$ -DECLARE - result numeric; - qualified_name text; -BEGIN - IF schema_name IS NULL THEN - qualified_name := Format('%I', target_table_name); - ELSE - qualified_name := Format('%I.%s', schema_name, target_table_name); - END IF; - EXECUTE Format(' - SELECT sum(%I*ST_Area(St_Intersection($1, a.the_geom))/ST_Area(a.the_geom)) - FROM %s AS a - WHERE $1 && a.the_geom - ', target_column, qualified_name) - USING geom - INTO result; - RETURN result; -END; -$$ LANGUAGE plpgsql; --- --- Creates N points randomly distributed arround the polygon --- --- @param g - the geometry to be turned in to points --- --- @param no_points - the number of points to generate --- --- @params max_iter_per_point - the function generates points in the polygon's bounding box --- and discards points which don't lie in the polygon. max_iter_per_point specifies how many --- misses per point the funciton accepts before giving up. --- --- Returns: Multipoint with the requested points -CREATE OR REPLACE FUNCTION cdb_dot_density(geom geometry , no_points Integer, max_iter_per_point Integer DEFAULT 1000) -RETURNS GEOMETRY AS $$ -DECLARE - extent GEOMETRY; - test_point Geometry; - width NUMERIC; - height NUMERIC; - x0 NUMERIC; - y0 NUMERIC; - xp NUMERIC; - yp NUMERIC; - no_left INTEGER; - remaining_iterations INTEGER; - points GEOMETRY[]; - bbox_line GEOMETRY; - intersection_line GEOMETRY; -BEGIN - extent := ST_Envelope(geom); - width := ST_XMax(extent) - ST_XMIN(extent); - height := ST_YMax(extent) - ST_YMIN(extent); - x0 := ST_XMin(extent); - y0 := ST_YMin(extent); - no_left := no_points; - - LOOP - if(no_left=0) THEN - EXIT; - END IF; - yp = y0 + height*random(); - bbox_line = ST_MakeLine( - ST_SetSRID(ST_MakePoint(yp, x0),4326), - ST_SetSRID(ST_MakePoint(yp, x0+width),4326) - ); - intersection_line = ST_Intersection(bbox_line,geom); - test_point = ST_LineInterpolatePoint(st_makeline(st_linemerge(intersection_line)),random()); - points := points || test_point; - no_left = no_left - 1 ; - END LOOP; - RETURN ST_Collect(points); -END; -$$ -LANGUAGE plpgsql VOLATILE; --- Make sure by default there are no permissions for publicuser --- NOTE: this happens at extension creation time, as part of an implicit transaction. --- REVOKE ALL PRIVILEGES ON SCHEMA cdb_crankshaft FROM PUBLIC, publicuser CASCADE; - --- Grant permissions on the schema to publicuser (but just the schema) -GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; - --- Revoke execute permissions on all functions in the schema by default --- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_crankshaft FROM PUBLIC, publicuser; diff --git a/pg/test/0.0.1/expected/01_install_test.out b/pg/test/0.0.1/expected/01_install_test.out deleted file mode 100644 index c14537c..0000000 --- a/pg/test/0.0.1/expected/01_install_test.out +++ /dev/null @@ -1,6 +0,0 @@ --- Install dependencies -CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis; -CREATE EXTENSION cartodb; --- Install the extension -CREATE EXTENSION crankshaft; diff --git a/python/Makefile b/python/Makefile deleted file mode 100644 index 07b41dd..0000000 --- a/python/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -# Install the package (needs root privileges) -install: - pip install ./crankshaft --upgrade - -# Test from source code -test: - (cd crankshaft && nosetests test/) - -# Test currently installed package -testinstalled: - nosetests crankshaft/test/ diff --git a/src/pg/.gitignore b/src/pg/.gitignore new file mode 100644 index 0000000..b58a014 --- /dev/null +++ b/src/pg/.gitignore @@ -0,0 +1,6 @@ +regression.diffs +regression.out +results/ +crankshaft--dev.sql +crankshaft--dev--current.sql +crankshaft--current--dev.sql diff --git a/src/pg/Makefile b/src/pg/Makefile new file mode 100644 index 0000000..4ff713d --- /dev/null +++ b/src/pg/Makefile @@ -0,0 +1,41 @@ +# Generation of a new development version 'dev' (with an alias 'current' for +# updating easily by upgrading to 'current', then 'dev') + +# sudo make install -- generate the 'dev' version from current source +# and make it available to PostgreSQL +# PGUSER=postgres make installcheck -- test the 'dev' extension + +EXTENSION = crankshaft + +DATA = $(EXTENSION)--dev.sql \ + $(EXTENSION)--current--dev.sql \ + $(EXTENSION)--dev--current.sql + +SOURCES_DATA_DIR = sql +SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) + +$(DATA): $(SOURCES_DATA) + cat $(SOURCES_DATA_DIR)/*.sql > $@ + +TEST_DIR = test +REGRESS = $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +# This seems to be needed at least for PG 9.3.11 +all: $(DATA) + +# WIP: goals for releasing the extension... + +EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") + +../release/$(EXTENSION).control: $(EXTENSION).control + cp $< $@ + +release: ../release/$(EXTENSION).control + cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql + # pending: create upgrade/downgrade scripts, + # commit, push, tag.... diff --git a/pg/README.md b/src/pg/README.md similarity index 100% rename from pg/README.md rename to src/pg/README.md diff --git a/pg/crankshaft.control b/src/pg/crankshaft.control similarity index 100% rename from pg/crankshaft.control rename to src/pg/crankshaft.control diff --git a/pg/sql/0.0.1/00_header.sql b/src/pg/sql/00_header.sql similarity index 100% rename from pg/sql/0.0.1/00_header.sql rename to src/pg/sql/00_header.sql diff --git a/pg/sql/0.0.1/01_random_seeds.sql b/src/pg/sql/01_random_seeds.sql similarity index 100% rename from pg/sql/0.0.1/01_random_seeds.sql rename to src/pg/sql/01_random_seeds.sql diff --git a/pg/sql/0.0.1/02_moran.sql b/src/pg/sql/02_moran.sql similarity index 100% rename from pg/sql/0.0.1/02_moran.sql rename to src/pg/sql/02_moran.sql diff --git a/pg/sql/0.0.1/03_overlap_sum.sql b/src/pg/sql/03_overlap_sum.sql similarity index 100% rename from pg/sql/0.0.1/03_overlap_sum.sql rename to src/pg/sql/03_overlap_sum.sql diff --git a/pg/sql/0.0.1/04_dot_density.sql b/src/pg/sql/04_dot_density.sql similarity index 100% rename from pg/sql/0.0.1/04_dot_density.sql rename to src/pg/sql/04_dot_density.sql diff --git a/pg/sql/0.0.1/90_permissions.sql b/src/pg/sql/90_permissions.sql similarity index 100% rename from pg/sql/0.0.1/90_permissions.sql rename to src/pg/sql/90_permissions.sql diff --git a/pg/test/0.0.1/results/01_install_test.out b/src/pg/test/expected/01_install_test.out similarity index 75% rename from pg/test/0.0.1/results/01_install_test.out rename to src/pg/test/expected/01_install_test.out index c14537c..e40d267 100644 --- a/pg/test/0.0.1/results/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -3,4 +3,4 @@ CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; CREATE EXTENSION cartodb; -- Install the extension -CREATE EXTENSION crankshaft; +CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/pg/test/0.0.1/expected/02_moran_test.out b/src/pg/test/expected/02_moran_test.out similarity index 100% rename from pg/test/0.0.1/expected/02_moran_test.out rename to src/pg/test/expected/02_moran_test.out diff --git a/pg/test/0.0.1/expected/03_overlap_sum_test.out b/src/pg/test/expected/03_overlap_sum_test.out similarity index 100% rename from pg/test/0.0.1/expected/03_overlap_sum_test.out rename to src/pg/test/expected/03_overlap_sum_test.out diff --git a/pg/test/0.0.1/expected/04_dot_density_test.out b/src/pg/test/expected/04_dot_density_test.out similarity index 100% rename from pg/test/0.0.1/expected/04_dot_density_test.out rename to src/pg/test/expected/04_dot_density_test.out diff --git a/pg/test/fixtures/polyg_values.sql b/src/pg/test/fixtures/polyg_values.sql similarity index 100% rename from pg/test/fixtures/polyg_values.sql rename to src/pg/test/fixtures/polyg_values.sql diff --git a/pg/test/fixtures/ppoints.sql b/src/pg/test/fixtures/ppoints.sql similarity index 100% rename from pg/test/fixtures/ppoints.sql rename to src/pg/test/fixtures/ppoints.sql diff --git a/pg/test/fixtures/ppoints2.sql b/src/pg/test/fixtures/ppoints2.sql similarity index 100% rename from pg/test/fixtures/ppoints2.sql rename to src/pg/test/fixtures/ppoints2.sql diff --git a/pg/test/0.0.1/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql similarity index 75% rename from pg/test/0.0.1/sql/01_install_test.sql rename to src/pg/test/sql/01_install_test.sql index 54117e5..fc3ea80 100644 --- a/pg/test/0.0.1/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -4,4 +4,4 @@ CREATE EXTENSION postgis; CREATE EXTENSION cartodb; -- Install the extension -CREATE EXTENSION crankshaft; +CREATE EXTENSION crankshaft VERSION 'dev'; diff --git a/pg/test/0.0.1/sql/02_moran_test.sql b/src/pg/test/sql/02_moran_test.sql similarity index 100% rename from pg/test/0.0.1/sql/02_moran_test.sql rename to src/pg/test/sql/02_moran_test.sql diff --git a/pg/test/0.0.1/sql/03_overlap_sum_test.sql b/src/pg/test/sql/03_overlap_sum_test.sql similarity index 100% rename from pg/test/0.0.1/sql/03_overlap_sum_test.sql rename to src/pg/test/sql/03_overlap_sum_test.sql diff --git a/pg/test/0.0.1/sql/04_dot_density_test.sql b/src/pg/test/sql/04_dot_density_test.sql similarity index 100% rename from pg/test/0.0.1/sql/04_dot_density_test.sql rename to src/pg/test/sql/04_dot_density_test.sql diff --git a/pg/test/0.0.1/sql/90_permissions.sql b/src/pg/test/sql/90_permissions.sql similarity index 100% rename from pg/test/0.0.1/sql/90_permissions.sql rename to src/pg/test/sql/90_permissions.sql diff --git a/python/.gitignore b/src/py/.gitignore similarity index 54% rename from python/.gitignore rename to src/py/.gitignore index 0d20b64..fb50fe5 100644 --- a/python/.gitignore +++ b/src/py/.gitignore @@ -1 +1,2 @@ *.pyc +dev/ diff --git a/src/py/Makefile b/src/py/Makefile new file mode 100644 index 0000000..27d1446 --- /dev/null +++ b/src/py/Makefile @@ -0,0 +1,9 @@ +# Install the package locally for development +install: + virtualenv dev + ./dev/bin/pip install ./crankshaft --upgrade + ./dev/bin/pip install nose + +# Test develpment install +testinstalled: + ./dev/bin/nosetests crankshaft/test/ diff --git a/python/README.md b/src/py/README.md similarity index 100% rename from python/README.md rename to src/py/README.md diff --git a/python/crankshaft/crankshaft/__init__.py b/src/py/crankshaft/crankshaft/__init__.py similarity index 100% rename from python/crankshaft/crankshaft/__init__.py rename to src/py/crankshaft/crankshaft/__init__.py diff --git a/python/crankshaft/crankshaft/clustering/__init__.py b/src/py/crankshaft/crankshaft/clustering/__init__.py similarity index 100% rename from python/crankshaft/crankshaft/clustering/__init__.py rename to src/py/crankshaft/crankshaft/clustering/__init__.py diff --git a/python/crankshaft/crankshaft/clustering/moran.py b/src/py/crankshaft/crankshaft/clustering/moran.py similarity index 100% rename from python/crankshaft/crankshaft/clustering/moran.py rename to src/py/crankshaft/crankshaft/clustering/moran.py diff --git a/python/crankshaft/crankshaft/random_seeds.py b/src/py/crankshaft/crankshaft/random_seeds.py similarity index 100% rename from python/crankshaft/crankshaft/random_seeds.py rename to src/py/crankshaft/crankshaft/random_seeds.py diff --git a/python/crankshaft/setup.py b/src/py/crankshaft/setup.py similarity index 100% rename from python/crankshaft/setup.py rename to src/py/crankshaft/setup.py diff --git a/python/crankshaft/test/fixtures/moran.json b/src/py/crankshaft/test/fixtures/moran.json similarity index 100% rename from python/crankshaft/test/fixtures/moran.json rename to src/py/crankshaft/test/fixtures/moran.json diff --git a/python/crankshaft/test/fixtures/neighbors.json b/src/py/crankshaft/test/fixtures/neighbors.json similarity index 100% rename from python/crankshaft/test/fixtures/neighbors.json rename to src/py/crankshaft/test/fixtures/neighbors.json diff --git a/python/crankshaft/test/helper.py b/src/py/crankshaft/test/helper.py similarity index 100% rename from python/crankshaft/test/helper.py rename to src/py/crankshaft/test/helper.py diff --git a/python/crankshaft/test/mock_plpy.py b/src/py/crankshaft/test/mock_plpy.py similarity index 100% rename from python/crankshaft/test/mock_plpy.py rename to src/py/crankshaft/test/mock_plpy.py diff --git a/python/crankshaft/test/test_clustering_moran.py b/src/py/crankshaft/test/test_clustering_moran.py similarity index 100% rename from python/crankshaft/test/test_clustering_moran.py rename to src/py/crankshaft/test/test_clustering_moran.py From 8e972128eb90105dfd4cdabda5b0bb2c2e3519e4 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 Mar 2016 15:00:50 +0100 Subject: [PATCH 02/22] Modify sql code to user the python virtualenv --- src/pg/sql/01_py.sql | 18 ++++++++++++++++++ ...01_random_seeds.sql => 02_random_seeds.sql} | 1 + src/pg/sql/{02_moran.sql => 10_moran.sql} | 2 ++ .../{03_overlap_sum.sql => 20_overlap_sum.sql} | 0 .../{04_dot_density.sql => 30_dot_density.sql} | 0 5 files changed, 21 insertions(+) create mode 100644 src/pg/sql/01_py.sql rename src/pg/sql/{01_random_seeds.sql => 02_random_seeds.sql} (80%) rename src/pg/sql/{02_moran.sql => 10_moran.sql} (89%) rename src/pg/sql/{03_overlap_sum.sql => 20_overlap_sum.sql} (100%) rename src/pg/sql/{04_dot_density.sql => 30_dot_density.sql} (100%) diff --git a/src/pg/sql/01_py.sql b/src/pg/sql/01_py.sql new file mode 100644 index 0000000..f1eaa31 --- /dev/null +++ b/src/pg/sql/01_py.sql @@ -0,0 +1,18 @@ +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + # activate virtualenv + # TODO: parameterize with environment variables or something + venv_path = '/home/ubuntu/crankshaft/src/py/dev' + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), + dict(__file__=activate_path)) + + # import something from virtualenv + # from crankshaft import random_seeds + + # do some stuff + # random_seeds.set_random_seeds(123) + # plpy.notice('here we are') +$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/01_random_seeds.sql b/src/pg/sql/02_random_seeds.sql similarity index 80% rename from src/pg/sql/01_random_seeds.sql rename to src/pg/sql/02_random_seeds.sql index 2b62be3..9a0cca6 100644 --- a/src/pg/sql/01_random_seeds.sql +++ b/src/pg/sql/02_random_seeds.sql @@ -4,6 +4,7 @@ CREATE OR REPLACE FUNCTION _cdb_random_seeds (seed_value INTEGER) RETURNS VOID AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft import random_seeds random_seeds.set_random_seeds(seed_value) $$ LANGUAGE plpythonu; diff --git a/src/pg/sql/02_moran.sql b/src/pg/sql/10_moran.sql similarity index 89% rename from src/pg/sql/02_moran.sql rename to src/pg/sql/10_moran.sql index d061b45..49c70c2 100644 --- a/src/pg/sql/02_moran.sql +++ b/src/pg/sql/10_moran.sql @@ -11,6 +11,7 @@ CREATE OR REPLACE FUNCTION w_type TEXT DEFAULT 'knn') RETURNS TABLE (moran FLOAT, quads TEXT, significance FLOAT, ids INT) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local # TODO: use named parameters or a dictionary return moran_local(t, attr, significance, num_ngbrs, permutations, geom_column, id_col, w_type) @@ -29,6 +30,7 @@ CREATE OR REPLACE FUNCTION w_type TEXT DEFAULT 'knn') RETURNS TABLE(moran FLOAT, quads TEXT, significance FLOAT, ids INT, y numeric) AS $$ + plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_activate_py()') from crankshaft.clustering import moran_local_rate # TODO: use named parameters or a dictionary return moran_local_rate(t, numerator, denominator, significance, num_ngbrs, permutations, geom_column, id_col, w_type) diff --git a/src/pg/sql/03_overlap_sum.sql b/src/pg/sql/20_overlap_sum.sql similarity index 100% rename from src/pg/sql/03_overlap_sum.sql rename to src/pg/sql/20_overlap_sum.sql diff --git a/src/pg/sql/04_dot_density.sql b/src/pg/sql/30_dot_density.sql similarity index 100% rename from src/pg/sql/04_dot_density.sql rename to src/pg/sql/30_dot_density.sql From 1810f02242c494a5ecb7f1ab726e732cbecf65cf Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 Mar 2016 15:03:17 +0100 Subject: [PATCH 03/22] Use SciPy from system package python-scipy --- README.md | 1 + src/py/Makefile | 6 +++--- src/py/crankshaft/setup.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 5a12c2b..c8869e7 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. ## Requirements * pip, virtualenv, PostgreSQL +* python-scipy system package # Working Process diff --git a/src/py/Makefile b/src/py/Makefile index 27d1446..16be269 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,8 +1,8 @@ # Install the package locally for development install: - virtualenv dev - ./dev/bin/pip install ./crankshaft --upgrade - ./dev/bin/pip install nose + virtualenv --system-site-packages dev + ./dev/bin/pip install -I ./crankshaft + ./dev/bin/pip install -I nose # Test develpment install testinstalled: diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index c0f8c50..aaa76a6 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -40,9 +40,9 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. - install_requires=['pysal==1.11.0','numpy==1.6.1','scipy==0.17.0'], + install_requires=['pysal==1.9.1'], - requires=['pysal', 'numpy'], + requires=['pysal', 'numpy' ], test_suite='test' ) From 0056f411b50be904c57c9d48ffd8a4ac886f5bc8 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 9 Mar 2016 19:04:21 +0100 Subject: [PATCH 04/22] Set the path to virtualenvs in the Makefile Also, version the virtualenv --- src/pg/Makefile | 11 ++++++++- src/pg/sql/01_py.sql | 18 --------------- src/pg/sql/01_version.sql | 12 ++++++++++ src/pg/sql/02_py.sql | 23 +++++++++++++++++++ ...2_random_seeds.sql => 03_random_seeds.sql} | 0 5 files changed, 45 insertions(+), 19 deletions(-) delete mode 100644 src/pg/sql/01_py.sql create mode 100644 src/pg/sql/01_version.sql create mode 100644 src/pg/sql/02_py.sql rename src/pg/sql/{02_random_seeds.sql => 03_random_seeds.sql} (100%) diff --git a/src/pg/Makefile b/src/pg/Makefile index 4ff713d..ded7121 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -5,6 +5,8 @@ # and make it available to PostgreSQL # PGUSER=postgres make installcheck -- test the 'dev' extension +SED = sed + EXTENSION = crankshaft DATA = $(EXTENSION)--dev.sql \ @@ -14,8 +16,14 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) +VIRTUALENV_PATH = $(realpath ../py/) +ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) + +REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ + -e 's/@@VIRTUALENV_PATH@@/$(ESC_VIRVIRTUALENV_PATH)/g' + $(DATA): $(SOURCES_DATA) - cat $(SOURCES_DATA_DIR)/*.sql > $@ + $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@ TEST_DIR = test REGRESS = $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))) @@ -28,6 +36,7 @@ include $(PGXS) # This seems to be needed at least for PG 9.3.11 all: $(DATA) + # WIP: goals for releasing the extension... EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") diff --git a/src/pg/sql/01_py.sql b/src/pg/sql/01_py.sql deleted file mode 100644 index f1eaa31..0000000 --- a/src/pg/sql/01_py.sql +++ /dev/null @@ -1,18 +0,0 @@ --- Use the crankshaft python module -CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() -RETURNS VOID -AS $$ - # activate virtualenv - # TODO: parameterize with environment variables or something - venv_path = '/home/ubuntu/crankshaft/src/py/dev' - activate_path = venv_path + '/bin/activate_this.py' - exec(open(activate_path).read(), - dict(__file__=activate_path)) - - # import something from virtualenv - # from crankshaft import random_seeds - - # do some stuff - # random_seeds.set_random_seeds(123) - # plpy.notice('here we are') -$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/01_version.sql b/src/pg/sql/01_version.sql new file mode 100644 index 0000000..90e6123 --- /dev/null +++ b/src/pg/sql/01_version.sql @@ -0,0 +1,12 @@ +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_crankshaft_version() +RETURNS text AS $$ + SELECT '@@VERSION@@'::text; +$$ language 'sql' IMMUTABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION cdb_crankshaft_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; +$$ language 'sql' IMMUTABLE STRICT; diff --git a/src/pg/sql/02_py.sql b/src/pg/sql/02_py.sql new file mode 100644 index 0000000..a233a0a --- /dev/null +++ b/src/pg/sql/02_py.sql @@ -0,0 +1,23 @@ +CREATE OR REPLACE FUNCTION _cdb_crankshaft_virtualenvs_path() +RETURNS text +AS $$ + BEGIN + -- RETURN '/opt/virtualenvs/crankshaft'; + RETURN '@@VIRTUALENV_PATH@@'; + END; +$$ language plpgsql IMMUTABLE STRICT; + +-- Use the crankshaft python module +CREATE OR REPLACE FUNCTION _cdb_crankshaft_activate_py() +RETURNS VOID +AS $$ + import os + # plpy.notice('%',str(os.environ)) + # activate virtualenv + crankshaft_version = plpy.execute('SELECT cdb_crankshaft.cdb_crankshaft_internal_version()')[0]['cdb_crankshaft_internal_version'] + base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] + default_venv_path = os.path.join(base_path, crankshaft_version) + venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) + activate_path = venv_path + '/bin/activate_this.py' + exec(open(activate_path).read(), dict(__file__=activate_path)) +$$ LANGUAGE plpythonu; diff --git a/src/pg/sql/02_random_seeds.sql b/src/pg/sql/03_random_seeds.sql similarity index 100% rename from src/pg/sql/02_random_seeds.sql rename to src/pg/sql/03_random_seeds.sql From b754ffe42a2720fd67ace6ab8ceea0784423e4b6 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 9 Mar 2016 18:51:04 +0100 Subject: [PATCH 05/22] Add info about python dependencies --- src/py/README.md | 90 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/src/py/README.md b/src/py/README.md index f342bf2..03926c3 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -7,3 +7,93 @@ cd crankshaft nosetests test/ ``` + +## Notes about python dependencies +* This extension is targeted at production databases. Therefore certain restrictions must be assumed about the production environment vs other experimental environments. +* We're using `pip` and `virtualenv` to generate a suitable isolated environment for python code that has all the dependencies +* Every dependency should be: + - Added to the `setup.py` file + - Installed through it + - Tested, when they have a test suite. + - Fixed in the `requirements.txt` +* At present we use Python version 2.7.3 + +--- + +### Sample session with virtualenv +#### Create and use a virtual env + + # Create the virtual environment for python + $ virtualenv myenv + + # Activate the virtualenv + $ source myenv/bin/activate + + # Install all the requirements + # expect this to take a while, as it will trigger a few compilations + (myenv) $ pip install -r requirements.txt + + # Add a new pip to the party + (myenv) $ pip install pandas + +#### Test the libraries with that virtual env +##### Test numpy library dependency: + + import numpy + numpy.test('full') + +output: +``` +====================================================================== +ERROR: test_multiarray.TestNewBufferProtocol.test_relaxed_strides +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/nose/case.py", line 197, in runTest + self.test(*self.arg) + File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/numpy/core/tests/test_multiarray.py", line 5366, in test_relaxed_strides + fd.write(c.data) +TypeError: 'buffer' does not have the buffer interface + +---------------------------------------------------------------------- +Ran 6153 tests in 84.561s + +FAILED (KNOWNFAIL=3, SKIP=5, errors=1) +Out[2]: +``` + +NOTE: this is expected to fail with Python 2.7.3, which is the version embedded in our postgresql installation + + +##### Run scipy tests + + import scipy + scipy.test('full') + +Output: +``` +Ran 21562 tests in 321.610s + +OK (KNOWNFAIL=130, SKIP=1840) +Out[2]: +``` +Ok, this looks good... + +##### Testing pysal +See [http://pysal.readthedocs.org/en/latest/developers/testing.html] + + import pysal + import nose + nose.runmodule('pysal') + +``` +Ran 537 tests in 42.182s + +FAILED (errors=48, failures=17) +An exception has occurred, use %tb to see the full traceback. +``` + +This doesn't look good... Taking a deeper look at the failures, many have the `IOError: [Errno 2] No such file or directory: 'streets.shp'` + +In the source code, there's the following [config](https://github.com/pysal/pysal/blob/master/setup.cfg) that seems to be missing in the pip package. By copying it to `lib/python2.7/site-packages` within the environment, it goes down to 17 failures. + +The remaining failures don't look good. I see two types: precision calculation errors and arrays/matrices missing 1 element when comparing... TODO: FIX this From 0206cc6c4457e6ca91f7a2cee1dee89e9df8b40f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Thu, 10 Mar 2016 19:13:46 +0100 Subject: [PATCH 06/22] Update documentation --- CONTRIBUTING.md | 84 ---------------------------------- README.md | 77 ++++++++++++++++++------------- src/pg/Makefile | 2 - src/py/README.md | 117 ++++++++++++++++++++++++++++++----------------- 4 files changed, 118 insertions(+), 162 deletions(-) delete mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 63670ca..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,84 +0,0 @@ -# Contributing guide - -## How to add new functions - -Try to put as little logic in the SQL extension as possible and -just use it as a wrapper to the Python module functionality. - -Once a function is defined it should never change its signature in subsequent -versions. To change a function's signature a new function with a different -name must be created. - -### Version numbers - -The version of both the SQL extension and the Python package shall -follow the [Semantic Versioning 2.0](http://semver.org/) guidelines: - -* When backwards incompatibility is introduced the major number is incremented -* When functionally is added (in a backwards-compatible manner) the minor number - is incremented -* When only fixes are introduced (backwards-compatible) the patch number is - incremented - -### Python Package - -... - -### SQL Extension - -* Generate a **new subfolder version** for `sql` and `test` folders to define - the new functions and tests - - Use symlinks to avoid file duplication between versions that don't update them - - Add new files or modify copies of the old files to add new functions or - modify existing functions (remember to rename a function if the signature - changes) - - Add or modify the corresponding documentation files in the `doc` folder. - Since we expect to have highly technical functions here, an extense - background explanation would be of great help to users of this extension. - - Create tests for the new functions/behaviour - -* Generate the **upgrade and downgrade files** for the extension - -* Update the control file and the Makefile to generate the complete SQL - file for the new created version. After running `make` a new - file `crankshaft--X.Y.Z.sql` will be created for the current version. - Additional files for migrating to/from the previous version A.B.Z should be - created: - - `crankshaft--X.Y.Z--A.B.C.sql` - - `crankshaft--A.B.C--X.Y.Z.sql` - All these new files must be added to git and pushed. - -* Update the public docs! ;-) - -## Conventions - -# SQL - -Use snake case (i.e. `snake_case` and not `CamelCase`) for all -functions. Prefix functions intended for public use with `cdb_` -and private functions (to be used only internally inside -the extension) with `_cdb_`. - -# Python - -... - -## Testing - -Running just the Python tests: - -``` -(cd python && make test) -``` - -Installing the Extension and running just the PostgreSQL tests: - -``` -(cd pg && sudo make install && PGUSER=postgres make installcheck) -``` - -Installing and testing everything: - -``` -sudo make install && PGUSER=postgres make testinstalled -``` diff --git a/README.md b/README.md index c8869e7..c9dd529 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,32 @@ CartoDB Spatial Analysis extension for PostgreSQL. * *src* source code * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code -* *release* reselesed versions +* *release* reseleased versions ## Requirements * pip, virtualenv, PostgreSQL -* python-scipy system package +* python-scipy system package (see src/py/README.md) # Working Process ## Development Work in `src/pg/sql`, `src/py/crankshaft`; -use topic branch. +use a topic branch. See src/py/README.md +for the procedure to work with the Python local environment. + +Take into account: + +* Always remember to add tests for any new functionality + documentation. +* Add or modify the corresponding documentation files in the `doc` folder. + Since we expect to have highly technical functions here, an extense + background explanation would be of great help to users of this extension. +* Convention: Use snake case (i.e. `snake_case` and not `CamelCase`) for all + functions. Prefix functions intended for public use with `cdb_` + and private functions (to be used only internally inside + the extension) with `_cdb_`. Update local installation with `sudo make install` (this will update the 'dev' version of the extension in 'src/pg/') @@ -42,50 +55,48 @@ should be dropped manually before the update. If the extension has not previously been installed in a database we can: -Add tests... - * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` -Test +Once the tests are succeeding a new Pull-Request can be created. +CI-tests must be checked to be successfull. + +Before merging a topic branch peer code reviewing of the code is a must. -Commit, push, create PR, wait for CI tests, CR, ... ## Release -To release current development version -(working directory should be clean in dev branch) +The release process of a new version of the extension +shall by performed by the designated *Release Manager*. -(process to be gradually automated) +Note that we expect to gradually automate this process. -For backwards compatible changes (no return value, num of arguments, etc. changes...) -new version number increasing either patch level (no new functionality) -or minor level (new functionality) => 'X.Y.Z'. -Update version in src/pg/crankshaft.control -Copy release/crankshaft--current.sql to release/crankshaft--X.Y.Z.sql -Prepare incremental downgrade, upgrade scripts.... +Having checkout the topic branch of the PR to be released: -Python: ... +The version number in `pg/cranckshaft.control` must first be updated. +To do so [Semantic Versioning 2.0](http://semver.org/) is in order. -Install the new release +We now will explain the process for the case of backwards-compatible +releases (updating the minor or patch version numbers). -`make install-release` +TODO: document the complex case of major releases. -Test the new release +The next command must be executed to produce the main installation +script for the new release, `release/cranckshaft--X.Y.Z.sql`. -`make test-release` +``` +make release +``` -Push the release +Then, the release manager shall produce upgrade and downgrade scripts +to migrate to/from the previous release. In the case of minor/patch +releases this simply consist in extracting the functions that have changed +and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` +file. -Wait for CI tests +TODO: configure the local enviroment to be used by the release; +currently should be directory `src/py/X.Y.Z`, but this must be fixed; +a possibility to explore is to use the `cdb_conf` table. -Merge into master +TODO: testing procedure for the new release -Deploy: install extension and python to production hosts, -update extension in databases (limited to team users, data observatory, ...) - -Release manager role: ... - -.sql release scripts -commit -tests: staging.... -merge, tag, deploy... +TODO: push, merge, tag, deploy procedures. diff --git a/src/pg/Makefile b/src/pg/Makefile index ded7121..ed65eba 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -46,5 +46,3 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/def release: ../release/$(EXTENSION).control cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql - # pending: create upgrade/downgrade scripts, - # commit, push, tag.... diff --git a/src/py/README.md b/src/py/README.md index 03926c3..d55b7d7 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -20,80 +20,111 @@ nosetests test/ --- -### Sample session with virtualenv +We have two possible approaches being considered as to how manage +the Python virtual environment: using a pure virtual enviroment +or combine it with some system packages that include depencencies +for the *hard-to-compile* packages (and pin them in somewhat old versions). + +### Alternative A: pure virtual environment + +In this case we will install all the packages needed in the +virtual environment. +This will involve, specially for the numerical packages compiling +and linking code that uses a number of third party libraries, +and requires having theses depencencies solved for the production +environments. + #### Create and use a virtual env +We'll use a virtual enviroment directory `dev` +under the `src/pg` directory. + # Create the virtual environment for python - $ virtualenv myenv + $ virtualenv dev # Activate the virtualenv - $ source myenv/bin/activate + $ source dev/bin/activate # Install all the requirements # expect this to take a while, as it will trigger a few compilations - (myenv) $ pip install -r requirements.txt + (dev) $ pip install -r requirements.txt # Add a new pip to the party - (myenv) $ pip install pandas + (dev) $ pip install pandas #### Test the libraries with that virtual env + ##### Test numpy library dependency: import numpy numpy.test('full') -output: -``` -====================================================================== -ERROR: test_multiarray.TestNewBufferProtocol.test_relaxed_strides ----------------------------------------------------------------------- -Traceback (most recent call last): - File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/nose/case.py", line 197, in runTest - self.test(*self.arg) - File "/home/ubuntu/www/crankshaft/src/py/dev2/lib/python2.7/site-packages/numpy/core/tests/test_multiarray.py", line 5366, in test_relaxed_strides - fd.write(c.data) -TypeError: 'buffer' does not have the buffer interface - ----------------------------------------------------------------------- -Ran 6153 tests in 84.561s - -FAILED (KNOWNFAIL=3, SKIP=5, errors=1) -Out[2]: -``` - -NOTE: this is expected to fail with Python 2.7.3, which is the version embedded in our postgresql installation - - ##### Run scipy tests import scipy scipy.test('full') -Output: -``` -Ran 21562 tests in 321.610s - -OK (KNOWNFAIL=130, SKIP=1840) -Out[2]: -``` -Ok, this looks good... - ##### Testing pysal + See [http://pysal.readthedocs.org/en/latest/developers/testing.html] +This will require putting this into `dev/lib/python2.7/site-packages/setup.cfg`: + +``` +[nosetests] +ignore-files=collection +exclude-dir=pysal/contrib + +[wheel] +universal=1 +``` + +And copying some files before executing the tests: +(we'll use a temporary directory from where the tests will be executed because +some tests expect some files in the current directory). Next must be executed +from + +``` +cp dev/lib/python2.7/site-packages/pysal/examples/geodanet/* dev/local/lib/python2.7/site-packages/pysal/examples +mkdir -p test_tmp && cd test_tmp && cp ../dev/lib/python2.7/site-packages/pysal/examples/geodanet/* ./ +``` + +Then, execute the tests with: + import pysal import nose nose.runmodule('pysal') -``` -Ran 537 tests in 42.182s -FAILED (errors=48, failures=17) -An exception has occurred, use %tb to see the full traceback. +### Alternative B: using some packaged modules + +This option avoids troublesome compilations/linkings, at the cost +of freezing some module versions as available in system packages, +namely numpy 1.6.1 and scipy 0.9.0. (in turn, this implies +the most recent version of PySAL we can use is 1.9.1) + + +TODO: to use this alternative the python-scipy package must be +installed (this will have to be included in server provisioning) + +``` +apt-get install -y python-scipy ``` -This doesn't look good... Taking a deeper look at the failures, many have the `IOError: [Errno 2] No such file or directory: 'streets.shp'` +#### Create and use a virtual env -In the source code, there's the following [config](https://github.com/pysal/pysal/blob/master/setup.cfg) that seems to be missing in the pip package. By copying it to `lib/python2.7/site-packages` within the environment, it goes down to 17 failures. +We'll use a `dev` enviroment as before, but will configure it to +use also system modules. -The remaining failures don't look good. I see two types: precision calculation errors and arrays/matrices missing 1 element when comparing... TODO: FIX this + + # Create the virtual environment for python + $ virtualenv --system-site-packages dev + + # Activate the virtualenv + $ source dev/bin/activate + + # Install all the requirements + # expect this to take a while, as it will trigger a few compilations + (dev) $ pip install -I ./crankshaft + +Then we can proceed to testing as in Alternative A. From e801c9cb60cdb2774238da313bdbfd6c58bb21c7 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 15 Mar 2016 18:48:46 +0100 Subject: [PATCH 07/22] Release tasks using release-specific virtual environments Refine the development process and define the procedure for releasing new versions. --- .gitignore | 1 + Makefile | 26 ++++++++++-- NEWS.md | 3 ++ README.md | 87 +++++++++++++++++++++++++++++--------- TODO.md | 9 ---- release/.gitignore | 0 release/python/.gitignore | 0 src/pg/Makefile | 59 +++++++++++++++++++++----- src/pg/README.md | 7 --- src/py/.gitignore | 1 - src/py/Makefile | 11 ++--- src/py/crankshaft/setup.py | 2 +- 12 files changed, 149 insertions(+), 57 deletions(-) create mode 100644 .gitignore create mode 100644 NEWS.md delete mode 100644 TODO.md create mode 100644 release/.gitignore create mode 100644 release/python/.gitignore delete mode 100644 src/pg/README.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..37819f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +envs/ diff --git a/Makefile b/Makefile index 5cae2cb..43b8480 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,31 @@ PYP_DIR = src/py .PHONY: install .PHONY: run_tests +.PHONY: release +.PHONY: deploy + +# Generate and install developmet versions of the extension +# and python package. +# The extension is named 'dev' with a 'current' alias for easily upgrading. +# The Python package is installed in a virtual environment envs/dev/ +# Requires sudo. install: $(MAKE) -C $(PYP_DIR) install $(MAKE) -C $(EXT_DIR) install -testinstalled: - $(MAKE) -C $(PYP_DIR) testinstalled - $(MAKE) -C $(EXT_DIR) installcheck +# Run the tests for the installed development extension and +# python package +test: + $(MAKE) -C $(PYP_DIR) test + $(MAKE) -C $(EXT_DIR) test + +# Generate a new release into release +release: + $(MAKE) -C $(EXT_DIR) release + +# Install the current release. +# The Python package is installed in a virtual environment envs/X.Y.Z/ +# Requires sudo. +deploy: + $(MAKE) -C $(EXT_DIR) deploy diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..201da51 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,3 @@ +0.0.1 (2016-03-15) +------------------ +* Preliminar release diff --git a/README.md b/README.md index c9dd529..75f7c2e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. * - *src/pg* contains the PostgreSQL extension source code * - *src/py* Python module source code * *release* reseleased versions +* *env* base directory for Python virtual environments ## Requirements @@ -17,16 +18,24 @@ CartoDB Spatial Analysis extension for PostgreSQL. # Working Process +We use the default `develop` branch as the basis for development. +This branch and `master` are maintained by the *Release Manager*. +The `master` branch is used to merge and tag releases to be +deployed in production. + +In addition to these two permanent branches, temporal *topic* +branches will be used for all modifications. + ## Development -Work in `src/pg/sql`, `src/py/crankshaft`; -use a topic branch. See src/py/README.md -for the procedure to work with the Python local environment. +A topic branch should be created out of the `develop` branch +and be used for the development process; see src/py/README.md +Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. Take into account: -* Always remember to add tests for any new functionality - documentation. +* Always remember to add tests (`src/pg/test`, `src/py/crankshaft/test`) + for any new functionality. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. @@ -35,12 +44,22 @@ Take into account: and private functions (to be used only internally inside the extension) with `_cdb_`. -Update local installation with `sudo make install` -(this will update the 'dev' version of the extension in 'src/pg/') +Update the local development installation with `sudo make install`. +This will update the 'dev' version of the extension in 'src/pg/' and +make it available to PostgreSQL. +It will also install the python package (crankshaft) in a virtual +environment `env/dev`. -Run the tests with `PGUSER=postgres make test` +Run the tests with `make test` -Update extension in working database with +To use the python extension for custom tests, activate the virtual +environment with: + +``` +source envs/dev/bin/activate +``` + +Update extension in a working database with: * `ALTER EXTENSION crankshaft VERSION TO 'current';` `ALTER EXTENSION crankshaft VERSION TO 'dev';` @@ -57,31 +76,37 @@ we can: * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` +Note: the development extension uses the development pyhton virtual +environment automatically. + Once the tests are succeeding a new Pull-Request can be created. -CI-tests must be checked to be successfull. - -Before merging a topic branch peer code reviewing of the code is a must. +CI-tests must be checked to be successful. +Before proceeding to the release process peer code reviewing of the code is a must. ## Release The release process of a new version of the extension shall by performed by the designated *Release Manager*. -Note that we expect to gradually automate this process. +Note that we expect to gradually automate more of this process. -Having checkout the topic branch of the PR to be released: +Having checked the topic branch of the PR to be released it shall be +merged back into the `develop` branch to prepare the new release. The version number in `pg/cranckshaft.control` must first be updated. To do so [Semantic Versioning 2.0](http://semver.org/) is in order. +Thew `NEWS.md` will be updated. + We now will explain the process for the case of backwards-compatible releases (updating the minor or patch version numbers). TODO: document the complex case of major releases. The next command must be executed to produce the main installation -script for the new release, `release/cranckshaft--X.Y.Z.sql`. +script for the new release, `release/cranckshaft--X.Y.Z.sql` and +also to copy the python package to `release/python/X.Y.Z/crankshaft`. ``` make release @@ -93,10 +118,32 @@ releases this simply consist in extracting the functions that have changed and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` file. -TODO: configure the local enviroment to be used by the release; -currently should be directory `src/py/X.Y.Z`, but this must be fixed; -a possibility to explore is to use the `cdb_conf` table. +The new release can be deployed for staging/smoke tests with this command: -TODO: testing procedure for the new release +``` +sudo make deploy +``` -TODO: push, merge, tag, deploy procedures. +This will make the 'X.Y.Z' version of the extension to PostgreSQL. +The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z` + +It can be activated with: + +``` +source envs/X.Y.Z/bin/activate +``` + +But note that this is needed only for using the package directly; +the 'X.Y.Z' version of the extension will automatically use the +python package from this virtual environment. + +The `sudo make deploy` operation can be also used for installing +the new version after it has been released. + +TODO: testing procedure for the new release. + +TODO: procedure for staging deployment. + +TODO: procedure for merging to master, tagging and deploying +in production. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 8a708e2..0000000 --- a/TODO.md +++ /dev/null @@ -1,9 +0,0 @@ -* [x] Support versioning -* [x] Test use of `plpy` from python Package -* [x] Add `pysal` etc. dependencies -* [x] Define documentation practices (general, per extension/package?) -* [x] Add initial function set (WIP) -* Unify style of function comments -* [x] Add integration tests -* Make target to open a new version development (create symlinks, etc.) -* [x] Should add cartodb ext. as a dependency? diff --git a/release/.gitignore b/release/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/release/python/.gitignore b/release/python/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/src/pg/Makefile b/src/pg/Makefile index ed65eba..fea7386 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -1,9 +1,13 @@ -# Generation of a new development version 'dev' (with an alias 'current' for -# updating easily by upgrading to 'current', then 'dev') - -# sudo make install -- generate the 'dev' version from current source -# and make it available to PostgreSQL -# PGUSER=postgres make installcheck -- test the 'dev' extension +# Development tasks: +# +# * install generates the control & script files into src/pg/ +# and installs then into the PostgreSQL extensions directory; +# requires sudo. In additionof the current development version +# named 'dev', an alias 'current' is generating for ease of +# update (upgrade to 'current', then to 'dev'). +# the python module is installed in a virtualenv in envs/dev/ +# * test runs the tests for the currently generated Development +# extension. SED = sed @@ -16,7 +20,7 @@ DATA = $(EXTENSION)--dev.sql \ SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) -VIRTUALENV_PATH = $(realpath ../py/) +VIRTUALENV_PATH = $(realpath ../../envs) ESC_VIRVIRTUALENV_PATH = $(subst /,\/,$(VIRTUALENV_PATH)) REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' \ @@ -36,13 +40,46 @@ include $(PGXS) # This seems to be needed at least for PG 9.3.11 all: $(DATA) +test: export PGUSER=postgres +test: installcheck -# WIP: goals for releasing the extension... +# Release tasks +PACKAGE = crankshaft EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") -../release/$(EXTENSION).control: $(EXTENSION).control +../../release/$(EXTENSION).control: $(EXTENSION).control cp $< $@ -release: ../release/$(EXTENSION).control - cp $(EXTENSION)--dev.sql $(EXTENSION)--$(EXTVERSION).sql +# Prepare new release from the currently installed development version, +# for the current version X.Y.Z (defined in the control file) +# producing the extension script and control files in releases/ +# and the python package in releases/python/X.Y.Z/crankshaft/ +release: ../../release/$(EXTENSION).control + cp $(EXTENSION)--dev.sql ../../release/$(EXTENSION)--$(EXTVERSION).sql + mkdir -p ../../release/python/$(EXTVERSION) + cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ + $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py + +# Install the current relese into the PostgreSQL extensions directory +# and the Python package in a virtual environment envs/X.Y.Z +deploy: + $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' + $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' + # TODO: install also upgrade/downgrade files (manually generated) + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(EXTVERSION) + $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I -U ../../release/python/$(EXTVERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose + +clean-dev: + rm $(EXTNAME)--*.sql + +clean-releases: + rm -rf ../../release/python/* + rm ../../release/$(EXTNAME)--*.sql + rm ../../release/$(EXTNAME).control + +clean-environments: + rm -rf ../../envs/* + +clean-all: clean-dev clean-releases clean-environments diff --git a/src/pg/README.md b/src/pg/README.md deleted file mode 100644 index 511fdae..0000000 --- a/src/pg/README.md +++ /dev/null @@ -1,7 +0,0 @@ - -# Running the tests: - -``` -sudo make install -PGUSER=postgres make installcheck -``` diff --git a/src/py/.gitignore b/src/py/.gitignore index fb50fe5..0d20b64 100644 --- a/src/py/.gitignore +++ b/src/py/.gitignore @@ -1,2 +1 @@ *.pyc -dev/ diff --git a/src/py/Makefile b/src/py/Makefile index 16be269..ab787ee 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,9 +1,10 @@ # Install the package locally for development install: - virtualenv --system-site-packages dev - ./dev/bin/pip install -I ./crankshaft - ./dev/bin/pip install -I nose + virtualenv --system-site-packages ../../envs/dev + # source ../../envs/dev/bin/activate + ../../envs/dev/bin/pip install -I ./crankshaft + ../../envs/dev/bin/pip install -I nose # Test develpment install -testinstalled: - ./dev/bin/nosetests crankshaft/test/ +test: + ../../envs/dev/bin/nosetests crankshaft/test/ diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index aaa76a6..8d5e622 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -10,7 +10,7 @@ from setuptools import setup, find_packages setup( name='crankshaft', - version='0.0.1', + version='0.0.0', description='CartoDB Spatial Analysis Python Library', From 82a738fe40fb0ab2f4f15f3a205748b4dc07b2ff Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Tue, 15 Mar 2016 19:22:27 +0100 Subject: [PATCH 08/22] Fix make clean tasks --- src/pg/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pg/Makefile b/src/pg/Makefile index fea7386..4e8a444 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -72,12 +72,12 @@ deploy: $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose clean-dev: - rm $(EXTNAME)--*.sql + rm -f $(EXTENSION)--*.sql clean-releases: rm -rf ../../release/python/* - rm ../../release/$(EXTNAME)--*.sql - rm ../../release/$(EXTNAME).control + rm -f ../../release/$(EXTNAME)--*.sql + rm -f ../../release/$(EXTNAME).control clean-environments: rm -rf ../../envs/* From 24e4037995f61d8d7387fcdc950901080b168339 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 11:11:16 +0100 Subject: [PATCH 09/22] Fix version number of released extension script --- src/pg/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/Makefile b/src/pg/Makefile index 4e8a444..d81023c 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -55,8 +55,8 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/def # for the current version X.Y.Z (defined in the control file) # producing the extension script and control files in releases/ # and the python package in releases/python/X.Y.Z/crankshaft/ -release: ../../release/$(EXTENSION).control - cp $(EXTENSION)--dev.sql ../../release/$(EXTENSION)--$(EXTVERSION).sql +release: ../../release/$(EXTENSION).control $(SOURCES_DATA) + $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql mkdir -p ../../release/python/$(EXTVERSION) cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py From a518034e65b9e3a5ae0b2f1931f72310a851b22c Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 11:13:26 +0100 Subject: [PATCH 10/22] Fix .pyc files need not only be ignored inside src/py --- .gitignore | 1 + src/py/.gitignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 src/py/.gitignore diff --git a/.gitignore b/.gitignore index 37819f7..8d2abce 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ envs/ +*.pyc diff --git a/src/py/.gitignore b/src/py/.gitignore deleted file mode 100644 index 0d20b64..0000000 --- a/src/py/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc From d13f167d47c9dfd90198ddd239b1d7ae17a33577 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 14:38:18 +0100 Subject: [PATCH 11/22] Add RELEASE_VERSION option to make deploy Now make deploy installs by default the current version, but can be made to install any prior specific version using a environmnt varialbe RELEASE_VERSION --- Makefile | 2 ++ src/pg/Makefile | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 43b8480..738b83e 100644 --- a/Makefile +++ b/Makefile @@ -29,5 +29,7 @@ release: # Install the current release. # The Python package is installed in a virtual environment envs/X.Y.Z/ # Requires sudo. +# Use the RELEASE_VERSION environment variable to deploy a specific version: +# sudo make deploy RELEASE_VERSION=1.0.0 deploy: $(MAKE) -C $(EXT_DIR) deploy diff --git a/src/pg/Makefile b/src/pg/Makefile index d81023c..ebe7009 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -61,15 +61,17 @@ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py +RELEASE_VERSION ?= $(EXTVERSION) + # Install the current relese into the PostgreSQL extensions directory # and the Python package in a virtual environment envs/X.Y.Z deploy: $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' # TODO: install also upgrade/downgrade files (manually generated) - virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(EXTVERSION) - $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I -U ../../release/python/$(EXTVERSION)/$(PACKAGE) - $(VIRTUALENV_PATH)/$(EXTVERSION)/bin/pip install -I nose + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose clean-dev: rm -f $(EXTENSION)--*.sql From dea6e2f1a7fcf3abdf341441a445fff72f7d3436 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:40:40 +0100 Subject: [PATCH 12/22] Refactor the Makefile Separate concerns properly for each subdirectory's Makefile --- Makefile | 45 ++++++++++++++++++++++++++++++++++++++++----- Makefile.global | 6 ++++++ src/pg/Makefile | 31 ++----------------------------- src/py/Makefile | 12 ++++++++++++ 4 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 Makefile.global diff --git a/Makefile b/Makefile index 738b83e..979c2b5 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +include ./Makefile.global + EXT_DIR = src/pg PYP_DIR = src/py @@ -6,30 +8,63 @@ PYP_DIR = src/py .PHONY: release .PHONY: deploy - # Generate and install developmet versions of the extension # and python package. # The extension is named 'dev' with a 'current' alias for easily upgrading. # The Python package is installed in a virtual environment envs/dev/ # Requires sudo. -install: +install: ## Generate and install development version of the extension; requires sudo. $(MAKE) -C $(PYP_DIR) install $(MAKE) -C $(EXT_DIR) install # Run the tests for the installed development extension and # python package -test: +test: ## Run the tests for the development version of the extension $(MAKE) -C $(PYP_DIR) test $(MAKE) -C $(EXT_DIR) test # Generate a new release into release -release: +release: ## Generate a new release of the extension. Only for telease manager $(MAKE) -C $(EXT_DIR) release + $(MAKE) -C $(PYP_DIR) release # Install the current release. # The Python package is installed in a virtual environment envs/X.Y.Z/ # Requires sudo. # Use the RELEASE_VERSION environment variable to deploy a specific version: # sudo make deploy RELEASE_VERSION=1.0.0 -deploy: +deploy: ## Deploy a released extension. Only for release manager. Requires sudo. $(MAKE) -C $(EXT_DIR) deploy + $(MAKE) -C $(PYP_DIR) deploy + +# Cleanup development extension script files +clean-dev: ## clean up development extension script files + rm -f src/pg/$(EXTENSION)--*.sql + +# # Cleanup all releases +# clean-releases: +# rm -rf release/python/* +# rm -f release/$(EXTENSION)--*.sql +# rm -f release/$(EXTENSION).control + +# Cleanup current/specific version +clean-release: ## clean up current release + rm -rf release/python/$(RELEASE_VERSION) + rm -f release/$(RELEASE_VERSION)--*.sql + +# Cleanup all virtual environments +clean-environments: ## clean up all virtual environments + rm -rf envs/* + +clean-all: clean-dev clean-release clean-environments + +help: + @IFS=$$'\n' ; \ + help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//'`); \ + for help_line in $${help_lines[@]}; do \ + IFS=$$'#' ; \ + help_split=($$help_line) ; \ + help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \ + help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \ + printf "%-30s %s\n" $$help_command $$help_info ; \ + done diff --git a/Makefile.global b/Makefile.global new file mode 100644 index 0000000..77f6c69 --- /dev/null +++ b/Makefile.global @@ -0,0 +1,6 @@ +SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +EXTENSION = crankshaft +PACKAGE = crankshaft +EXTVERSION = $(shell grep default_version $(SELF_DIR)/src/pg/$(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +RELEASE_VERSION ?= $(EXTVERSION) +SED = sed diff --git a/src/pg/Makefile b/src/pg/Makefile index ebe7009..8a745c4 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -1,3 +1,5 @@ +include ../../Makefile.global + # Development tasks: # # * install generates the control & script files into src/pg/ @@ -9,10 +11,6 @@ # * test runs the tests for the currently generated Development # extension. -SED = sed - -EXTENSION = crankshaft - DATA = $(EXTENSION)--dev.sql \ $(EXTENSION)--current--dev.sql \ $(EXTENSION)--dev--current.sql @@ -45,9 +43,6 @@ test: installcheck # Release tasks -PACKAGE = crankshaft -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") - ../../release/$(EXTENSION).control: $(EXTENSION).control cp $< $@ @@ -57,31 +52,9 @@ EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/def # and the python package in releases/python/X.Y.Z/crankshaft/ release: ../../release/$(EXTENSION).control $(SOURCES_DATA) $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql - mkdir -p ../../release/python/$(EXTVERSION) - cp -r ../py/$(PACKAGE) ../../release/python/$(EXTVERSION)/ - $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py - -RELEASE_VERSION ?= $(EXTVERSION) # Install the current relese into the PostgreSQL extensions directory # and the Python package in a virtual environment envs/X.Y.Z deploy: $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' - # TODO: install also upgrade/downgrade files (manually generated) - virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) - $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose - -clean-dev: - rm -f $(EXTENSION)--*.sql - -clean-releases: - rm -rf ../../release/python/* - rm -f ../../release/$(EXTNAME)--*.sql - rm -f ../../release/$(EXTNAME).control - -clean-environments: - rm -rf ../../envs/* - -clean-all: clean-dev clean-releases clean-environments diff --git a/src/py/Makefile b/src/py/Makefile index ab787ee..90b22b8 100644 --- a/src/py/Makefile +++ b/src/py/Makefile @@ -1,3 +1,5 @@ +include ../../Makefile.global + # Install the package locally for development install: virtualenv --system-site-packages ../../envs/dev @@ -8,3 +10,13 @@ install: # Test develpment install test: ../../envs/dev/bin/nosetests crankshaft/test/ + +release: ../../release/$(EXTENSION).control $(SOURCES_DATA) + mkdir -p ../../release/python/$(EXTVERSION) + cp -r ./$(PACKAGE) ../../release/python/$(EXTVERSION)/ + $(SED) -i -r 's/version='"'"'[0-9]+\.[0-9]+\.[0-9]+'"'"'/version='"'"'$(EXTVERSION)'"'"'/g' ../../release/python/$(EXTVERSION)/$(PACKAGE)/setup.py + +deploy: + virtualenv --system-site-packages $(VIRTUALENV_PATH)/$(RELEASE_VERSION) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I -U ../../release/python/$(RELEASE_VERSION)/$(PACKAGE) + $(VIRTUALENV_PATH)/$(RELEASE_VERSION)/bin/pip install -I nose From 4ffb2c966447a60b8676e029efd32397dae14524 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:45:13 +0100 Subject: [PATCH 13/22] Review and fix the documentation --- README.md | 59 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 75f7c2e..5c52541 100644 --- a/README.md +++ b/README.md @@ -24,33 +24,43 @@ The `master` branch is used to merge and tag releases to be deployed in production. In addition to these two permanent branches, temporal *topic* -branches will be used for all modifications. +branches must be used for all modifications. ## Development -A topic branch should be created out of the `develop` branch +For any modification of crankshaft, including adding new features, +a topic branch must be created out of the `develop` branch and be used for the development process; see src/py/README.md +for further details about the Pyhton package development. Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. Take into account: -* Always remember to add tests (`src/pg/test`, `src/py/crankshaft/test`) - for any new functionality. +* Test must be added for any new functionality + (`src/pg/test`, `src/py/crankshaft/test`) as well as for + detected any bugs corrected. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. -* Convention: Use snake case (i.e. `snake_case` and not `CamelCase`) for all - functions. Prefix functions intended for public use with `cdb_` +* Convention: snake case(i.e. `snake_case` and not `CamelCase`) + shall be used for all function names. + Prefix function names intended for public use with `cdb_` and private functions (to be used only internally inside the extension) with `_cdb_`. -Update the local development installation with `sudo make install`. -This will update the 'dev' version of the extension in 'src/pg/' and +Once the code is ready to be tested, update the local development installation +with `sudo make install`. +This will update the 'dev' version of the extension in `src/pg/` and make it available to PostgreSQL. It will also install the python package (crankshaft) in a virtual environment `env/dev`. -Run the tests with `make test` +The version number of the Python package, defined in +`src/pg/crankshaft/setup.py` will be overridden when +the package is released and always match the extension version number, +but for development it shall be kept as '0.0.0'. + +Run the tests with `make test`. To use the python extension for custom tests, activate the virtual environment with: @@ -71,23 +81,27 @@ changes involve incompatible function changes such as a different return type; in that case the offending function (or the whole extension) should be dropped manually before the update. -If the extension has not previously been installed in a database -we can: +If the extension has not previously been installed in a database, +it can be installed directly with: * `CREATE EXTENSION crankshaft WITH VERSION 'dev';` -Note: the development extension uses the development pyhton virtual +Note: the development extension uses the development python virtual environment automatically. -Once the tests are succeeding a new Pull-Request can be created. -CI-tests must be checked to be successful. +Once the tests are succeeding a new Pull-Request can be created +to the develop branch. CI-tests must be checked to be successful. -Before proceeding to the release process peer code reviewing of the code is a must. +Before proceeding to the release process peer code reviewing of the code is +a must. + +When the code is accepted by the peer reviewing (and CI tests succeed) +a request for release will be send to the Release Manager. ## Release The release process of a new version of the extension -shall by performed by the designated *Release Manager*. +shall be performed by the designated *Release Manager*. Note that we expect to gradually automate more of this process. @@ -124,9 +138,9 @@ The new release can be deployed for staging/smoke tests with this command: sudo make deploy ``` -This will make the 'X.Y.Z' version of the extension to PostgreSQL. -The corresponding Python extension will be installed in a -virtual environment in `envs/X.Y.Z` +This will copy the current 'X.Y.Z' released version of the extension to +PostgreSQL. The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z`. It can be activated with: @@ -141,6 +155,13 @@ python package from this virtual environment. The `sudo make deploy` operation can be also used for installing the new version after it has been released. +To install a specific version 'X.Y.Z' different from the current one +(which must be present in `releases/`) you can: + +``` +sudo make deploy RELEASE_VERSION=X.Y.Z +``` + TODO: testing procedure for the new release. TODO: procedure for staging deployment. From ef3bcaeee891310ef0cd376068f652900345518f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:52:47 +0100 Subject: [PATCH 14/22] Restore commented-out make target --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 979c2b5..9c79e00 100644 --- a/Makefile +++ b/Makefile @@ -41,11 +41,11 @@ deploy: ## Deploy a released extension. Only for release manager. Requires sudo. clean-dev: ## clean up development extension script files rm -f src/pg/$(EXTENSION)--*.sql -# # Cleanup all releases -# clean-releases: -# rm -rf release/python/* -# rm -f release/$(EXTENSION)--*.sql -# rm -f release/$(EXTENSION).control +# Cleanup all releases +clean-releases: + rm -rf release/python/* + rm -f release/$(EXTENSION)--*.sql + rm -f release/$(EXTENSION).control # Cleanup current/specific version clean-release: ## clean up current release From 935c7f99639dae006b30179f1c542846b0789007 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:54:39 +0100 Subject: [PATCH 15/22] Add missing Makefile comment --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9c79e00..6c3e219 100644 --- a/Makefile +++ b/Makefile @@ -42,7 +42,7 @@ clean-dev: ## clean up development extension script files rm -f src/pg/$(EXTENSION)--*.sql # Cleanup all releases -clean-releases: +clean-releases: ## clean up all releases rm -rf release/python/* rm -f release/$(EXTENSION)--*.sql rm -f release/$(EXTENSION).control From 4706442a1ddb539646f1130d204e8bed7e7dcc8f Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 15:56:19 +0100 Subject: [PATCH 16/22] Add documentation about useful make targets --- README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/README.md b/README.md index 5c52541..d0305a1 100644 --- a/README.md +++ b/README.md @@ -168,3 +168,35 @@ TODO: procedure for staging deployment. TODO: procedure for merging to master, tagging and deploying in production. + +## Relevant tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +# Development tasks + +* `sudo make install` will generate the extension scripts for the development + version ('dev'/'current') and install the python package into the + development virtual environment `envs/dev`. + Intended for use by developers. + +* `make test` will run the tests for the installed development extension. + Intended for use by developers. + +# Release tasks + +* `make release` will generate a new release (version number defined in + `src/pg/crankshaft.control`) into `release/`. + Intended for use by the release manager. + +* `sudo make deploy` will install the current release X.Y.Z from the + `release/` files into PostgreSQL and a Python virtual environment + `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. + +* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version + previously generated in `release/` + into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. +``` From bad09ffd7be56da0ea50d97300c526d182b7a757 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 16:30:03 +0100 Subject: [PATCH 17/22] Remove abandoned alternatives from the documentation --- src/py/README.md | 70 ++++++++++-------------------------------------- 1 file changed, 14 insertions(+), 56 deletions(-) diff --git a/src/py/README.md b/src/py/README.md index d55b7d7..b9bf64d 100644 --- a/src/py/README.md +++ b/src/py/README.md @@ -20,37 +20,29 @@ nosetests test/ --- -We have two possible approaches being considered as to how manage -the Python virtual environment: using a pure virtual enviroment -or combine it with some system packages that include depencencies -for the *hard-to-compile* packages (and pin them in somewhat old versions). +To avoid troublesome compilations/linkings we will use +the available system package `python-scipy`. +This package and its dependencies provide numpy 1.6.1 +and scipy 0.9.0. To be able to use these versions we cannot +PySAL 1.10 or later, so we'll stick to 1.9.1. -### Alternative A: pure virtual environment +``` +apt-get install -y python-scipy +``` -In this case we will install all the packages needed in the -virtual environment. -This will involve, specially for the numerical packages compiling -and linking code that uses a number of third party libraries, -and requires having theses depencencies solved for the production -environments. +We'll use virtual environments to install our packages, +but configued to use also system modules so that the +mentioned scipy and numpy are used. -#### Create and use a virtual env - -We'll use a virtual enviroment directory `dev` -under the `src/pg` directory. - - # Create the virtual environment for python - $ virtualenv dev + # Create a virtual environment for python + $ virtualenv --system-site-packages dev # Activate the virtualenv $ source dev/bin/activate # Install all the requirements # expect this to take a while, as it will trigger a few compilations - (dev) $ pip install -r requirements.txt - - # Add a new pip to the party - (dev) $ pip install pandas + (dev) $ pip install -I ./crankshaft #### Test the libraries with that virtual env @@ -94,37 +86,3 @@ Then, execute the tests with: import pysal import nose nose.runmodule('pysal') - - -### Alternative B: using some packaged modules - -This option avoids troublesome compilations/linkings, at the cost -of freezing some module versions as available in system packages, -namely numpy 1.6.1 and scipy 0.9.0. (in turn, this implies -the most recent version of PySAL we can use is 1.9.1) - - -TODO: to use this alternative the python-scipy package must be -installed (this will have to be included in server provisioning) - -``` -apt-get install -y python-scipy -``` - -#### Create and use a virtual env - -We'll use a `dev` enviroment as before, but will configure it to -use also system modules. - - - # Create the virtual environment for python - $ virtualenv --system-site-packages dev - - # Activate the virtualenv - $ source dev/bin/activate - - # Install all the requirements - # expect this to take a while, as it will trigger a few compilations - (dev) $ pip install -I ./crankshaft - -Then we can proceed to testing as in Alternative A. From a7e42e93cc5781e0208eb0e4589543bf4ca4ec6e Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 16:41:54 +0100 Subject: [PATCH 18/22] Rename cdb_crankshaft_internal_version as internal function --- src/pg/sql/01_version.sql | 2 +- src/pg/sql/02_py.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/01_version.sql b/src/pg/sql/01_version.sql index 90e6123..5c701fd 100644 --- a/src/pg/sql/01_version.sql +++ b/src/pg/sql/01_version.sql @@ -6,7 +6,7 @@ $$ language 'sql' IMMUTABLE STRICT; -- Internal identifier of the installed extension instence -- e.g. 'dev' for current development version -CREATE OR REPLACE FUNCTION cdb_crankshaft_internal_version() +CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() RETURNS text AS $$ SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; $$ language 'sql' IMMUTABLE STRICT; diff --git a/src/pg/sql/02_py.sql b/src/pg/sql/02_py.sql index a233a0a..7da5f47 100644 --- a/src/pg/sql/02_py.sql +++ b/src/pg/sql/02_py.sql @@ -14,7 +14,7 @@ AS $$ import os # plpy.notice('%',str(os.environ)) # activate virtualenv - crankshaft_version = plpy.execute('SELECT cdb_crankshaft.cdb_crankshaft_internal_version()')[0]['cdb_crankshaft_internal_version'] + crankshaft_version = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_internal_version()')[0]['_cdb_crankshaft_internal_version'] base_path = plpy.execute('SELECT cdb_crankshaft._cdb_crankshaft_virtualenvs_path()')[0]['_cdb_crankshaft_virtualenvs_path'] default_venv_path = os.path.join(base_path, crankshaft_version) venv_path = os.environ.get('CRANKSHAFT_VENV', default_venv_path) From 689ec8a925297b0f85e5def6223c063db4b53ba0 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 17:09:50 +0100 Subject: [PATCH 19/22] Change version function from IMMUTABLE to STABLE These functions' results will change when the extension is updated. --- src/pg/sql/01_version.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/01_version.sql b/src/pg/sql/01_version.sql index 5c701fd..f73c764 100644 --- a/src/pg/sql/01_version.sql +++ b/src/pg/sql/01_version.sql @@ -2,11 +2,11 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft_version() RETURNS text AS $$ SELECT '@@VERSION@@'::text; -$$ language 'sql' IMMUTABLE STRICT; +$$ language 'sql' STABLE STRICT; -- Internal identifier of the installed extension instence -- e.g. 'dev' for current development version CREATE OR REPLACE FUNCTION _cdb_crankshaft_internal_version() RETURNS text AS $$ SELECT installed_version FROM pg_available_extensions where name='crankshaft' and pg_available_extensions IS NOT NULL; -$$ language 'sql' IMMUTABLE STRICT; +$$ language 'sql' STABLE STRICT; From b5281d06813214b2045e4a8eb26d4c6f9cfef1a3 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 17:19:21 +0100 Subject: [PATCH 20/22] Documentation clarifications and corrections. --- README.md | 50 ++++++++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d0305a1..5003292 100644 --- a/README.md +++ b/README.md @@ -18,27 +18,42 @@ CartoDB Spatial Analysis extension for PostgreSQL. # Working Process +We distinguish two roles regarding the development cycle of crankshaft: + +* *developers* will implement new functionality and bugfixes into + the codebase and will request for new releases of the extension. +* A *release manager* will attend these requests and will handle + the release process. The release process is sequential: + no concurrent releases will ever be in the works. + We use the default `develop` branch as the basis for development. This branch and `master` are maintained by the *Release Manager*. The `master` branch is used to merge and tag releases to be deployed in production. -In addition to these two permanent branches, temporal *topic* -branches must be used for all modifications. +Developers shall create a new topic branch from `develop` for any new feature +or bugfix and commit their changes to it and eventually merge back into +the `develop` branch. When a new release is required a Pull Request +will be open againt the `develop` branch. + +The `develop` pull requests will be handled by the release manage, +who will merge into master where new releases are prepared and tagged. +The `master` branch is the sole responsibility of the release masters +and developers must not commit or merge into it. ## Development -For any modification of crankshaft, including adding new features, -a topic branch must be created out of the `develop` branch -and be used for the development process; see src/py/README.md -for further details about the Pyhton package development. +For any modification of crankshaft, such as adding new features, +refactoring or bug-fixing, topic branch must be created out of the `develop` +branch and be used for the development process. Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. + Take into account: -* Test must be added for any new functionality - (`src/pg/test`, `src/py/crankshaft/test`) as well as for - detected any bugs corrected. +* Tests must be added for any new functionality + (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to + detect any bugs that are being fixed. * Add or modify the corresponding documentation files in the `doc` folder. Since we expect to have highly technical functions here, an extense background explanation would be of great help to users of this extension. @@ -89,14 +104,17 @@ it can be installed directly with: Note: the development extension uses the development python virtual environment automatically. -Once the tests are succeeding a new Pull-Request can be created -to the develop branch. CI-tests must be checked to be successful. - Before proceeding to the release process peer code reviewing of the code is a must. -When the code is accepted by the peer reviewing (and CI tests succeed) -a request for release will be send to the Release Manager. +Once the feature or bugfix is completed, all the tests are passing +and the code has been accepted by peer reviewing, +the topic branch can be merged back into the `develop` branch and a +new Pull-Request can be created on it. +CI-tests must be checked to be successful. + +The release manage will take hold of the PR at this moment to proceed +to the release process for a new revision of the extension. ## Release @@ -105,8 +123,8 @@ shall be performed by the designated *Release Manager*. Note that we expect to gradually automate more of this process. -Having checked the topic branch of the PR to be released it shall be -merged back into the `develop` branch to prepare the new release. +Having checked PR to be released it shall be +merged back into the `master` branch to prepare the new release. The version number in `pg/cranckshaft.control` must first be updated. To do so [Semantic Versioning 2.0](http://semver.org/) is in order. From 9f03a9b0754a790a000701cdfd744f242bd4e2a2 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 17:42:28 +0100 Subject: [PATCH 21/22] Reorganize the documentation into separate files Keep a "Quickstart Guide" in the README, add separate detailed sections for development (CONTRIBUTING) and release/deployment (RELEASE). --- CONTRIBUTING.md | 90 +++++++++++++++++++++++ DEPLOYING.md | 43 ----------- README.md | 186 +++++------------------------------------------- RELEASE.md | 93 ++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 210 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 DEPLOYING.md create mode 100644 RELEASE.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c6422a2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,90 @@ +# Development process + +Please read the Working Process/Quickstart Guide in README.md first. + +For any modification of crankshaft, such as adding new features, +refactoring or bug-fixing, topic branch must be created out of the `develop` +branch and be used for the development process. + +Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. + +Take into account: + +* Tests must be added for any new functionality + (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to + detect any bugs that are being fixed. +* Add or modify the corresponding documentation files in the `doc` folder. + Since we expect to have highly technical functions here, an extense + background explanation would be of great help to users of this extension. +* Convention: snake case(i.e. `snake_case` and not `CamelCase`) + shall be used for all function names. + Prefix function names intended for public use with `cdb_` + and private functions (to be used only internally inside + the extension) with `_cdb_`. + +Once the code is ready to be tested, update the local development installation +with `sudo make install`. +This will update the 'dev' version of the extension in `src/pg/` and +make it available to PostgreSQL. +It will also install the python package (crankshaft) in a virtual +environment `env/dev`. + +The version number of the Python package, defined in +`src/pg/crankshaft/setup.py` will be overridden when +the package is released and always match the extension version number, +but for development it shall be kept as '0.0.0'. + +Run the tests with `make test`. + +To use the python extension for custom tests, activate the virtual +environment with: + +``` +source envs/dev/bin/activate +``` + +Update extension in a working database with: + +* `ALTER EXTENSION crankshaft VERSION TO 'current';` + `ALTER EXTENSION crankshaft VERSION TO 'dev';` + +Note: we keep the current development version install as 'dev' always; +we update through the 'current' alias to allow changing the extension +contents but not the version identifier. This will fail if the +changes involve incompatible function changes such as a different +return type; in that case the offending function (or the whole extension) +should be dropped manually before the update. + +If the extension has not previously been installed in a database, +it can be installed directly with: + +* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` + +Note: the development extension uses the development python virtual +environment automatically. + +Before proceeding to the release process peer code reviewing of the code is +a must. + +Once the feature or bugfix is completed, all the tests are passing +and the code has been accepted by peer reviewing, +the topic branch can be merged back into the `develop` branch and a +new Pull-Request can be created on it. +CI-tests must be checked to be successful. + +The release manage will take hold of the PR at this moment to proceed +to the release process for a new revision of the extension. + +## Relevant development tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +* `sudo make install` will generate the extension scripts for the development + version ('dev'/'current') and install the python package into the + development virtual environment `envs/dev`. + Intended for use by developers. + +* `make test` will run the tests for the installed development extension. + Intended for use by developers. +``` diff --git a/DEPLOYING.md b/DEPLOYING.md deleted file mode 100644 index 5b21f30..0000000 --- a/DEPLOYING.md +++ /dev/null @@ -1,43 +0,0 @@ -# Workflow - -... (branching/merging flow) - -# Deployment - -... - -Deployment to db servers: the next command will install both the Python -package and the extension. - -``` -sudo make install -``` - -Installing only the Python package: - -``` -sudo pip install python/crankshaft --upgrade -``` - -Caveat: note that `pip install ./crankshaft` will install -from local files, but `pip install crankshaft` will not. - -CI: Install and run the tests on the installed extension and package: - -``` -(sudo make install && PGUSER=postgres make testinstalled) -``` - -Installing the extension in user databases: -Once installed in a server, the extension can be added -to a database with the next SQL command: - -``` -CREATE EXTENSION crankshaft; -``` - -To upgrade the extension to an specific version X.Y.Z: - -``` -ALTER EXTENSION crankshaft UPGRADE TO 'X.Y.Z'; -``` diff --git a/README.md b/README.md index 5003292..cf8de5a 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ CartoDB Spatial Analysis extension for PostgreSQL. * pip, virtualenv, PostgreSQL * python-scipy system package (see src/py/README.md) -# Working Process +# Working Process -- Quickstart Guide We distinguish two roles regarding the development cycle of crankshaft: @@ -41,180 +41,32 @@ who will merge into master where new releases are prepared and tagged. The `master` branch is the sole responsibility of the release masters and developers must not commit or merge into it. -## Development +## Development Guidelines -For any modification of crankshaft, such as adding new features, -refactoring or bug-fixing, topic branch must be created out of the `develop` -branch and be used for the development process. +For a detailed description of the development process please see +the CONTRIBUTING.md guide. -Modifications are done inside `src/pg/sql` and `src/py/crankshaft`. +Any modification to the source code (`src/pg/sql` for the SQL extension, +`src/py/crankshaft` for the Python package) shall always be done +in a topic branch created from the `develop` branch. -Take into account: +Tests, documentation and peer code reviewing are required for all +modifications. -* Tests must be added for any new functionality - (inside `src/pg/test`, `src/py/crankshaft/test`) as well as to - detect any bugs that are being fixed. -* Add or modify the corresponding documentation files in the `doc` folder. - Since we expect to have highly technical functions here, an extense - background explanation would be of great help to users of this extension. -* Convention: snake case(i.e. `snake_case` and not `CamelCase`) - shall be used for all function names. - Prefix function names intended for public use with `cdb_` - and private functions (to be used only internally inside - the extension) with `_cdb_`. - -Once the code is ready to be tested, update the local development installation -with `sudo make install`. -This will update the 'dev' version of the extension in `src/pg/` and -make it available to PostgreSQL. -It will also install the python package (crankshaft) in a virtual -environment `env/dev`. - -The version number of the Python package, defined in -`src/pg/crankshaft/setup.py` will be overridden when -the package is released and always match the extension version number, -but for development it shall be kept as '0.0.0'. - -Run the tests with `make test`. - -To use the python extension for custom tests, activate the virtual -environment with: +The tests (both for SQL and Pyhton) are executed by running, +from the top directory: ``` -source envs/dev/bin/activate +sudo make install +make test ``` -Update extension in a working database with: - -* `ALTER EXTENSION crankshaft VERSION TO 'current';` - `ALTER EXTENSION crankshaft VERSION TO 'dev';` - -Note: we keep the current development version install as 'dev' always; -we update through the 'current' alias to allow changing the extension -contents but not the version identifier. This will fail if the -changes involve incompatible function changes such as a different -return type; in that case the offending function (or the whole extension) -should be dropped manually before the update. - -If the extension has not previously been installed in a database, -it can be installed directly with: - -* `CREATE EXTENSION crankshaft WITH VERSION 'dev';` - -Note: the development extension uses the development python virtual -environment automatically. - -Before proceeding to the release process peer code reviewing of the code is -a must. - -Once the feature or bugfix is completed, all the tests are passing -and the code has been accepted by peer reviewing, -the topic branch can be merged back into the `develop` branch and a -new Pull-Request can be created on it. -CI-tests must be checked to be successful. - -The release manage will take hold of the PR at this moment to proceed -to the release process for a new revision of the extension. +To request a new release, which will be handled by them +release manager, a Pull Request must be created in the `develop` +branch. ## Release -The release process of a new version of the extension -shall be performed by the designated *Release Manager*. - -Note that we expect to gradually automate more of this process. - -Having checked PR to be released it shall be -merged back into the `master` branch to prepare the new release. - -The version number in `pg/cranckshaft.control` must first be updated. -To do so [Semantic Versioning 2.0](http://semver.org/) is in order. - -Thew `NEWS.md` will be updated. - -We now will explain the process for the case of backwards-compatible -releases (updating the minor or patch version numbers). - -TODO: document the complex case of major releases. - -The next command must be executed to produce the main installation -script for the new release, `release/cranckshaft--X.Y.Z.sql` and -also to copy the python package to `release/python/X.Y.Z/crankshaft`. - -``` -make release -``` - -Then, the release manager shall produce upgrade and downgrade scripts -to migrate to/from the previous release. In the case of minor/patch -releases this simply consist in extracting the functions that have changed -and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` -file. - -The new release can be deployed for staging/smoke tests with this command: - -``` -sudo make deploy -``` - -This will copy the current 'X.Y.Z' released version of the extension to -PostgreSQL. The corresponding Python extension will be installed in a -virtual environment in `envs/X.Y.Z`. - -It can be activated with: - -``` -source envs/X.Y.Z/bin/activate -``` - -But note that this is needed only for using the package directly; -the 'X.Y.Z' version of the extension will automatically use the -python package from this virtual environment. - -The `sudo make deploy` operation can be also used for installing -the new version after it has been released. - -To install a specific version 'X.Y.Z' different from the current one -(which must be present in `releases/`) you can: - -``` -sudo make deploy RELEASE_VERSION=X.Y.Z -``` - -TODO: testing procedure for the new release. - -TODO: procedure for staging deployment. - -TODO: procedure for merging to master, tagging and deploying -in production. - -## Relevant tasks available in the Makefile - -``` -* `make help` show a short description of the available targets - -# Development tasks - -* `sudo make install` will generate the extension scripts for the development - version ('dev'/'current') and install the python package into the - development virtual environment `envs/dev`. - Intended for use by developers. - -* `make test` will run the tests for the installed development extension. - Intended for use by developers. - -# Release tasks - -* `make release` will generate a new release (version number defined in - `src/pg/crankshaft.control`) into `release/`. - Intended for use by the release manager. - -* `sudo make deploy` will install the current release X.Y.Z from the - `release/` files into PostgreSQL and a Python virtual environment - `envs/X.Y.Z`. - Intended for use by the release manager and deployment jobs. - -* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version - previously generated in `release/` - into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. - Intended for use by the release manager and deployment jobs. -``` +The release and deployment process is described in the +RELEASE.md guide and it is the responsibility of the designated +release manager. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..0db48a2 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,93 @@ +# Release & Deployment Process + +Please read the Working Process/Quickstart Guide in README.md +and the Development guidelines in CONTRIBUTING.md. + +The release process of a new version of the extension +shall be performed by the designated *Release Manager*. + +Note that we expect to gradually automate more of this process. + +Having checked PR to be released it shall be +merged back into the `master` branch to prepare the new release. + +The version number in `pg/cranckshaft.control` must first be updated. +To do so [Semantic Versioning 2.0](http://semver.org/) is in order. + +Thew `NEWS.md` will be updated. + +We now will explain the process for the case of backwards-compatible +releases (updating the minor or patch version numbers). + +TODO: document the complex case of major releases. + +The next command must be executed to produce the main installation +script for the new release, `release/cranckshaft--X.Y.Z.sql` and +also to copy the python package to `release/python/X.Y.Z/crankshaft`. + +``` +make release +``` + +Then, the release manager shall produce upgrade and downgrade scripts +to migrate to/from the previous release. In the case of minor/patch +releases this simply consist in extracting the functions that have changed +and placing them in the proper `release/cranckshaft--X.Y.Z--A.B.C.sql` +file. + +The new release can be deployed for staging/smoke tests with this command: + +``` +sudo make deploy +``` + +This will copy the current 'X.Y.Z' released version of the extension to +PostgreSQL. The corresponding Python extension will be installed in a +virtual environment in `envs/X.Y.Z`. + +It can be activated with: + +``` +source envs/X.Y.Z/bin/activate +``` + +But note that this is needed only for using the package directly; +the 'X.Y.Z' version of the extension will automatically use the +python package from this virtual environment. + +The `sudo make deploy` operation can be also used for installing +the new version after it has been released. + +To install a specific version 'X.Y.Z' different from the current one +(which must be present in `releases/`) you can: + +``` +sudo make deploy RELEASE_VERSION=X.Y.Z +``` + +TODO: testing procedure for the new release. + +TODO: procedure for staging deployment. + +TODO: procedure for merging to master, tagging and deploying +in production. + +## Relevant release & deployment tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +* `make release` will generate a new release (version number defined in + `src/pg/crankshaft.control`) into `release/`. + Intended for use by the release manager. + +* `sudo make deploy` will install the current release X.Y.Z from the + `release/` files into PostgreSQL and a Python virtual environment + `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. + +* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version + previously generated in `release/` + into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. +``` From 47e0253652f496b173744f57d1e9c3e1c9187201 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 16 Mar 2016 18:18:59 +0100 Subject: [PATCH 22/22] Fixes to the documentation --- CONTRIBUTING.md | 11 ++++++----- README.md | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c6422a2..bcdde4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,11 +66,12 @@ environment automatically. Before proceeding to the release process peer code reviewing of the code is a must. -Once the feature or bugfix is completed, all the tests are passing -and the code has been accepted by peer reviewing, -the topic branch can be merged back into the `develop` branch and a -new Pull-Request can be created on it. -CI-tests must be checked to be successful. +Once the feature or bugfix is completed and all the tests are passing +a Pull-Request shall be created on the topic branch, reviewed by a peer +and then merged back into the `develop` branch when all CI tests pass. + +When the changes in the `develop` branch are to be released in a new +version of the extension, a PR must be created on the `develop` branch. The release manage will take hold of the PR at this moment to proceed to the release process for a new revision of the extension. diff --git a/README.md b/README.md index cf8de5a..3ecb1d0 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,6 @@ We distinguish two roles regarding the development cycle of crankshaft: no concurrent releases will ever be in the works. We use the default `develop` branch as the basis for development. -This branch and `master` are maintained by the *Release Manager*. The `master` branch is used to merge and tag releases to be deployed in production.