From 34c9a7d4cfa9988d0c6ae68ba3d323124c3de189 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 13:40:30 +0200 Subject: [PATCH 1/7] First version of the compat check script --- check-compatibility.sh | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100755 check-compatibility.sh diff --git a/check-compatibility.sh b/check-compatibility.sh new file mode 100755 index 0000000..545f99f --- /dev/null +++ b/check-compatibility.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +export PGUSER=postgres + +DBNAME=crankshaft_compatcheck + +function die { + echo $1 + exit -1 +} + +# Create fresh DB +psql -c "CREATE DATABASE $DBNAME;" || die "Could not create DB" + +# Hook for cleanup +function cleanup { + psql -c "DROP DATABASE IF EXISTS crankshaft_compatcheck;" +} +trap cleanup EXIT + +# Deploy previous release +(cd src/py && sudo make deploy RUN_OPTIONS="--no-deps") || die "Could not deploy python extension" +(cd src/pg && sudo make deploy) || die " Could not deploy last release" +psql -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'crankshaft';" + +# Install in the fresh DB +psql $DBNAME <<'EOF' +-- Install dependencies +CREATE EXTENSION plpythonu; +CREATE EXTENSION postgis VERSION '2.2.2'; + +-- Create role publicuser if it does not exist +DO +$$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_user + WHERE usename = 'publicuser') THEN + + CREATE ROLE publicuser LOGIN; + END IF; +END +$$ LANGUAGE plpgsql; + +-- Install the default version +CREATE EXTENSION crankshaft; +\dx +EOF + +# TODO save public functions and signatures + +# Check it can be upgraded +psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" + + + +# TODO check against saved public functions and signatures + + From cdc072b09847b2b99463daaef93dc0b31c46a242 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 15:38:28 +0200 Subject: [PATCH 2/7] Add upgrade path from release to dev version --- src/pg/.gitignore | 1 + src/pg/Makefile | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/pg/.gitignore b/src/pg/.gitignore index b58a014..56825ae 100644 --- a/src/pg/.gitignore +++ b/src/pg/.gitignore @@ -4,3 +4,4 @@ results/ crankshaft--dev.sql crankshaft--dev--current.sql crankshaft--current--dev.sql +crankshaft--*--dev.sql diff --git a/src/pg/Makefile b/src/pg/Makefile index 178ed08..6775aad 100644 --- a/src/pg/Makefile +++ b/src/pg/Makefile @@ -10,14 +10,15 @@ include ../../Makefile.global # * test runs the tests for the currently generated Development # extension. -DATA = $(EXTENSION)--dev.sql \ - $(EXTENSION)--current--dev.sql \ - $(EXTENSION)--dev--current.sql +DATA = \ + $(EXTENSION)--dev.sql \ + $(EXTENSION)--current--dev.sql \ + $(EXTENSION)--dev--current.sql \ + $(EXTENSION)--$(RELEASE_VERSION)--dev.sql SOURCES_DATA_DIR = sql SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) - REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' $(DATA): $(SOURCES_DATA) From abe07166de544ed1ac1d17ffd0a3267be2f14779 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 15:39:28 +0200 Subject: [PATCH 3/7] Add installation of current dev version --- check-compatibility.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/check-compatibility.sh b/check-compatibility.sh index 545f99f..f6930e7 100755 --- a/check-compatibility.sh +++ b/check-compatibility.sh @@ -50,11 +50,13 @@ EOF # TODO save public functions and signatures +# Deploy current dev branch +make clean-dev || die "Could not clean dev files" +sudo make install || die "Could not deploy current dev branch" + # Check it can be upgraded psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" # TODO check against saved public functions and signatures - - From e74d80f3eab23a2c6663adda5e2d8229c543207d Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 15:48:45 +0200 Subject: [PATCH 4/7] Add compat checks as a build step --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cfd9c52..c3b166e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -50,3 +50,4 @@ install: script: - make test || { cat src/pg/test/regression.diffs; false; } + - ./check-compatibility.sh From ac51256463909bdf6b0f75141ad67b93c8e1a4f4 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 18:12:43 +0200 Subject: [PATCH 5/7] Fix for stale builds --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index c3b166e..a165028 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,9 @@ language: c +env: + global: + - PAGER=cat + before_install: - ./check-up-to-date-with-master.sh - sudo apt-get -y install python-pip From 4118b57f1f12cece4899156902625d4159ec36c2 Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 18:56:11 +0200 Subject: [PATCH 6/7] Create aggregates only if they do not exist --- src/pg/sql/04_py_agg.sql | 26 +++++++++++++++++++------- src/pg/sql/11_kmeans.sql | 26 ++++++++++++++++++++------ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/src/pg/sql/04_py_agg.sql b/src/pg/sql/04_py_agg.sql index c38e323..a3e881b 100644 --- a/src/pg/sql/04_py_agg.sql +++ b/src/pg/sql/04_py_agg.sql @@ -10,10 +10,22 @@ CREATE OR REPLACE FUNCTION END $$ LANGUAGE plpgsql; - -CREATE AGGREGATE CDB_PyAgg(NUMERIC[])( - SFUNC = CDB_PyAggS, - STYPE = Numeric[], - INITCOND = "{}" -); - +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_pyagg' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) ( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + INITCOND = "{}" + ); + END IF; +END +$$ LANGUAGE plpgsql; diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index 125aac3..f20942f 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -41,9 +41,23 @@ BEGIN END $$ LANGUAGE plpgsql; -CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC)( - SFUNC = CDB_WeightedMeanS, - FINALFUNC = CDB_WeightedMeanF, - STYPE = Numeric[], - INITCOND = "{0.0,0.0,0.0}" -); +-- Create aggregate if it did not exist +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'cdb_crankshaft' + AND p.proname = 'cdb_weightedmean' + AND p.proisagg) + THEN + CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) ( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + INITCOND = "{0.0,0.0,0.0}" + ); + END IF; +END +$$ LANGUAGE plpgsql; From 4886c187c4352d2fa95d7882fba991d87441885e Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 10 Aug 2016 19:30:51 +0200 Subject: [PATCH 7/7] Check function signatures --- check-compatibility.sh | 50 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/check-compatibility.sh b/check-compatibility.sh index f6930e7..966850c 100755 --- a/check-compatibility.sh +++ b/check-compatibility.sh @@ -48,7 +48,26 @@ CREATE EXTENSION crankshaft; \dx EOF -# TODO save public functions and signatures +# Save public function signatures +psql $DBNAME <<'EOF' +CREATE TABLE release_function_signatures AS + SELECT + p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE + WHEN p.proisagg THEN 'agg' + WHEN p.proiswindow THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4; +EOF # Deploy current dev branch make clean-dev || die "Could not clean dev files" @@ -57,6 +76,33 @@ sudo make install || die "Could not deploy current dev branch" # Check it can be upgraded psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" +# Check against saved public function signatures +psql $DBNAME <<'EOF' +CREATE TABLE dev_function_signatures AS + SELECT + p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE + WHEN p.proisagg THEN 'agg' + WHEN p.proiswindow THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4; +EOF +echo "Functions in development not in latest release (ok):" +psql $DBNAME -c "SELECT * FROM dev_function_signatures EXCEPT SELECT * FROM release_function_signatures;" -# TODO check against saved public functions and signatures +echo "Functions in latest release not in development (compat issue):" +psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;" + +# Fail if there's a signature mismatch / missing functions +psql $DBNAME -c "SELECT * FROM release_function_signatures EXCEPT SELECT * FROM dev_function_signatures;" | fgrep '(0 rows)' \ + || die "Function signatures changed"