From 54646acc565dc3f3365ad14a30d00d39c48b95c1 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Thu, 7 Feb 2019 18:30:22 +0100 Subject: [PATCH] PG 11 --- .travis.yml | 68 +++++-------- NEWS.md | 5 + check-compatibility.sh | 119 +++++++++++++++-------- src/pg/crankshaft.control | 2 +- src/pg/sql/04_py_agg.sql | 29 ++---- src/pg/sql/11_kmeans.sql | 32 +++--- src/pg/test/expected/07_gravity_test.out | 20 ++-- src/pg/test/expected/09_voronoi_test.out | 8 +- src/pg/test/sql/09_voronoi_test.sql | 4 +- src/py/crankshaft/requirements.txt | 8 +- src/py/crankshaft/setup.py | 2 +- 11 files changed, 151 insertions(+), 146 deletions(-) diff --git a/.travis.yml b/.travis.yml index e9725b3..6f96e5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,60 +1,40 @@ +dist: xenial language: c -dist: precise sudo: required env: global: - PAGER=cat + - PGUSER=postgres + - PGDATABASE=postgres + - PGOPTIONS='-c client_min_messages=NOTICE' + - PGPORT=5432 + - POSTGIS_VERSION="2.5" + + matrix: + - POSTGRESQL_VERSION="10" + - POSTGRESQL_VERSION="11" before_install: - - ./check-up-to-date-with-master.sh - - sudo apt-get -y install python-pip - - sudo apt-get -y install python-software-properties - - sudo add-apt-repository -y ppa:cartodb/sci - - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5 - - sudo add-apt-repository -y ppa:cartodb/gis - - sudo add-apt-repository -y ppa:cartodb/gis-testing - - sudo apt-get update + - sudo service postgresql stop; + - sudo apt-get remove postgresql* -y + - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common + - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis postgresql-plpython-$POSTGRESQL_VERSION + - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main + - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION + - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main -- -A trust + - sudo /etc/init.d/postgresql start $POSTGRESQL_VERSION || sudo journalctl -xe - - sudo apt-get -y install python-joblib=0.8.3-1-cdb1 - - sudo apt-get -y install python-numpy=1:1.6.1-6ubuntu1 - - - sudo apt-get -y install python-scipy=0.14.0-2-cdb6 - - sudo apt-get -y --no-install-recommends install python-sklearn-lib=0.14.1-3-cdb2 - - sudo apt-get -y --no-install-recommends install python-sklearn=0.14.1-3-cdb2 - - sudo apt-get -y --no-install-recommends install python-scikits-learn=0.14.1-3-cdb2 - - # Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis) - - sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2 - - # Install postgres db and build deps - - sudo /etc/init.d/postgresql stop # stop travis default instance - - sudo apt-get -y remove --purge postgresql-9.1 - - sudo apt-get -y remove --purge postgresql-9.2 - - sudo apt-get -y remove --purge postgresql-9.3 - - sudo apt-get -y remove --purge postgresql-9.4 - - sudo apt-get -y remove --purge postgresql-9.5 - - sudo rm -rf /var/lib/postgresql/ - - sudo rm -rf /var/log/postgresql/ - - sudo rm -rf /etc/postgresql/ - - sudo apt-get -y remove --purge postgis-2.2 - - sudo apt-get -y autoremove - - - sudo apt-get -y install postgresql-9.5=9.5.2-3cdb3 - - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb3 - - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb3 - - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2 - - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2 - - # configure it to accept local connections from postgres - - echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \ - | sudo tee /etc/postgresql/9.5/main/pg_hba.conf - - sudo /etc/init.d/postgresql restart 9.5 + - sudo apt-get -y install python-pip python-software-properties python-joblib python-nose install: - sudo make install script: - - make test || { cat src/pg/test/regression.diffs; false; } + - make test - ./check-compatibility.sh + +after_failure: + - pg_lsclusters + - cat src/pg/test/regression.diffs diff --git a/NEWS.md b/NEWS.md index 5e29b9c..db78952 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,8 @@ +0.8.2 (2019-02-07) +* Update dependencies to match what it's being used in production. +* Update travis to xenial, PG10 and 11, and postgis 2.6 +* Compatibility with PG11 + 0.8.1 (2018-03-12) ------------------ * Adds improperly added version files diff --git a/check-compatibility.sh b/check-compatibility.sh index 966850c..3b606e3 100755 --- a/check-compatibility.sh +++ b/check-compatibility.sh @@ -27,7 +27,7 @@ psql -c "SELECT * FROM pg_available_extension_versions WHERE name LIKE 'cranksha psql $DBNAME <<'EOF' -- Install dependencies CREATE EXTENSION plpythonu; -CREATE EXTENSION postgis VERSION '2.2.2'; +CREATE EXTENSION postgis; -- Create role publicuser if it does not exist DO @@ -48,26 +48,49 @@ CREATE EXTENSION crankshaft; \dx EOF + +# Check PG version +PG_VERSION=`psql -q -t -c "SELECT current_setting('server_version_num')"` + # Save public function signatures -psql $DBNAME <<'EOF' -CREATE TABLE release_function_signatures AS - SELECT - p.proname as name, - pg_catalog.pg_get_function_result(p.oid) as result_type, - pg_catalog.pg_get_function_arguments(p.oid) as arguments, - CASE - WHEN p.proisagg THEN 'agg' - WHEN p.proiswindow THEN 'window' - WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' - ELSE 'normal' - END as type - FROM pg_catalog.pg_proc p - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE - n.nspname = 'cdb_crankshaft' - AND p.proname LIKE 'cdb_%' - ORDER BY 1, 2, 4; -EOF +if [[ "$PG_VERSION" -lt 110000 ]]; then + psql $DBNAME -c " + CREATE TABLE release_function_signatures AS + SELECT + p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE + WHEN p.proisagg THEN 'agg' + WHEN p.proiswindow THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4;" +else + psql $DBNAME -c " + CREATE TABLE release_function_signatures AS + SELECT + p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE WHEN p.prokind = 'a' THEN 'agg' + WHEN p.prokind = 'w' THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4;" +fi # Deploy current dev branch make clean-dev || die "Could not clean dev files" @@ -76,26 +99,42 @@ sudo make install || die "Could not deploy current dev branch" # Check it can be upgraded psql $DBNAME -c "ALTER EXTENSION crankshaft update to 'dev';" || die "Cannot upgrade to dev version" -# Check against saved public function signatures -psql $DBNAME <<'EOF' -CREATE TABLE dev_function_signatures AS - SELECT - p.proname as name, - pg_catalog.pg_get_function_result(p.oid) as result_type, - pg_catalog.pg_get_function_arguments(p.oid) as arguments, - CASE - WHEN p.proisagg THEN 'agg' - WHEN p.proiswindow THEN 'window' - WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' - ELSE 'normal' - END as type - FROM pg_catalog.pg_proc p - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE - n.nspname = 'cdb_crankshaft' - AND p.proname LIKE 'cdb_%' - ORDER BY 1, 2, 4; -EOF +if [[ $PG_VERSION -lt 110000 ]]; then + psql $DBNAME -c " + CREATE TABLE dev_function_signatures AS + SELECT p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE WHEN p.proisagg THEN 'agg' + WHEN p.proiswindow THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4;" +else + psql $DBNAME -c " + CREATE TABLE dev_function_signatures AS + SELECT p.proname as name, + pg_catalog.pg_get_function_result(p.oid) as result_type, + pg_catalog.pg_get_function_arguments(p.oid) as arguments, + CASE WHEN p.prokind = 'a' THEN 'agg' + WHEN p.prokind = 'w' THEN 'window' + WHEN p.prorettype = 'pg_catalog.trigger'::pg_catalog.regtype THEN 'trigger' + ELSE 'normal' + END as type + FROM pg_catalog.pg_proc p + LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE + n.nspname = 'cdb_crankshaft' + AND p.proname LIKE 'cdb_%' + ORDER BY 1, 2, 4;" +fi + echo "Functions in development not in latest release (ok):" psql $DBNAME -c "SELECT * FROM dev_function_signatures EXCEPT SELECT * FROM release_function_signatures;" diff --git a/src/pg/crankshaft.control b/src/pg/crankshaft.control index b33d9e5..79d68b4 100644 --- a/src/pg/crankshaft.control +++ b/src/pg/crankshaft.control @@ -1,5 +1,5 @@ comment = 'CartoDB Spatial Analysis extension' -default_version = '0.8.1' +default_version = '0.8.2' requires = 'plpythonu, postgis' superuser = true schema = cdb_crankshaft diff --git a/src/pg/sql/04_py_agg.sql b/src/pg/sql/04_py_agg.sql index e08efdc..ce033fe 100644 --- a/src/pg/sql/04_py_agg.sql +++ b/src/pg/sql/04_py_agg.sql @@ -11,22 +11,13 @@ CREATE OR REPLACE FUNCTION $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; -- Create aggregate if it did not exist -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT * - FROM pg_catalog.pg_proc p - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname = 'cdb_crankshaft' - AND p.proname = 'cdb_pyagg' - AND p.proisagg) - THEN - CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) ( - SFUNC = CDB_PyAggS, - STYPE = Numeric[], - PARALLEL = SAFE, - INITCOND = "{}" - ); - END IF; -END -$$ LANGUAGE plpgsql; +DO $$ BEGIN + CREATE AGGREGATE CDB_PyAgg(NUMERIC[]) ( + SFUNC = CDB_PyAggS, + STYPE = Numeric[], + PARALLEL = SAFE, + INITCOND = "{}" + ); +EXCEPTION + WHEN duplicate_function THEN NULL; +END $$; diff --git a/src/pg/sql/11_kmeans.sql b/src/pg/sql/11_kmeans.sql index b0362b8..f17e395 100644 --- a/src/pg/sql/11_kmeans.sql +++ b/src/pg/sql/11_kmeans.sql @@ -85,24 +85,16 @@ BEGIN END $$ LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE; + -- Create aggregate if it did not exist -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT * - FROM pg_catalog.pg_proc p - LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname = 'cdb_crankshaft' - AND p.proname = 'cdb_weightedmean' - AND p.proisagg) - THEN - CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) ( - SFUNC = CDB_WeightedMeanS, - FINALFUNC = CDB_WeightedMeanF, - STYPE = Numeric[], - PARALLEL = SAFE, - INITCOND = "{0.0,0.0,0.0}" - ); - END IF; -END -$$ LANGUAGE plpgsql; +DO $$ BEGIN + CREATE AGGREGATE CDB_WeightedMean(geometry(Point, 4326), NUMERIC) ( + SFUNC = CDB_WeightedMeanS, + FINALFUNC = CDB_WeightedMeanF, + STYPE = Numeric[], + PARALLEL = SAFE, + INITCOND = "{0.0,0.0,0.0}" +); +EXCEPTION + WHEN duplicate_function THEN NULL; +END $$; diff --git a/src/pg/test/expected/07_gravity_test.out b/src/pg/test/expected/07_gravity_test.out index 064f3c5..8552b72 100644 --- a/src/pg/test/expected/07_gravity_test.out +++ b/src/pg/test/expected/07_gravity_test.out @@ -1,14 +1,14 @@ SET client_min_messages TO WARNING; \set ECHO none - the_geom | h | hpop | dist ---------------------------------------------+-------------------------+--------------------------+---------------- - 01010000001361C3D32B650140DD24068195B34440 | 1.51078258369747945249 | 12.08626066957983561994 | 4964.714459152 - 01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954183620807430 | 688.08116679285345652007 | 99.955141922 - 0101000000A167B3EA733501401D5A643BDFAF4440 | 63.70532894711274639196 | 382.23197368267647835174 | 2488.330566505 - 010100000062A1D634EF380140BE9F1A2FDDB44440 | 35.35415870080995954879 | 176.77079350404979774397 | 4359.370460594 - 010100000052B81E85EB510140355EBA490CB24440 | 33.12290506987740864904 | 132.49162027950963459615 | 3703.664449828 - 0101000000C286A757CA320140736891ED7CAF4440 | 65.45251754279248087849 | 196.35755262837744263547 | 2512.092358644 - 01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792471345639225 | 125.67855584942691278449 | 2926.25725244 - 0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726651871279586 | 53.54905726651871279586 | 3744.515577777 + the_geom | h | hpop | dist +--------------------------------------------+-------------------------+--------------------------+--------------- + 01010000001361C3D32B650140DD24068195B34440 | 1.51078258364442906811 | 12.08626066915543254492 | 4964.71445918 + 01010000002497FF907EFB0040713D0AD7A3B04440 | 98.29730954188030446151 | 688.08116679316213123058 | 99.95514192 + 0101000000A167B3EA733501401D5A643BDFAF4440 | 63.70532894705201858635 | 382.23197368231211151808 | 2488.33056652 + 010100000062A1D634EF380140BE9F1A2FDDB44440 | 35.35415870089041483312 | 176.77079350445207416562 | 4359.37046061 + 010100000052B81E85EB510140355EBA490CB24440 | 33.12290506980103110470 | 132.49162027920412441880 | 3703.66444985 + 0101000000C286A757CA320140736891ED7CAF4440 | 65.45251754269189849135 | 196.35755262807569547404 | 2512.09235866 + 01010000007DD0B359F5390140C976BE9F1AAF4440 | 62.83927792460778869550 | 125.67855584921557739101 | 2926.25725246 + 0101000000D237691A140D01407E6FD39FFDB44440 | 53.54905726665024972591 | 53.54905726665024972591 | 3744.51557778 (8 rows) diff --git a/src/pg/test/expected/09_voronoi_test.out b/src/pg/test/expected/09_voronoi_test.out index 58b9e50..6f11a2a 100644 --- a/src/pg/test/expected/09_voronoi_test.out +++ b/src/pg/test/expected/09_voronoi_test.out @@ -1,7 +1,5 @@ -SET client_min_messages TO WARNING; +\pset format unaligned \set ECHO none - avg_area ----------------------- - 0.000178661700690617 +within_tolerance +t (1 row) - diff --git a/src/pg/test/sql/09_voronoi_test.sql b/src/pg/test/sql/09_voronoi_test.sql index 032eeeb..88c84a4 100644 --- a/src/pg/test/sql/09_voronoi_test.sql +++ b/src/pg/test/sql/09_voronoi_test.sql @@ -1,4 +1,4 @@ -SET client_min_messages TO WARNING; +\pset format unaligned \set ECHO none WITH a AS ( @@ -11,5 +11,5 @@ b as( FROM a ) SELECT - avg(st_area(result)) as avg_area + abs(avg(st_area(result)) - 0.000178661700690617) < 1e-6 as within_tolerance FROM b; diff --git a/src/py/crankshaft/requirements.txt b/src/py/crankshaft/requirements.txt index 88c0a9e..f573571 100644 --- a/src/py/crankshaft/requirements.txt +++ b/src/py/crankshaft/requirements.txt @@ -1,5 +1,5 @@ -joblib==0.8.3 -numpy==1.6.1 -scipy==0.14.0 +joblib==0.9.4 +numpy==1.11.0 +scipy==0.17.0 pysal==1.14.3 -scikit-learn==0.14.1 +scikit-learn==0.17.0 diff --git a/src/py/crankshaft/setup.py b/src/py/crankshaft/setup.py index ffbdc19..9fdba30 100644 --- a/src/py/crankshaft/setup.py +++ b/src/py/crankshaft/setup.py @@ -41,7 +41,7 @@ setup( # The choice of component versions is dictated by what's # provisioned in the production servers. # IMPORTANT NOTE: please don't change this line. Instead issue a ticket to systems for evaluation. - install_requires=['joblib==0.8.3', 'numpy==1.6.1', 'scipy==0.14.0', 'pysal==1.14.3', 'scikit-learn==0.14.1'], + install_requires=['joblib==0.9.4', 'numpy==1.11.0', 'scipy==0.17.0', 'pysal==1.14.3', 'scikit-learn==0.17.0'], requires=['pysal', 'numpy', 'sklearn'],