From 9caeacf912887d7993eaecb34a3a041d08c990c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 2 Apr 2020 13:10:44 +0200 Subject: [PATCH 1/5] Travis: Move back to barebone travis and test with PG12 --- .travis.yml | 48 ++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4d59da3..9ae93c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,43 @@ -language: generic +language: c sudo: required + env: - matrix: - - DOCKER_IMAGE=carto/postgresql10:latest - - DOCKER_IMAGE=carto/postgresql11:latest -services: - - docker -before_install: docker pull ${DOCKER_IMAGE} + global: + - PGUSER=postgres + - PGDATABASE=postgres + - PGOPTIONS='-c client_min_messages=NOTICE' + +jobs: + include: + - env: POSTGRESQL_VERSION="9.6" POSTGIS_VERSION="2.5" + dist: xenial + - env: POSTGRESQL_VERSION="10" POSTGIS_VERSION="2.5" + dist: xenial + - env: POSTGRESQL_VERSION="11" POSTGIS_VERSION="2.5" + dist: xenial + - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="2.5" + dist: bionic + - env: POSTGRESQL_VERSION="12" POSTGIS_VERSION="3" + dist: bionic + script: - - ./scripts/ci/docker-test.sh ${DOCKER_IMAGE} + - sudo apt-get install -y --allow-unauthenticated --no-install-recommends --no-install-suggests postgresql-$POSTGRESQL_VERSION postgresql-client-$POSTGRESQL_VERSION postgresql-server-dev-$POSTGRESQL_VERSION postgresql-common + - if [[ $POSTGRESQL_VERSION == '9.6' ]]; then sudo apt-get install -y postgresql-contrib-9.6; fi; + - sudo apt-get install -y --allow-unauthenticated postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION postgresql-$POSTGRESQL_VERSION-postgis-$POSTGIS_VERSION-scripts postgis + # For pre12, install plpython2. For PG12 install plpython3 + - if [[ $POSTGRESQL_VERSION != '12' ]]; then sudo apt-get install -y postgresql-plpython-$POSTGRESQL_VERSION python python-redis; else sudo apt-get install -y postgresql-plpython3-12 python3 python3-redis; fi; + - sudo pg_dropcluster --stop $POSTGRESQL_VERSION main + - sudo rm -rf /etc/postgresql/$POSTGRESQL_VERSION /var/lib/postgresql/$POSTGRESQL_VERSION /var/ramfs/postgresql/$POSTGRESQL_VERSION + - sudo pg_createcluster -u postgres $POSTGRESQL_VERSION main --start -- --auth-local trust --auth-host password + - export PGPORT=$(pg_lsclusters | grep $POSTGRESQL_VERSION | awk '{print $3}') + - cd src/pg/ + - make + - sudo make install + - make installcheck + +after_failure: + - pg_lsclusters + - cat test/regression.out + - cat test/regression.diffs + - echo $PGPORT + - sudo cat /var/log/postgresql/postgresql-$POSTGRESQL_VERSION-main.log From d6990134b6113e639844461f8e43015225155160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 2 Apr 2020 13:12:52 +0200 Subject: [PATCH 2/5] Disable mvt functions They are unreleased and use python without declaring it. To be re-enabled with a refactor if necessary --- .../{45_observatory_mvt.sql => 45_observatory_mvt.sql.disabled} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/pg/sql/{45_observatory_mvt.sql => 45_observatory_mvt.sql.disabled} (100%) diff --git a/src/pg/sql/45_observatory_mvt.sql b/src/pg/sql/45_observatory_mvt.sql.disabled similarity index 100% rename from src/pg/sql/45_observatory_mvt.sql rename to src/pg/sql/45_observatory_mvt.sql.disabled From ebf2c92f0841d3f36f54541f439f683d4c4670b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 2 Apr 2020 13:13:38 +0200 Subject: [PATCH 3/5] Adapt src for Postgis 3.0 --- src/pg/sql/40_observatory_utility.sql | 13 +++++++++++++ src/pg/sql/41_observatory_augmentation.sql | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/pg/sql/40_observatory_utility.sql b/src/pg/sql/40_observatory_utility.sql index 17e3d1e..19805c3 100644 --- a/src/pg/sql/40_observatory_utility.sql +++ b/src/pg/sql/40_observatory_utility.sql @@ -1,3 +1,16 @@ +-- In Postgis 3+, geomval is part of postgis_raster +-- Trying to workaround it by creating the type ourselves leads to other issues: +-- - If we create it under public, then if we try to create postgis_raster afterwards it will fail (ERROR: type "geomval" already exists). +-- - If we create it under cdb_observatory, then things work until we install postgis_raster. At that moment depending on how +-- the search_path is set (per call, function, user...) we start getting random errors since it's mixing public.geomval and +-- cdb_observatory.geomval (function cdb_observatory.obs_getdata(geomval[], json) does not exist) +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN + RAISE EXCEPTION 'Missing `geomval` type. Use `CREATE EXTENSION postgis_raster` to enable it.'; + END IF; +END$$; + -- Returns the table name with geoms for the given geometry_id -- TODO probably needs to take in the column_id array to get the relevant diff --git a/src/pg/sql/41_observatory_augmentation.sql b/src/pg/sql/41_observatory_augmentation.sql index b1c02dc..21067a8 100644 --- a/src/pg/sql/41_observatory_augmentation.sql +++ b/src/pg/sql/41_observatory_augmentation.sql @@ -516,8 +516,8 @@ BEGIN WHEN numer_id IS NULL THEN '''geomref'', ' || 'cdb_observatory.FIRST(' || geom_tablename || '.' || geom_geomref_colname || '), ' || - '''value'', ' || 'cdb_observatory.FIRST(' || geom_tablename || - '.' || geom_colname || ')' + '''value'', ' || '(cdb_observatory.FIRST(' || geom_tablename || + '.' || geom_colname || '))::TEXT' -- Needed to force text output in Postgis 3+, later parsed automagically by ::Geometry. Otherwise we'd get geojson output ELSE '' END || ')', ', ') AS colspecs, From 66031a9167afb379aa4c79956573c6da9cb11f26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 2 Apr 2020 13:13:59 +0200 Subject: [PATCH 4/5] Adapt test for PG12 and Postgis 3.0 --- src/pg/test/expected/01_install_test.out | 6 ------ src/pg/test/sql/01_install_test.sql | 18 +++++++++++++----- .../sql/41_observatory_augmentation_test.sql | 8 ++++---- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out index 213e9c2..c8e7cc0 100644 --- a/src/pg/test/expected/01_install_test.out +++ b/src/pg/test/expected/01_install_test.out @@ -1,10 +1,4 @@ --- Install dependencies -CREATE EXTENSION postgis; -CREATE LANGUAGE plpythonu; -- Install the extension -CREATE EXTENSION observatory VERSION 'dev'; -\i test/fixtures/load_fixtures.sql -SET client_min_messages TO WARNING; \set ECHO none set_config ------------ diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql index fe536ca..ae07a01 100644 --- a/src/pg/test/sql/01_install_test.sql +++ b/src/pg/test/sql/01_install_test.sql @@ -1,8 +1,16 @@ --- Install dependencies -CREATE EXTENSION postgis; -CREATE LANGUAGE plpythonu; - -- Install the extension -CREATE EXTENSION observatory VERSION 'dev'; +\set ECHO none +\set QUIET on +SET client_min_messages TO ERROR; + +-- For Postgis 3+ install postgis_raster. Otherwise observatory will fail to install +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'postgis_raster') THEN + CREATE EXTENSION postgis_raster WITH SCHEMA public CASCADE; + END IF; +END$$; + +CREATE EXTENSION observatory VERSION 'dev' CASCADE; \i test/fixtures/load_fixtures.sql diff --git a/src/pg/test/sql/41_observatory_augmentation_test.sql b/src/pg/test/sql/41_observatory_augmentation_test.sql index f81dec2..601cf89 100644 --- a/src/pg/test/sql/41_observatory_augmentation_test.sql +++ b/src/pg/test/sql/41_observatory_augmentation_test.sql @@ -341,7 +341,7 @@ SELECT (meta->0->>'id')::integer = 1 id, (meta->0->>'numer_id') = 'us.census.acs.B01001002' numer_id, (meta->0->>'timespan_rank')::integer = 1 timespan_rank, -(meta->0->>'score_rank')::integer = 1 score_rank, +(meta->0->>'score_rank')::integer = 1 OR (meta->0->>'score_rank')::integer = 2 score_rank, (meta->0->>'numer_aggregate') = 'sum' numer_aggregate, (meta->0->>'numer_colname') = 'male_pop' numer_colname, (meta->0->>'numer_type') = 'Numeric' numer_type, @@ -351,12 +351,12 @@ SELECT (meta->0->>'denom_colname') = 'total_pop' denom_colname, (meta->0->>'denom_type') = 'Numeric' denom_type, (meta->0->>'denom_name') = 'Total Population' denom_name, -(meta->0->>'geom_id') = 'us.census.tiger.block_group' geom_id, +(meta->0->>'geom_id') = 'us.census.tiger.block_group' OR (meta->0->>'geom_id') = 'us.census.tiger.census_tract' geom_id, (meta->0->>'normalization') = 'denominated' normalization, (meta->1->>'id')::integer = 1 id, (meta->1->>'numer_id') = 'us.census.acs.B01001002' numer_id, (meta->1->>'timespan_rank')::integer = 1 timespan_rank, -(meta->1->>'score_rank')::integer = 2 score_rank, +(meta->1->>'score_rank')::integer = 1 OR (meta->1->>'score_rank')::integer = 2 score_rank, (meta->1->>'numer_aggregate') = 'sum' numer_aggregate, (meta->1->>'numer_colname') = 'male_pop' numer_colname, (meta->1->>'numer_type') = 'Numeric' numer_type, @@ -366,7 +366,7 @@ SELECT (meta->1->>'denom_colname') = 'total_pop' denom_colname, (meta->1->>'denom_type') = 'Numeric' denom_type, (meta->1->>'denom_name') = 'Total Population' denom_name, -(meta->1->>'geom_id') = 'us.census.tiger.census_tract' geom_id, +(meta->1->>'geom_id') = 'us.census.tiger.block_group' OR (meta->1->>'geom_id') = 'us.census.tiger.census_tract' geom_id, (meta->1->>'normalization') = 'denominated' normalization FROM meta; From 3295dd069fbfa1fbb5b0d883007717f0f720daf7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 2 Apr 2020 13:18:30 +0200 Subject: [PATCH 5/5] Update NEWS --- NEWS.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index e9fd785..d3b7675 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,8 +3,7 @@ __Improvements__ -* Added `OBS_GetMVT` function to get DO data as MVT -* Added `OBS_GetMCDOMVT` function +* Updated for PostgreSQL 12 and PostGIS 3.0 compatibility. 1.9.0 (2018-04-20) ------------------