From d49313a95854531d218f249bd2d993931c338cb8 Mon Sep 17 00:00:00 2001 From: Stuart Lynn Date: Thu, 3 Mar 2016 15:50:32 -0500 Subject: [PATCH] docker implementation for running tests --- Dockerfile | 193 ++++++++++++++++++++++++++++++++++++++++++++++ init_db.sh | 77 ++++++++++++++++++ init_db_script.sh | 3 + pgpass | 1 + postgresql.sh | 5 ++ run_tests.sh | 23 ++++++ 6 files changed, 302 insertions(+) create mode 100644 Dockerfile create mode 100755 init_db.sh create mode 100755 init_db_script.sh create mode 100644 pgpass create mode 100755 postgresql.sh create mode 100755 run_tests.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7daa4d4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,193 @@ +# PostgreSQL GIS stack +# +# This image includes the following tools +# - PostgreSQL 9.5 +# - PostGIS 2.2 with raster, topology and sfcgal support +# - OGR Foreign Data Wrapper +# - PgRouting +# - PDAL master +# - PostgreSQL PointCloud version master +# +# Version 1.7 + +FROM phusion/baseimage +MAINTAINER Vincent Picavet, vincent.picavet@oslandia.com + +# Set correct environment variables. +ENV HOME /root + +# Regenerate SSH host keys. baseimage-docker does not contain any, so you +# have to do that yourself. You may also comment out this instruction; the +# init system will auto-generate one during boot. +RUN /etc/my_init.d/00_regen_ssh_host_keys.sh + +# Use baseimage-docker's init system. +CMD ["/sbin/my_init"] + + +RUN apt-get update && apt-get install -y wget ca-certificates + +# Use APT postgresql repositories for 9.5 version +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ wheezy-pgdg main 9.5" > /etc/apt/sources.list.d/pgdg.list && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + +# packages needed for compilation +RUN apt-get update + +RUN apt-get install -y autoconf build-essential cmake docbook-mathml docbook-xsl libboost-dev libboost-thread-dev libboost-filesystem-dev libboost-system-dev libboost-iostreams-dev libboost-program-options-dev libboost-timer-dev libcunit1-dev libgdal-dev libgeos++-dev libgeotiff-dev libgmp-dev libjson0-dev libjson-c-dev liblas-dev libmpfr-dev libopenscenegraph-dev libpq-dev libproj-dev libxml2-dev postgresql-server-dev-9.5 xsltproc git build-essential wget + +RUN add-apt-repository ppa:fkrull/deadsnakes &&\ + apt-get update &&\ + apt-get install -y python3.2 + + + + +# application packages +RUN apt-get install -y postgresql-9.5 postgresql-plpython-9.5 + + +# Download and compile CGAL +RUN wget https://gforge.inria.fr/frs/download.php/file/32994/CGAL-4.3.tar.gz &&\ + tar -xzf CGAL-4.3.tar.gz &&\ + cd CGAL-4.3 &&\ + mkdir build && cd build &&\ + cmake .. &&\ + make -j3 && make install + +# orig sfcgal method +# download and compile SFCGAL +# RUN git clone https://github.com/Oslandia/SFCGAL.git +# RUN cd SFCGAL && cmake . && make -j3 && make install +# # cleanup +# RUN rm -Rf SFCGAL + +# andrewxhill fix for stable sfcgal version +RUN wget https://github.com/Oslandia/SFCGAL/archive/v1.2.0.tar.gz +RUN tar -xzf v1.2.0.tar.gz +RUN cd SFCGAL-1.2.0 && cmake . && make -j 1 && make install +RUN rm -Rf v1.2.0.tar.gz SFCGAL-1.2.0 + +# download and install GEOS 3.5 +RUN wget http://download.osgeo.org/geos/geos-3.5.0.tar.bz2 &&\ + tar -xjf geos-3.5.0.tar.bz2 &&\ + cd geos-3.5.0 &&\ + ./configure && make && make install &&\ + cd .. && rm -Rf geos-3.5.0 geos-3.5.0.tar.bz2 + +# Download and compile PostGIS +RUN wget http://download.osgeo.org/postgis/source/postgis-2.2.0.tar.gz +RUN tar -xzf postgis-2.2.0.tar.gz +RUN cd postgis-2.2.0 && ./configure --with-sfcgal=/usr/local/bin/sfcgal-config --with-geos=/usr/local/bin/geos-config +RUN cd postgis-2.2.0 && make && make install +# cleanup +RUN rm -Rf postgis-2.2.0.tar.gz postgis-2.2.0 + +# Download and compile pgrouting +RUN git clone https://github.com/pgRouting/pgrouting.git &&\ + cd pgrouting &&\ + mkdir build && cd build &&\ + cmake -DWITH_DOC=OFF -DWITH_DD=ON .. &&\ + make -j3 && make install +# cleanup +RUN rm -Rf pgrouting + + + +# Download and compile ogr_fdw +RUN git clone https://github.com/pramsey/pgsql-ogr-fdw.git &&\ + cd pgsql-ogr-fdw &&\ + make && make install &&\ + cd .. && rm -Rf pgsql-ogr-fdw + +# Compile PDAL +RUN git clone https://github.com/PDAL/PDAL.git pdal +RUN mkdir PDAL-build && \ + cd PDAL-build && \ + cmake ../pdal && \ + make -j3 && \ + make install +# cleanup +RUN rm -Rf pdal && rm -Rf PDAL-build + +# Compile PointCloud +RUN git clone https://github.com/pramsey/pointcloud.git +RUN cd pointcloud && ./autogen.sh && ./configure && make -j3 && make install +# cleanup +RUN rm -Rf pointcloud + + +RUN git clone https://github.com/CartoDB/cartodb-postgresql.git &&\ + cd cartodb-postgresql &&\ + make all install &&\ + cd .. && rm -Rf cartodb-postgresql + +# install pip + +RUN apt-get -y install python-dev python-pip liblapack-dev + +# +RUN pip install numpy +RUN pip install pandas + +RUN apt-get -y install gfortran libyaml-dev + +RUN pip install scipy +RUN pip install theano +RUN pip install keras +RUN pip install sklearn +# get compiled libraries recognized +RUN ldconfig + +# clean packages + +# all -dev packages +# RUN apt-get remove -y --purge autotools-dev libgeos-dev libgif-dev libgl1-mesa-dev libglu1-mesa-dev libgnutls-dev libgpg-error-dev libhdf4-alt-dev libhdf5-dev libicu-dev libidn11-dev libjasper-dev libjbig-dev libjpeg8-dev libjpeg-dev libjpeg-turbo8-dev libkrb5-dev libldap2-dev libltdl-dev liblzma-dev libmysqlclient-dev libnetcdf-dev libopenthreads-dev libp11-kit-dev libpng12-dev libpthread-stubs0-dev librtmp-dev libspatialite-dev libsqlite3-dev libssl-dev libstdc++-4.8-dev libtasn1-6-dev libtiff5-dev libwebp-dev libx11-dev libx11-xcb-dev libxau-dev libxcb1-dev libxcb-dri2-0-dev libxcb-dri3-dev libxcb-glx0-dev libxcb-present-dev libxcb-randr0-dev libxcb-render0-dev libxcb-shape0-dev libxcb-sync-dev libxcb-xfixes0-dev libxdamage-dev libxdmcp-dev libxerces-c-dev libxext-dev libxfixes-dev libxshmfence-dev libxxf86vm-dev linux-libc-dev manpages-dev mesa-common-dev libgcrypt11-dev unixodbc-dev uuid-dev x11proto-core-dev x11proto-damage-dev x11proto-dri2-dev x11proto-fixes-dev x11proto-gl-dev x11proto-input-dev x11proto-kb-dev x11proto-xext-dev x11proto-xf86vidmode-dev xtrans-dev zlib1g-dev + +# installed packages +# RUN apt-get remove -y --purge autoconf build-essential cmake docbook-mathml docbook-xsl libboost-dev libboost-filesystem-dev libboost-timer-dev libcgal-dev libcunit1-dev libgdal-dev libgeos++-dev libgeotiff-dev libgmp-dev libjson0-dev libjson-c-dev liblas-dev libmpfr-dev libopenscenegraph-dev libpq-dev libproj-dev libxml2-dev postgresql-server-dev-9.5 xsltproc git build-essential wget + +# additional compilation packages +# RUN apt-get remove -y --purge automake m4 make + + + + +# ---------- SETUP -------------- + +# add a baseimage PostgreSQL init script +RUN mkdir /etc/service/postgresql +ADD postgresql.sh /etc/service/postgresql/run + +# Adjust PostgreSQL configuration so that remote connections to the +# database are possible. +RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/9.5/main/pg_hba.conf + +# And add ``listen_addresses`` to ``/etc/postgresql/9.5/main/postgresql.conf`` +RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf + +# Expose PostgreSQL +EXPOSE 5432 + +# Add VOLUMEs to allow backup of config, logs and databases +VOLUME ["/data", "/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"] + +# Add pip + + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + + +# add database setup upon image start +ADD pgpass /root/.pgpass +RUN chmod 700 /root/.pgpass +RUN mkdir -p /etc/my_init.d +ADD init_db_script.sh /etc/my_init.d/init_db_script.sh +ADD init_db.sh /root/init_db.sh +ADD run_tests.sh /root/run_tests.sh + +# ---------- Final cleanup -------------- +# +# Clean up APT when done. +# RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/init_db.sh b/init_db.sh new file mode 100755 index 0000000..2d06d3e --- /dev/null +++ b/init_db.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# wait for pg server to be ready +echo "Waiting for PostgreSQL to run..." +sleep 1 +while ! /usr/bin/pg_isready -q +do + sleep 1 + echo -n "." +done + +# PostgreSQL running +echo "PostgreSQL running, initializing database." + +# PostgreSQL user +# +# create postgresql user pggis + +/sbin/setuser postgres /usr/bin/psql -c "CREATE USER pggis with SUPERUSER PASSWORD 'pggis';" +/sbin/setuser postgres /usr/bin/psql -c "CREATE role publicuser;" +# == Auto restore dumps == +# +# If we find some postgresql dumps in /data/restore, then we load it +# in new databases +shopt -s nullglob +for f in /data/restore/*.backup +do + echo "Found database dump to restore : $f" + DBNAME=$(basename -s ".backup" "$f") + echo "Creating a new database $DBNAME.." + /usr/bin/psql -U pggis -h localhost -c "CREATE DATABASE $DBNAME WITH OWNER = pggis ENCODING = 'UTF8' TEMPLATE = template0 CONNECTION LIMIT = -1;" postgres + /usr/bin/psql -U pggis -h localhost -w -c "CREATE EXTENSION citext; CREATE EXTENSION pg_trgm; CREATE EXTENSION btree_gist; CREATE EXTENSION hstore; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION unaccent; CREATE EXTENSION postgres_fdw; CREATE EXTENSION pgcrypto; CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology; CREATE EXTENSION pgrouting; CREATE EXTENSION pointcloud; CREATE EXTENSION pointcloud_postgis; CREATE EXTENSION postgis_sfcgal; drop type if exists texture; create type texture as (url text,uv float[][]);CREATE ROLE publicuser;" $DBNAME + +# /usr/bin/psql -U pggis -h localhost -w -f /usr/share/postgresql/9.5/contrib/postgis-2.1/sfcgal.sql -d $DBNAME + + + echo "Restoring database $DBNAME.." + /usr/bin/pg_restore -U pggis -h localhost -d $DBNAME -w "$f" + + echo "creating public user" + /usr/bin/psql -U pggis -h localhost -w -c "CREATE ROLE publicuser;" + + echo "Restore done." + +done + +# == Auto restore SQL backups == +# +# If we find some postgresql sql scripts /data/restore, then we load it +# in new databases +shopt -s nullglob +for f in /data/restore/*.sql +do + echo "Found database SQL dump to restore : $f" + DBNAME=$(basename -s ".sql" "$f") + echo "Creating a new database $DBNAME.." + /usr/bin/psql -U pggis -h localhost -c "CREATE DATABASE $DBNAME WITH OWNER = pggis ENCODING = 'UTF8' TEMPLATE = template0 CONNECTION LIMIT = -1;" postgres + /usr/bin/psql -U pggis -h localhost -w -c "CREATE EXTENSION citext; CREATE EXTENSION pg_trgm; CREATE EXTENSION btree_gist; CREATE EXTENSION hstore; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION unaccent; CREATE EXTENSION postgres_fdw; CREATE EXTENSION pgcrypto; CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology; CREATE EXTENSION postgis_sfcgal; CREATE EXTENSION pgrouting; CREATE EXTENSION pointcloud; CREATE EXTENSION pointcloud_postgis; drop type if exists texture; create type texture as (url text,uv float[][]);" $DBNAME +# /usr/bin/psql -U pggis -h localhost -w -f /usr/share/postgresql/9.5/contrib/postgis-2.1/sfcgal.sql -d $DBNAME + echo "Restoring database $DBNAME.." + /usr/bin/psql -U pggis -h localhost -d $DBNAME -w -f "$f" + echo "Restore done." +done + +# == create new database pggis == +echo "Creating a new empty database..." +# create user and main database +/usr/bin/psql -U pggis -h localhost -c "CREATE DATABASE pggis WITH OWNER = pggis ENCODING = 'UTF8' TEMPLATE = template0 CONNECTION LIMIT = -1;" postgres + +# activate all needed extension in pggis database +/usr/bin/psql -U pggis -h localhost -w -c "CREATE EXTENSION citext; CREATE EXTENSION pg_trgm; CREATE EXTENSION btree_gist; CREATE EXTENSION hstore; CREATE EXTENSION fuzzystrmatch; CREATE EXTENSION unaccent; CREATE EXTENSION postgres_fdw; CREATE EXTENSION pgcrypto; CREATE EXTENSION plpythonu; CREATE EXTENSION postgis; CREATE EXTENSION postgis_topology; CREATE EXTENSION postgis_sfcgal; CREATE EXTENSION pgrouting; CREATE EXTENSION pointcloud; CREATE EXTENSION pointcloud_postgis; drop type if exists texture; +create type texture as (url text,uv float[][]);" pggis +#/usr/bin/psql -U pggis -h localhost -w -f /usr/share/postgresql/9.5/contrib/postgis-2.1/sfcgal.sql -d pggis + +echo "Database initialized. Connect from host with :" +echo "psql -h localhost -p -U pggis -W pggis" +echo "Get value with 'docker ps'" diff --git a/init_db_script.sh b/init_db_script.sh new file mode 100755 index 0000000..b8b6e89 --- /dev/null +++ b/init_db_script.sh @@ -0,0 +1,3 @@ +#!/bin/sh +# Script for my_init.d, so as to run database init without blocking +/root/init_db.sh & diff --git a/pgpass b/pgpass new file mode 100644 index 0000000..11a29d9 --- /dev/null +++ b/pgpass @@ -0,0 +1 @@ +localhost:5432:*:pggis:pggis diff --git a/postgresql.sh b/postgresql.sh new file mode 100755 index 0000000..e457e81 --- /dev/null +++ b/postgresql.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# `/sbin/setuser postgres` runs the given command as the user `postgres`. +# If you omit that part, the command will be run as root. +rm -rf /etc/ssl/private-copy; mkdir /etc/ssl/private-copy; mv /etc/ssl/private/* /etc/ssl/private-copy/; rm -r /etc/ssl/private; mv /etc/ssl/private-copy /etc/ssl/private; chmod -R 0700 /etc/ssl/private; chown -R postgres /etc/ssl/private +exec /sbin/setuser postgres /usr/lib/postgresql/9.5/bin/postgres -D /var/lib/postgresql/9.5/main -c config_file=/etc/postgresql/9.5/main/postgresql.conf >> /var/log/postgresql.log 2>&1 diff --git a/run_tests.sh b/run_tests.sh new file mode 100755 index 0000000..3eefa51 --- /dev/null +++ b/run_tests.sh @@ -0,0 +1,23 @@ +#!/bin/bash +/sbin/my_init & + +echo "Waiting for PostgreSQL to run..." +sleep 1 +while ! /usr/bin/pg_isready -q +do + sleep 1 + echo -n "." +done + +cd /crankshaft/pg +make install +PGUSER=pggis PGPASSOWRD=pggis PGHOST=localhost make installcheck + + + +if [ "$?" -eq "0" ] +then + echo "PASSED" +else + cat /crankshaft/pg/test/0.0.1/regression.diffs +fi