From fba43cf3b50b88b7debad0f4d510df691de842ff Mon Sep 17 00:00:00 2001 From: Rafa de la Torre Date: Wed, 6 Apr 2016 20:03:04 +0200 Subject: [PATCH] Initial commit with what I think we need --- .gitignore | 1 + CONTRIBUTING.md | 70 ++++++++++++++++++++++++ LICENSE | 27 +++++++++ Makefile | 59 ++++++++++++++++++++ Makefile.global | 5 ++ NEWS.md | 3 + README.md | 58 ++++++++++++++++++++ RELEASE.md | 69 +++++++++++++++++++++++ doc/.gitkeep | 0 release/.gitkeep | 0 src/pg/Makefile | 54 ++++++++++++++++++ src/pg/observatory.control | 5 ++ src/pg/sql/00_header.sql | 3 + src/pg/sql/10_version.sql | 12 ++++ src/pg/sql/90_permissions.sql | 10 ++++ src/pg/test/expected/01_install_test.out | 5 ++ src/pg/test/fixtures/.gitkeep | 0 src/pg/test/sql/01_install_test.sql | 6 ++ src/pg/test/sql/90_permissions.sql | 0 19 files changed, 387 insertions(+) create mode 100644 .gitignore create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 Makefile.global create mode 100644 NEWS.md create mode 100644 README.md create mode 100644 RELEASE.md create mode 100644 doc/.gitkeep create mode 100644 release/.gitkeep create mode 100644 src/pg/Makefile create mode 100644 src/pg/observatory.control create mode 100644 src/pg/sql/00_header.sql create mode 100644 src/pg/sql/10_version.sql create mode 100644 src/pg/sql/90_permissions.sql create mode 100644 src/pg/test/expected/01_install_test.out create mode 100644 src/pg/test/fixtures/.gitkeep create mode 100644 src/pg/test/sql/01_install_test.sql create mode 100644 src/pg/test/sql/90_permissions.sql diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e43b0f9 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1c3c8c2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,70 @@ +# Development process + +Please read the Working Process/Quickstart Guide in [[README.md]] first. + +For any modification of the observatory-extension, such as adding new features, +refactoring or bug-fixing, a topic branch must be created out of the `develop` +branch and be used for the development process. + +Modifications are done inside `src/pg/sql`. + +Take into account: + +* Tests must be added for any new functionality + (inside `src/pg/test` as well as to + detect any bugs that are being fixed. +* Add or modify the corresponding documentation files in the `doc` folder. +* Naming Conventions: we will use CamelCase with a prefix, as it is done in postgis. E.g: `OBS_GetSegmentationSnapshot` + Prefix function names intended for public use with `OBS_` + and private functions (to be used only internally inside + the extension) with `_OBS_`. + +Once the code is ready to be tested, update the local development installation +with `sudo make install`. +This will update the 'dev' version of the extension in `src/pg/` and +make it available to PostgreSQL. + +Run the tests with `make test`. + +Update extension in a working database with: + +* `ALTER EXTENSION observatory VERSION TO 'current';` + `ALTER EXTENSION observatory VERSION TO 'dev';` + +Note: we keep the current development version install as 'dev' always; +we update through the 'current' alias to allow changing the extension +contents but not the version identifier. This will fail if the +changes involve incompatible function changes such as a different +return type; in that case the offending function (or the whole extension) +should be dropped manually before the update. + +If the extension has not previously been installed in a database, +it can be installed directly with: + +* `CREATE EXTENSION observatory WITH VERSION 'dev';` + +Before proceeding to the release process a peer review of the code is +a must. + +Once the feature or bugfix is completed and all the tests are passing +a Pull-Request shall be created on the topic branch, reviewed by a peer +and then merged back into the `develop` branch when all CI tests pass. + +When the changes in the `develop` branch are to be released in a new +version of the extension, a PR must be created on the `develop` branch. + +The release manage will take hold of the PR at this moment to proceed +to the release process for a new revision of the extension. + +## Relevant development tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +* `sudo make install` will generate the extension scripts for the development + version ('dev'/'current'). + Intended for use by developers. + +* `make test` will run the tests for the installed development extension. + Intended for use by developers. +``` diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..101f587 --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2016, CartoDB +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2b6831c --- /dev/null +++ b/Makefile @@ -0,0 +1,59 @@ +include ./Makefile.global + +EXT_DIR = src/pg + +.PHONY: install +.PHONY: run_tests +.PHONY: release +.PHONY: deploy + +# Generate and install developmet versions of the extension. +# The extension is named 'dev' with a 'current' alias for easily upgrading. +# It requires sudo. +install: ## Generate and install development version of the extension; requires sudo. + $(MAKE) -C $(EXT_DIR) install + +# Run the tests for the installed development extension. +test: ## Run the tests for the development version of the extension + $(MAKE) -C $(EXT_DIR) test + +# Generate a new release into release +release: ## Generate a new release of the extension. Only for telease manager + $(MAKE) -C $(EXT_DIR) release + +# Install the current release. +# It requires sudo. +# Use the RELEASE_VERSION environment variable to deploy a specific version: +# sudo make deploy RELEASE_VERSION=1.0.0 +deploy: ## Deploy a released extension. Only for release manager. Requires sudo. + $(MAKE) -C $(EXT_DIR) deploy + +# Cleanup development extension script files +clean-dev: ## clean up development extension script files + rm -f src/pg/$(EXTENSION)--*.sql + +# Cleanup all releases +clean-releases: ## clean up all releases + rm -f release/$(EXTENSION)--*.sql + rm -f release/$(EXTENSION).control + +# Cleanup current/specific version +clean-release: ## clean up current release + rm -f release/$(RELEASE_VERSION)--*.sql + +# Cleanup all virtual environments +clean-environments: ## clean up all virtual environments + rm -rf envs/* + +clean-all: clean-dev clean-release clean-environments + +help: + @IFS=$$'\n' ; \ + help_lines=(`fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//'`); \ + for help_line in $${help_lines[@]}; do \ + IFS=$$'#' ; \ + help_split=($$help_line) ; \ + help_command=`echo $${help_split[0]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \ + help_info=`echo $${help_split[2]} | sed -e 's/^ *//' -e 's/ *$$//'` ; \ + printf "%-30s %s\n" $$help_command $$help_info ; \ + done diff --git a/Makefile.global b/Makefile.global new file mode 100644 index 0000000..0d13291 --- /dev/null +++ b/Makefile.global @@ -0,0 +1,5 @@ +SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST))) +EXTENSION = observatory +EXTVERSION = $(shell grep default_version $(SELF_DIR)/src/pg/$(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +RELEASE_VERSION ?= $(EXTVERSION) +SED = sed diff --git a/NEWS.md b/NEWS.md new file mode 100644 index 0000000..7d01917 --- /dev/null +++ b/NEWS.md @@ -0,0 +1,3 @@ +0.0.1 (open date) +------------------ +* First iteration of `OBS_GetDemographicSnapshot(location Geometry(Point,4326))`; diff --git a/README.md b/README.md new file mode 100644 index 0000000..cec3000 --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# Observatory extension + +CartoDB extension that implements the functions needed by the Observatory Service. + +## Code organization + +* *doc* documentation +* *src* source code +* - *src/pg* contains the PostgreSQL extension source code +* *release* reseleased versions + +# Development workflow + +We distinguish two roles regarding the development cycle: + +* *developers* will implement new functionality and bugfixes into + the codebase and will request for new releases of the extension. +* A *release manager* will attend these requests and will handle + the release process. The release process is sequential: + no concurrent releases will ever be in the works. + +We use the default `develop` branch as the basis for development. +The `master` branch is used to merge and tag releases to be +deployed in production. + +Developers shall create a new topic branch from `develop` for any new feature +or bugfix and commit their changes to it and eventually merge back into +the `develop` branch. When a new release is required a Pull Request +will be open against the `develop` branch. + +The `develop` pull requests will be handled by the release manage, +who will merge into master where new releases are prepared and tagged. +The `master` branch is the sole responsibility of the release masters +and developers must not commit or merge into it. + +## Development Guidelines + +For a detailed description of the development process please see +the [[CONTRIBUTING.md]] guide. + +Any modification to the source code +shall always be done in a topic branch created from the `develop` branch. + +Tests, documentation and peer code reviews are required for all +modifications. + +The tests (both for SQL and Python) are executed by running, +from the top directory: + +``` +sudo make install +make test +``` +## Release + +The release and deployment process is described in the +[[RELEASE.md]] guide and it is the responsibility of the designated +release manager. diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..f13a660 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,69 @@ +# Release & Deployment Process + +Please read the Working Process/Quickstart Guide in [README.md]] +and the Development guidelines in [[CONTRIBUTING.md]]. + +The release process of a new version of the extension +shall be performed by the designated *Release Manager*. + +Having checked PR to be released it shall be +merged back into the `master` branch to prepare the new release. + +The version number in `pg/observatory.control` must first be updated. +To do so [Semantic Versioning 2.0](http://semver.org/) is in order. + +Thew `NEWS.md` will be updated. + +The next command must be executed to produce the main installation +script for the new release, `release/observatory--X.Y.Z.sql` and +also to copy the python package to `release/python/X.Y.Z/observatory`. + +``` +make release +``` + +Then, the release manager shall produce upgrade and downgrade scripts +to migrate to/from the previous release. In the case of minor/patch +releases this simply consist in extracting the functions that have changed +and placing them in the proper `release/observatory--X.Y.Z--A.B.C.sql` +file. + +The new release can be deployed for staging/smoke tests with this command: + +``` +sudo make deploy +``` + +This will copy the current 'X.Y.Z' released version of the extension to +PostgreSQL. + +The `sudo make deploy` operation can be also used for installing +the new version after it has been released. + +To install a specific version 'X.Y.Z' different from the current one +(which must be present in `releases/`) you can: + +``` +sudo make deploy RELEASE_VERSION=X.Y.Z +``` + + +## Relevant release & deployment tasks available in the Makefile + +``` +* `make help` show a short description of the available targets + +* `make release` will generate a new release (version number defined in + `src/pg/observatory.control`) into `release/`. + Intended for use by the release manager. + +* `sudo make deploy` will install the current release X.Y.Z from the + `release/` files into PostgreSQL and a Python virtual environment + `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. + +* `sudo make deploy RELEASE_VERSION=X.Y.Z` will install specified version + previously generated in `release/` + into PostgreSQL and a Python virtual environment `envs/X.Y.Z`. + Intended for use by the release manager and deployment jobs. +``` diff --git a/doc/.gitkeep b/doc/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/release/.gitkeep b/release/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/pg/Makefile b/src/pg/Makefile new file mode 100644 index 0000000..6dc25f1 --- /dev/null +++ b/src/pg/Makefile @@ -0,0 +1,54 @@ +include ../../Makefile.global + +# Development tasks: +# +# * install generates the control & script files into src/pg/ +# and installs then into the PostgreSQL extensions directory; +# requires sudo. In additionof the current development version +# named 'dev', an alias 'current' is generating for ease of +# update (upgrade to 'current', then to 'dev'). +# * test runs the tests for the currently generated Development +# extension. + +DATA = $(EXTENSION)--dev.sql \ + $(EXTENSION)--current--dev.sql \ + $(EXTENSION)--dev--current.sql + +SOURCES_DATA_DIR = sql +SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) + + +REPLACEMENTS = -e 's/@@VERSION@@/$(EXTVERSION)/g' + +$(DATA): $(SOURCES_DATA) + $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > $@ + +TEST_DIR = test +REGRESS = $(notdir $(basename $(wildcard $(TEST_DIR)/sql/*test.sql))) +REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' + +PG_CONFIG = pg_config +PGXS := $(shell $(PG_CONFIG) --pgxs) +include $(PGXS) + +# This seems to be needed at least for PG 9.3.11 +all: $(DATA) + +test: export PGUSER=postgres +test: installcheck + +# Release tasks + +../../release/$(EXTENSION).control: $(EXTENSION).control + cp $< $@ + +# Prepare new release from the currently installed development version, +# for the current version X.Y.Z (defined in the control file) +# producing the extension script and control files in releases/ +release: ../../release/$(EXTENSION).control $(SOURCES_DATA) + $(SED) $(REPLACEMENTS) $(SOURCES_DATA_DIR)/*.sql > ../../release/$(EXTENSION)--$(EXTVERSION).sql + +# Install the current relese into the PostgreSQL extensions directory +deploy: + $(INSTALL_DATA) ../../release/$(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' + $(INSTALL_DATA) ../../release/*.sql '$(DESTDIR)$(datadir)/extension/' diff --git a/src/pg/observatory.control b/src/pg/observatory.control new file mode 100644 index 0000000..3d17f0c --- /dev/null +++ b/src/pg/observatory.control @@ -0,0 +1,5 @@ +comment = 'CartoDB Observatory backend extension' +default_version = '0.0.1' +requires = 'postgis, cartodb' +superuser = true +schema = cdb_observatory diff --git a/src/pg/sql/00_header.sql b/src/pg/sql/00_header.sql new file mode 100644 index 0000000..875e4b9 --- /dev/null +++ b/src/pg/sql/00_header.sql @@ -0,0 +1,3 @@ +--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES +-- Complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION observatory" to load this file. \quit diff --git a/src/pg/sql/10_version.sql b/src/pg/sql/10_version.sql new file mode 100644 index 0000000..2ca0831 --- /dev/null +++ b/src/pg/sql/10_version.sql @@ -0,0 +1,12 @@ +-- Version number of the extension release +CREATE OR REPLACE FUNCTION cdb_observatory_version() +RETURNS text AS $$ + SELECT '@@VERSION@@'::text; +$$ language 'sql' STABLE STRICT; + +-- Internal identifier of the installed extension instence +-- e.g. 'dev' for current development version +CREATE OR REPLACE FUNCTION _cdb_observatory_internal_version() +RETURNS text AS $$ + SELECT installed_version FROM pg_available_extensions where name='observatory' and pg_available_extensions IS NOT NULL; +$$ language 'sql' STABLE STRICT; diff --git a/src/pg/sql/90_permissions.sql b/src/pg/sql/90_permissions.sql new file mode 100644 index 0000000..9e54061 --- /dev/null +++ b/src/pg/sql/90_permissions.sql @@ -0,0 +1,10 @@ +-- Placeholder for permission tweaks at creation time. +-- Make sure by default there are no permissions for publicuser +-- NOTE: this happens at extension creation time, as part of an implicit transaction. +-- REVOKE ALL PRIVILEGES ON SCHEMA cdb_observatory FROM PUBLIC, publicuser CASCADE; + +-- Grant permissions on the schema to publicuser (but just the schema) +-- GRANT USAGE ON SCHEMA cdb_crankshaft TO publicuser; + +-- Revoke execute permissions on all functions in the schema by default +-- REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory FROM PUBLIC, publicuser; diff --git a/src/pg/test/expected/01_install_test.out b/src/pg/test/expected/01_install_test.out new file mode 100644 index 0000000..99da48c --- /dev/null +++ b/src/pg/test/expected/01_install_test.out @@ -0,0 +1,5 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION cartodb; +-- Install the extension +CREATE EXTENSION observatory VERSION 'dev'; diff --git a/src/pg/test/fixtures/.gitkeep b/src/pg/test/fixtures/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/pg/test/sql/01_install_test.sql b/src/pg/test/sql/01_install_test.sql new file mode 100644 index 0000000..58502b1 --- /dev/null +++ b/src/pg/test/sql/01_install_test.sql @@ -0,0 +1,6 @@ +-- Install dependencies +CREATE EXTENSION postgis; +CREATE EXTENSION cartodb; + +-- Install the extension +CREATE EXTENSION observatory VERSION 'dev'; diff --git a/src/pg/test/sql/90_permissions.sql b/src/pg/test/sql/90_permissions.sql new file mode 100644 index 0000000..e69de29