cartodb4/dataservices-api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Server: Simplify PG12/Postgis 3 interaction

Browse Source
This commit is contained in:
Raúl Marín
2020-04-07 12:12:05 +02:00
parent 4b6c5a23db
commit ef7c158e4b
8 changed files with 64 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
server/extension/.gitignore vendored
View File

@@ -6,4 +6,6 @@ cdb_geocoder_server--0.1.0.sql
cdb_geocoder_server--0.2.0.sql
cdb_geocoder_server--0.3.0.sql
cdb_geocoder_server--0.4.0.sql
cdb_geocoder_server--0.5.0.sql
cdb_geocoder_server--0.5.0.sql
cdb_dataservices_server.control
./test/sql/366_empty_table_test.sql

154
server/extension/Makefile
View File

@@ -1,17 +1,32 @@
# Makefile to generate the extension out of separate sql source files.
# Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
EXTENSION = cdb_dataservices_server
EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
EXTVERSION = 0.38.0
# The new version to be generated from templates
SED = sed
ERB = erb
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
AWK = awk
SED ?= sed
ERB ?= erb
AWK ?= awk
PG_CONFIG ?= pg_config
# Parallel support macros
PG_CONFIG = pg_config
PG_CONFIG ?= pg_config
PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
# PG12 compatibility
PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
PLPYTHONU := plpythonu
POSTGIS := postgis
ifeq ($(PG_12_GE), true)
PLPYTHONU := plpython3u
POSTGIS := postgis, postgis_raster
endif
REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
REPLACEMENTS_TEST_REVERSE = -e 's/$(PLPYTHONU)/@@plpythonu@@/g'
NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
TEST_DIR = test
@@ -29,48 +44,29 @@ SOURCES_DATA_DIR = sql/
SOURCES_DATA = $(wildcard sql/*.sql)
# postgres build stuff
PG_CONFIG = pg_config
# The targets listed under REGRESS_PREP are executed before regress
PYTHON_TESTS = ./test/sql/366_empty_table_test.sql
REGRESS_PREP = $(PYTHON_TESTS)
./test/sql/366_empty_table_test.sql: ./test/sql/366_empty_table_test.sql.template
$(SED) $(REPLACEMENTS) $@.template > $@
PGXS := $(shell $(PG_CONFIG) --pgxs)
include $(PGXS)
# PG12 compatibility
PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
PLPYTHONU := plpythonu
ifeq ($(PG_12_GE), true)
PLPYTHONU := plpython3u
endif
REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
ifneq ($(NEW_VERSION),)
REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
endif
$(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
rm -f $@
cat $(SOURCES_DATA_DIR)/*.sql >> $@
ifeq ($(PG_PARALLEL), 0)
# Remove PARALLEL in aggregates and functions
$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
mv $(TMPFILE) $@
endif
# $(EXTENSION).control: $(EXTENSION).control.in Makefile
# $(SED) $(REPLACEMENTS) $< > $@
.PHONY: all
all: $(DATA)
.PHONY: release
release: $(EXTENSION).control $(SOURCES_DATA)
release: $(SOURCES_DATA)
test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
for f in $(wildcard *.sql); do \
git mv $${f} old_versions/$${f}; \
done
$(SED) -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' $(EXTENSION).control
git add $(EXTENSION).control
git mv *.sql old_versions
cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
@@ -84,78 +80,32 @@ release: $(EXTENSION).control $(SOURCES_DATA)
.PHONY: devclean
devclean:
rm -f $(NEW_EXTENSION_ARTIFACT)
rm -f $(PYTHON_TESTS)
clean: restore_copies
# If needed remove PARALLEL tags from the release files
release_remove_parallel_deploy:
ifeq ($(PG_PARALLEL), 0)
# Replace variables (and PARALLEL tags if necessary) and deploy files
.PHONY: replace_variables_and_deploy
replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
for n in $(wildcard old_versions/*.sql *.sql); do \
$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
mv $(TMPFILE) $$n; \
if [ "$(PG_PARALLEL)" -eq "0" ]; then \
$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
mv $(TMPFILE) $$n; \
fi; \
$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
done
endif
.PHONY: $(EXTENSION).control
$(EXTENSION).control:
$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
# Install the current release into the PostgreSQL extensions directory
deploy: release_remove_parallel_deploy
.PHONY: deploy
deploy: replace_variables_and_deploy $(EXTENSION).control
$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
restore_copies:
# tests
for f in $(basename $(wildcard test/sql/*.copy)); do \
cat $${f}.copy > $${f}; \
done
for f in $(basename $(wildcard test/expected/*.copy)); do \
cat $${f}.copy > $${f}; \
done
rm -f test/sql/*.copy;
rm -f test/expected/*.copy;
# data
for f in $(basename $(wildcard sql/*.copy)); do \
cat $${f}.copy > $${f}; \
done
rm -f sql/*.copy;
# old_versions
for f in $(basename $(wildcard old_versions/*.copy)); do \
cat $${f}.copy > $${f}; \
done
rm -f old_versions/*.copy;
# current scripts
for f in $(basename $(wildcard *.copy)); do \
cat $${f}.copy > $${f}; \
done
rm -f *.copy;
# %.sql:
# $(SED) $(REPLACEMENTS) $< > $@
# Replacing variables defined within test files
replace_variables: restore_copies
# tests
for f in $(sort $(wildcard test/sql/*test.sql)); do \
sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
done
for f in $(sort $(wildcard test/expected/*test.out)); do \
sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
done
# data
for f in $(SOURCES_DATA); do \
sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
done
# old_versions
for f in $(wildcard old_versions/*.sql); do \
sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
done
# current scripts
for f in $(wildcard *.sql); do \
sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \
done
sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $(EXTENSION).control;
install: replace_variables deploy
reinstall: install
psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;"
.PHONY: install
install: deploy
@echo "Install overriden"

4
server/extension/cdb_dataservices_server.control → server/extension/cdb_dataservices_server.control.in
View File

@@ -1,5 +1,5 @@
comment = 'CartoDB dataservices server extension'
default_version = '0.38.0'
requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
default_version = '@@EXTVERSION@@'
requires = '@@plpythonu@@, plproxy, @@postgis@@, cdb_geocoder'
superuser = true
schema = cdb_dataservices_server

11
server/extension/sql/05_postgis3_adapter.sql
View File

@@ -1,11 +0,0 @@
-- PG12_DEPRECATED
-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
CREATE TYPE cdb_dataservices_server.geomval AS (
geom geometry,
val double precision
);
END IF;
END$$;

11
server/extension/test/expected/00_install_test.out
View File

@@ -1,13 +1,9 @@
-- Only show warning or error messages in the tests output
\set QUIET on
SET client_min_messages TO WARNING;
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION @@plpythonu@@;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
-- Install the extension
CREATE EXTENSION cdb_dataservices_server;
CREATE EXTENSION cdb_dataservices_server CASCADE;
\unset QUIET
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
cdb_conf_setconf
@@ -71,6 +67,7 @@ BEGIN
END
$$
LANGUAGE plpgsql;
CREATE FUNCTION
-- Set user quota
SELECT cartodb.CDB_SetUserQuotaInBytes(0);
cdb_setuserquotainbytes

0
server/extension/test/expected/366_empty_table_test.out → server/extension/test/expected/366_empty_table_test_plpython.out
View File

10
server/extension/test/sql/00_install_test.sql
View File

@@ -1,14 +1,10 @@
-- Only show warning or error messages in the tests output
\set QUIET on
SET client_min_messages TO WARNING;
-- Install dependencies
CREATE EXTENSION postgis;
CREATE EXTENSION @@plpythonu@@;
CREATE EXTENSION plproxy;
CREATE EXTENSION cartodb;
CREATE EXTENSION cdb_geocoder;
-- Install the extension
CREATE EXTENSION cdb_dataservices_server;
CREATE EXTENSION cdb_dataservices_server CASCADE;
\unset QUIET
-- Mock the redis server connection to point to this very test db
SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');

0
server/extension/test/sql/366_empty_table_test.sql → server/extension/test/sql/366_empty_table_test.sql.template
View File