diff --git a/server/extension/.gitignore b/server/extension/.gitignore index f103abf..756b7a4 100644 --- a/server/extension/.gitignore +++ b/server/extension/.gitignore @@ -6,4 +6,6 @@ cdb_geocoder_server--0.1.0.sql cdb_geocoder_server--0.2.0.sql cdb_geocoder_server--0.3.0.sql cdb_geocoder_server--0.4.0.sql -cdb_geocoder_server--0.5.0.sql \ No newline at end of file +cdb_geocoder_server--0.5.0.sql +cdb_dataservices_server.control +./test/sql/366_empty_table_test.sql diff --git a/server/extension/Makefile b/server/extension/Makefile index c95289a..0e96eda 100644 --- a/server/extension/Makefile +++ b/server/extension/Makefile @@ -1,17 +1,32 @@ # Makefile to generate the extension out of separate sql source files. # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed. EXTENSION = cdb_dataservices_server -EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/") +EXTVERSION = 0.38.0 + # The new version to be generated from templates -SED = sed -ERB = erb -NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql -AWK = awk +SED ?= sed +ERB ?= erb +AWK ?= awk +PG_CONFIG ?= pg_config # Parallel support macros -PG_CONFIG = pg_config +PG_CONFIG ?= pg_config PG_PARALLEL := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) +# PG12 compatibility +PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') +PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) +PLPYTHONU := plpythonu +POSTGIS := postgis +ifeq ($(PG_12_GE), true) +PLPYTHONU := plpython3u +POSTGIS := postgis, postgis_raster +endif + +REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g' +REPLACEMENTS_TEST_REVERSE = -e 's/$(PLPYTHONU)/@@plpythonu@@/g' +NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql + REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql)))) REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out)))) TEST_DIR = test @@ -29,48 +44,29 @@ SOURCES_DATA_DIR = sql/ SOURCES_DATA = $(wildcard sql/*.sql) # postgres build stuff -PG_CONFIG = pg_config + +# The targets listed under REGRESS_PREP are executed before regress +PYTHON_TESTS = ./test/sql/366_empty_table_test.sql +REGRESS_PREP = $(PYTHON_TESTS) + +./test/sql/366_empty_table_test.sql: ./test/sql/366_empty_table_test.sql.template + $(SED) $(REPLACEMENTS) $@.template > $@ + PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) -# PG12 compatibility -PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}') -PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true) -PLPYTHONU := plpythonu -ifeq ($(PG_12_GE), true) -PLPYTHONU := plpython3u -endif - -REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -ifneq ($(NEW_VERSION),) -REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g' -endif $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA) rm -f $@ cat $(SOURCES_DATA_DIR)/*.sql >> $@ -ifeq ($(PG_PARALLEL), 0) -# Remove PARALLEL in aggregates and functions - $(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX)) - $(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \ - -e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE) - mv $(TMPFILE) $@ -endif - -# $(EXTENSION).control: $(EXTENSION).control.in Makefile -# $(SED) $(REPLACEMENTS) $< > $@ .PHONY: all all: $(DATA) .PHONY: release -release: $(EXTENSION).control $(SOURCES_DATA) +release: $(SOURCES_DATA) test -n "$(NEW_VERSION)" # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0 - for f in $(wildcard *.sql); do \ - git mv $${f} old_versions/$${f}; \ - done - $(SED) -i 's/$(EXTVERSION)/$(NEW_VERSION)/g' $(EXTENSION).control - git add $(EXTENSION).control + git mv *.sql old_versions cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql $(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql $(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql @@ -84,78 +80,32 @@ release: $(EXTENSION).control $(SOURCES_DATA) .PHONY: devclean devclean: rm -f $(NEW_EXTENSION_ARTIFACT) + rm -f $(PYTHON_TESTS) -clean: restore_copies - -# If needed remove PARALLEL tags from the release files -release_remove_parallel_deploy: -ifeq ($(PG_PARALLEL), 0) +# Replace variables (and PARALLEL tags if necessary) and deploy files +.PHONY: replace_variables_and_deploy +replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT) + mkdir -p '$(DESTDIR)$(datadir)/extension/'; \ for n in $(wildcard old_versions/*.sql *.sql); do \ $(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \ - $(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \ - mv $(TMPFILE) $$n; \ + if [ "$(PG_PARALLEL)" -eq "0" ]; then \ + $(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \ + mv $(TMPFILE) $$n; \ + fi; \ + $(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \ + mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \ done -endif + + +.PHONY: $(EXTENSION).control +$(EXTENSION).control: + $(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control # Install the current release into the PostgreSQL extensions directory -deploy: release_remove_parallel_deploy +.PHONY: deploy +deploy: replace_variables_and_deploy $(EXTENSION).control $(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/' - $(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/' -restore_copies: - # tests - for f in $(basename $(wildcard test/sql/*.copy)); do \ - cat $${f}.copy > $${f}; \ - done - for f in $(basename $(wildcard test/expected/*.copy)); do \ - cat $${f}.copy > $${f}; \ - done - rm -f test/sql/*.copy; - rm -f test/expected/*.copy; - # data - for f in $(basename $(wildcard sql/*.copy)); do \ - cat $${f}.copy > $${f}; \ - done - rm -f sql/*.copy; - # old_versions - for f in $(basename $(wildcard old_versions/*.copy)); do \ - cat $${f}.copy > $${f}; \ - done - rm -f old_versions/*.copy; - # current scripts - for f in $(basename $(wildcard *.copy)); do \ - cat $${f}.copy > $${f}; \ - done - rm -f *.copy; - - -# %.sql: -# $(SED) $(REPLACEMENTS) $< > $@ - -# Replacing variables defined within test files -replace_variables: restore_copies - # tests - for f in $(sort $(wildcard test/sql/*test.sql)); do \ - sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \ - done - for f in $(sort $(wildcard test/expected/*test.out)); do \ - sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \ - done - # data - for f in $(SOURCES_DATA); do \ - sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \ - done - # old_versions - for f in $(wildcard old_versions/*.sql); do \ - sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \ - done - # current scripts - for f in $(wildcard *.sql); do \ - sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $${f}; \ - done - sed --in-place=.copy -e 's/@@plpythonu@@/$(PLPYTHONU)/g' $(EXTENSION).control; - -install: replace_variables deploy - -reinstall: install - psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;" +.PHONY: install +install: deploy + @echo "Install overriden" diff --git a/server/extension/cdb_dataservices_server.control b/server/extension/cdb_dataservices_server.control.in similarity index 50% rename from server/extension/cdb_dataservices_server.control rename to server/extension/cdb_dataservices_server.control.in index c0d362e..e19b3f8 100644 --- a/server/extension/cdb_dataservices_server.control +++ b/server/extension/cdb_dataservices_server.control.in @@ -1,5 +1,5 @@ comment = 'CartoDB dataservices server extension' -default_version = '0.38.0' -requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder' +default_version = '@@EXTVERSION@@' +requires = '@@plpythonu@@, plproxy, @@postgis@@, cdb_geocoder' superuser = true schema = cdb_dataservices_server diff --git a/server/extension/sql/05_postgis3_adapter.sql b/server/extension/sql/05_postgis3_adapter.sql deleted file mode 100644 index addad2e..0000000 --- a/server/extension/sql/05_postgis3_adapter.sql +++ /dev/null @@ -1,11 +0,0 @@ --- PG12_DEPRECATED --- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster) -DO $$ -BEGIN - IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN - CREATE TYPE cdb_dataservices_server.geomval AS ( - geom geometry, - val double precision - ); - END IF; -END$$; \ No newline at end of file diff --git a/server/extension/test/expected/00_install_test.out b/server/extension/test/expected/00_install_test.out index 5099ede..8bc5c36 100644 --- a/server/extension/test/expected/00_install_test.out +++ b/server/extension/test/expected/00_install_test.out @@ -1,13 +1,9 @@ -- Only show warning or error messages in the tests output +\set QUIET on SET client_min_messages TO WARNING; --- Install dependencies -CREATE EXTENSION postgis; -CREATE EXTENSION @@plpythonu@@; -CREATE EXTENSION plproxy; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder; -- Install the extension -CREATE EXTENSION cdb_dataservices_server; +CREATE EXTENSION cdb_dataservices_server CASCADE; +\unset QUIET -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); cdb_conf_setconf @@ -71,6 +67,7 @@ BEGIN END $$ LANGUAGE plpgsql; +CREATE FUNCTION -- Set user quota SELECT cartodb.CDB_SetUserQuotaInBytes(0); cdb_setuserquotainbytes diff --git a/server/extension/test/expected/366_empty_table_test.out b/server/extension/test/expected/366_empty_table_test_plpython.out similarity index 100% rename from server/extension/test/expected/366_empty_table_test.out rename to server/extension/test/expected/366_empty_table_test_plpython.out diff --git a/server/extension/test/sql/00_install_test.sql b/server/extension/test/sql/00_install_test.sql index f0a1e8e..c8e41eb 100644 --- a/server/extension/test/sql/00_install_test.sql +++ b/server/extension/test/sql/00_install_test.sql @@ -1,14 +1,10 @@ -- Only show warning or error messages in the tests output +\set QUIET on SET client_min_messages TO WARNING; --- Install dependencies -CREATE EXTENSION postgis; -CREATE EXTENSION @@plpythonu@@; -CREATE EXTENSION plproxy; -CREATE EXTENSION cartodb; -CREATE EXTENSION cdb_geocoder; -- Install the extension -CREATE EXTENSION cdb_dataservices_server; +CREATE EXTENSION cdb_dataservices_server CASCADE; +\unset QUIET -- Mock the redis server connection to point to this very test db SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}'); diff --git a/server/extension/test/sql/366_empty_table_test.sql b/server/extension/test/sql/366_empty_table_test.sql.template similarity index 100% rename from server/extension/test/sql/366_empty_table_test.sql rename to server/extension/test/sql/366_empty_table_test.sql.template