Merge pull request #597 from CartoDB/development

Client 0.30.0, Server 0.39.0 -> master

.github/workflows/main.yml

@@ -4,6 +4,7 @@ jobs:
   dataservices-api:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         pg_version: [10, 12]
     env:
@@ -48,9 +49,9 @@ jobs:
         timeout-minutes: 5
 
       - name: Run server tests
-        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test_out/regression.diffs && false)"
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test/regression.diffs && false)"
         timeout-minutes: 5
 
       - name: Run client tests
-        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /home/ubuntu/dataservices-api/client/test_out/regression.diffs && false)"
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /dataservices-api/client/test/regression.diffs && false)"
         timeout-minutes: 5

NEWS.md

@@ -1,3 +1,18 @@
+Apr 7th, 2020
+=============
+* Version `0.30.0` of the client extension
+    * PG12: Use postgis_raster instead of custom geomval.
+* Version `0.39.0` of the server extension
+    * PG12: Use postgis_raster instead of custom geomval.
+
+Mar 17th, 2020
+==============
+* Version `0.29.0` of the client extension
+    * Adapted client to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Version `0.38.0` of the server extension
+    * Adapted server to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Update dependencies to fix vulnerabilities
+
 Mar 6th, 2020
 ==============
 * Version `0.23.1` of the Python library

client/.gitignore

@@ -9,3 +9,4 @@ cdb_geocoder_client--0.0.1.sql
 cdb_geocoder_client--0.1.0.sql
 cdb_geocoder_client--0.2.0.sql
 cdb_geocoder_client--0.3.0.sql
+cdb_dataservices_client.control

client/Makefile

@@ -1,27 +1,30 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_client
-EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.30.0
 
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
 # PG12 compatibility
 PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
 PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
 PLPYTHONU := plpythonu
+POSTGIS := postgis
 ifeq ($(PG_12_GE), true)
 PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
 endif
 
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 # OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # DATA =  $(NEW_EXTENSION_ARTIFACT) \
 #   $(OLD_VERSIONS) \
@@ -35,7 +38,7 @@ SOURCES_DATA_DIR = sql/
 
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
 REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
-TEST_DIR = test_out
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -51,7 +54,6 @@ TEMPLATE_FILES = $(wildcard $(TEMPLATE_DIR)/*.erb)
 GENERATED_SQL_FILES = $(patsubst $(TEMPLATE_DIR)/%.erb, $(SOURCES_DATA_DIR)/%.sql, $(TEMPLATE_FILES))
 
 # postgres build stuff
-PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
@@ -62,29 +64,21 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR)/*.sql | \
-	$(SED) -e 's/@@plpythonu@@/$(PLPYTHONU)/g' >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
+	cat $(SOURCES_DATA_DIR)/*.sql >> $@
 
 .PHONY: all
 all: $(DATA)
 
 .PHONY: release
-release: $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control
-	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
+	$(SED) -i -e 's/^EXTVERSION =.*/EXTVERSION = $(NEW_VERSION)/g' Makefile
+	git add Makefile
 	@echo
 	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
 	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
@@ -95,37 +89,30 @@ release: $(EXTENSION).control $(SOURCES_DATA)
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
 	rm -f $(GENERATED_SQL_FILES)
-	rm -rf $(TEST_DIR)
 
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
 
-# Replacing variables defined within test files
-replace_variables:
-	mkdir -p $(TEST_DIR)
-	mkdir -p $(TEST_DIR)/expected
-	mkdir -p $(TEST_DIR)/sql
-	for f in $(REGRESS); do \
-		cat test/sql/$${f}.sql | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		>> $(TEST_DIR)/sql/$${f}.sql; \
-	done
-	for f in $(REGRESS_EXPEC); do \
-		cat test/expected/$${f}.out | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		>> $(TEST_DIR)/expected/$${f}.out; \
-	done
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
-install: replace_variables deploy
+.PHONY: install
+install: deploy
+	@echo "Install overriden"

client/README.md

@@ -37,7 +37,7 @@ psql -U postgres cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db
 and then:
 
 ```sql
-CREATE EXTENSION cdb_dataservices_client;
+CREATE EXTENSION cdb_dataservices_client CASCADE;
 ```
 
 The extension creation in the user's db requires **superuser** privileges.

client/carto-package.json

@@ -4,7 +4,7 @@
     "requires": {
       "postgresql": "^11.0.0",
       "postgis": "^2.5.0.0",
-      "carto_postgresql_ext": "^0.32.0"
+      "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
     }

client/cdb_dataservices_client--0.29.0--0.30.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.30.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+-- In 0.30.0 we removed cdb_dataservices_client.geomval and rely on postgis_raster if necessary
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/cdb_dataservices_client--0.30.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+-- Create cdb_dataservices_client.geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/cdb_dataservices_client.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.29.0'
-requires = 'plproxy, cartodb'
+default_version = '@@EXTVERSION@@'
+requires = 'plproxy, cartodb, @@plpythonu@@, @@postgis@@'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.28.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/old_versions/cdb_dataservices_client--0.29.0--0.28.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/sql/05_utils.sql

@@ -18,16 +18,3 @@ $func$ LANGUAGE plpgsql;
 CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
     SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
 $f$ LANGUAGE sql IMMUTABLE;
-
-
--- PG12_DEPRECATED
--- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
-DO $$
-BEGIN
-    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
-        CREATE TYPE cdb_dataservices_client.geomval AS (
-            geom geometry,
-            val double precision
-        );
-    END IF;
-END$$;

client/test/expected/00_installation_test.out

@@ -1,30 +1,17 @@
 -- Only show warning or error messages in the tests output
-SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION plproxy;
--- Install the extension
-CREATE EXTENSION cdb_dataservices_client;
--- Mock the server connection to point to this very test db
-SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');
+\set ECHO none
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
--- Mock the user configuration
-SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}');
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
--- Mock the server schema
-CREATE SCHEMA cdb_dataservices_server;
--- Create a test user to check permissions
-DROP ROLE IF EXISTS test_regular_user;
-CREATE ROLE test_regular_user;
-GRANT publicuser TO test_regular_user;
-ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_dataservices_client;
+CREATE SCHEMA
+DROP ROLE
+CREATE ROLE
+GRANT ROLE
+ALTER ROLE

client/test/sql/00_installation_test.sql

@@ -1,13 +1,13 @@
 -- Only show warning or error messages in the tests output
+\set ECHO none
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION plproxy;
 
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_client;
+CREATE EXTENSION cdb_dataservices_client CASCADE;
+
+\unset ECHO
+\unset QUIET
 
 -- Mock the server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');

server/extension/.gitignore

@@ -6,4 +6,6 @@ cdb_geocoder_server--0.1.0.sql
 cdb_geocoder_server--0.2.0.sql
 cdb_geocoder_server--0.3.0.sql
 cdb_geocoder_server--0.4.0.sql
-cdb_geocoder_server--0.5.0.sql
+cdb_geocoder_server--0.5.0.sql
+cdb_dataservices_server.control
+./test/sql/366_empty_table_test.sql

server/extension/Makefile

@@ -1,20 +1,35 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
-EXTVERSION = $(shell grep default_version $(EXTENSION).control.in | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.39.0
+
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
+PG_CONFIG ?= pg_config
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+POSTGIS := postgis
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
+endif
+
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+REPLACEMENTS_TEST_REVERSE = -e 's/$(PLPYTHONU)/@@plpythonu@@/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
 REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
-TEST_DIR = test_out
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -27,106 +42,72 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 		 $(OLD_VERSIONS)
 SOURCES_DATA_DIR = sql/
 SOURCES_DATA = $(wildcard sql/*.sql)
-SOURCES_DATA_DIR_OUT = sql_out/
 
 # postgres build stuff
-PG_CONFIG = pg_config
+
+# The targets listed under REGRESS_PREP are executed before regress
+PYTHON_TESTS = ./test/sql/366_empty_table_test.sql
+REGRESS_PREP = $(PYTHON_TESTS)
+
+./test/sql/366_empty_table_test.sql: ./test/sql/366_empty_table_test.sql.template
+	$(SED) $(REPLACEMENTS) $@.template > $@
+
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-# PG12 compatibility
-PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
-PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
-PLPYTHONU := plpythonu
-ifeq ($(PG_12_GE), true)
-PLPYTHONU := plpython3u
-endif
-
-REPLACEMENTS = -e 's/@@plpythonu@@/$(PLPYTHONU)/g'
-ifneq ($(NEW_VERSION),)
-REPLACEMENTS += -e 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-endif
 
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
-	cat $(SOURCES_DATA_DIR_OUT)/*.sql >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
-
-$(EXTENSION).control: $(EXTENSION).control.in Makefile
-	$(SED) $(REPLACEMENTS)  $< > $@
+	cat $(SOURCES_DATA_DIR)/*.sql >> $@
 
 .PHONY: all
-all: replace_variables $(DATA)
+all: $(DATA)
 
 .PHONY: release
-release: replace_variables $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
-	git add $(EXTENSION).control
-	cat $(SOURCES_DATA_DIR_OUT)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
+	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
+	$(SED) -i -e 's/^EXTVERSION =.*/EXTVERSION = $(NEW_VERSION)/g' Makefile
+	git add Makefile
 	@echo
-	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
-	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
+	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql.in and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
+	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql.in and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
 	@echo
 
 # Only meant for development time, do not use once a version is released
 .PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
-	rm -rf $(TEST_DIR)
-	rm -rf $(SOURCES_DATA_DIR_OUT)
-	rm -rf $(EXTENSION).control
+	rm -f $(PYTHON_TESTS)
 
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
+	
+
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
-
-# Replacing variables defined within test files
-replace_variables:
-	# tests
-	mkdir -p $(TEST_DIR)
-	mkdir -p $(TEST_DIR)/expected
-	mkdir -p $(TEST_DIR)/sql
-	for f in $(REGRESS); do \
-		cat test/sql/$${f}.sql | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		> $(TEST_DIR)/sql/$${f}.sql; \
-	done
-	for f in $(REGRESS_EXPEC); do \
-		cat test/expected/$${f}.out | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		> $(TEST_DIR)/expected/$${f}.out; \
-	done
-	# data
-	mkdir -p $(SOURCES_DATA_DIR_OUT)
-	for f in $(notdir $(SOURCES_DATA)); do \
-		cat $(SOURCES_DATA_DIR)/$${f} | \
-		sed -e 's/@@plpythonu@@/$(PLPYTHONU)/g' \
-		> $(SOURCES_DATA_DIR_OUT)/$${f}; \
-	done
 
+.PHONY: install
 install: deploy
-
-reinstall: install
-	psql -U postgres -d dataservices_db -c "drop extension if exists cdb_dataservices_server; create extension cdb_dataservices_server;"
+	@echo "Install overriden"

server/extension/carto-package.json

@@ -4,11 +4,11 @@
     "requires": {
       "postgresql": "^10.0.0",
       "postgis": "^2.4.0.0",
-      "carto_postgresql_ext": "^0.23.0"
+      "carto_postgresql_ext": "^0.36.0"
     },
     "works_with": {
-      "dataservices-api-server-python-lib": "^0.19.1",
+      "dataservices-api-server-python-lib": "^0.23.1",
       "observatory-server-extension": "^1.9.0"
     }
   }
-}
+}

server/extension/cdb_dataservices_server--0.38.0--0.39.0.sql

@@ -0,0 +1,11 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.39.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
+-- In 0.39.0 we removed cdb_dataservices_client.geomval and rely on postgis_raster if necessary
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_server.geomval RESTRICT;
+END$$;

server/extension/cdb_dataservices_server--0.39.0--0.38.0.sql

@@ -1,4 +1,9 @@
--- PG12_DEPRECATED
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.38.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+
 -- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
 DO $$
 BEGIN
@@ -8,4 +13,4 @@ BEGIN
             val double precision
         );
     END IF;
-END$$;
+END$$;

server/extension/cdb_dataservices_server.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.38.0'
-requires = '@@plpythonu@@, plproxy, postgis, cdb_geocoder'
+default_version = '@@EXTVERSION@@'
+requires = '@@plpythonu@@, plproxy, @@postgis@@, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.37.0--0.38.0.sql

@@ -0,0 +1,76 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.38.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_server.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  from plpy import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/old_versions/cdb_dataservices_server--0.38.0--0.37.0.sql

@@ -0,0 +1,71 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.37.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_server.geomval RESTRICT;
+END$$;
+
+---- cdb_geocode_namedplace_point(city_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3) as point;", ["text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, NULL, $4) as point;", ["text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;
+
+---- cdb_geocode_namedplace_point(city_name text, admin1_name text, country_name text)
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_geocode_namedplace_point(username text, orgname text, city_name text, admin1_name text, country_name text)
+RETURNS Geometry AS $$
+  import spiexceptions
+  from cartodb_services.tools import Logger,LoggerConfig
+
+  plpy.execute("SELECT cdb_dataservices_server._get_logger_config()")
+  logger_config = GD["logger_config"]
+  logger = Logger(logger_config)
+
+  try:
+    street_point = plpy.prepare("SELECT cdb_dataservices_server.cdb_geocode_street_point($1, $2, $3, NULL, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(street_point, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+  except spiexceptions.ExternalRoutineException as e:
+    import sys
+    logger.error('Error geocoding namedplace using geocode street point, falling back to internal geocoder', sys.exc_info(), data={"username": username, "orgname": orgname})
+    internal_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_internal_geocode_namedplace($1, $2, $3, $4, $5) as point;", ["text", "text", "text", "text", "text"])
+    return plpy.execute(internal_plan, [username, orgname, city_name, admin1_name, country_name])[0]['point']
+$$ LANGUAGE @@plpythonu@@ STABLE PARALLEL RESTRICTED;

server/extension/test/expected/00_install_test.out

@@ -1,13 +1,9 @@
 -- Only show warning or error messages in the tests output
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION plproxy;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION cdb_geocoder;
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_server;
+CREATE EXTENSION cdb_dataservices_server CASCADE;
+\unset QUIET
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
  cdb_conf_setconf 
@@ -71,6 +67,7 @@ BEGIN
 END
 $$
 LANGUAGE plpgsql;
+CREATE FUNCTION
 -- Set user quota
 SELECT cartodb.CDB_SetUserQuotaInBytes(0);
  cdb_setuserquotainbytes 

server/extension/test/sql/00_install_test.sql

@@ -1,14 +1,10 @@
 -- Only show warning or error messages in the tests output
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION @@plpythonu@@;
-CREATE EXTENSION plproxy;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION cdb_geocoder;
 
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_server;
+CREATE EXTENSION cdb_dataservices_server CASCADE;
+\unset QUIET
 
 -- Mock the redis server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');

server/lib/python/cartodb_services/cartodb_services/here/routing.py

@@ -115,7 +115,7 @@ class HereMapsRoutingIsoline(Traceable):
 
     def __parse_source_param(self, source, options):
         key = 'start'
-        if 'is_destination' in options and options['is_destination']:
+        if 'is_destination' in options and options['is_destination'].lower() == 'true':
             key = 'destination'
 
         return {key: source}

server/lib/python/cartodb_services/requirements.txt

@@ -4,7 +4,7 @@ python-dateutil==2.2
 googlemaps==2.5.1
 rollbar==0.13.2
 # Dependency for googlemaps package
-requests==2.9.1
+requests==2.20.0
 rratelimit==0.0.4
 mapbox==0.14.0
 pygeocodio==0.11.1

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.23.1',
+    version='0.23.2',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_heremapsrouting.py

@@ -204,6 +204,16 @@ class HereMapsRoutingIsolineTestCase(unittest.TestCase):
                          'fastest;car;traffic:false;motorway:-1')
 
     def test_source_parameters_works_properly(self, req_mock):
+        req_mock.register_uri('GET', requests_mock.ANY,
+                              text=self.GOOD_RESPONSE)
+        response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car',
+                                                    ['1000', '2000'],
+                                                    ['is_destination=false'])
+        parsed_url = urlparse(req_mock.request_history[0].url)
+        url_params = parse_qs(parsed_url.query)
+        self.assertEqual(url_params['start'][0], 'geo!33.0,1.0')
+
+    def test_destination_parameters_works_properly(self, req_mock):
         req_mock.register_uri('GET', requests_mock.ANY,
                               text=self.GOOD_RESPONSE)
         response = self.routing.calculate_isochrone('geo!33.0,1.0', 'car',

test/requirements.txt

@@ -1,3 +1,3 @@
 # Integration tests dependencies
-requests==2.9.1
+requests==2.20.0
 nose==1.3.7