Merge pull request #629 from CartoDB/development

fixed gcloud github actions setup

.github/workflows/main.yml

@@ -0,0 +1,60 @@
+name: dataservices-api PR testing
+on: push
+jobs:
+  dataservices-api:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        pg_version: [10, 12]
+    env:
+      PG_VERSION: ${{ matrix.pg_version }}
+    steps:
+      - uses: actions/checkout@v1
+
+      - name: Set CLOUDSDK_PYTHON path
+        run: echo "CLOUDSDK_PYTHON=/usr/bin/python" >> $GITHUB_ENV
+
+      - name: Setup gcloud authentication
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GCS }}
+
+      - name: Pull base image
+        run: docker pull gcr.io/cartodb-on-gcp-ci-testing/cartodb-postgresql-base:${{ matrix.pg_version }}
+
+      - name: Checkout ci tools repository
+        uses: actions/checkout@v2
+        with:
+          repository: CartoDB/ci-tools
+          path: ci-tools
+          token: ${{ secrets.CARTOFANTE_PAT }}
+
+      - name: Copy ci files to root
+        run: cp ci-tools/repos/${{ github.event.repository.name }}/* .
+
+      - name: Start docker-compose services
+        run: docker-compose -f docker-compose.yaml up -d
+
+      - name: Install required python3 libs
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && sudo pip3 install -U -r ./lib/python/cartodb_services/requirements.txt && sudo pip3 install -U ./lib/python/cartodb_services"
+        if: env.PG_VERSION == 12
+
+      - name: Install required python libs
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && pip install -U -r ./lib/python/cartodb_services/requirements.txt && pip install -U ./lib/python/cartodb_services"
+
+      - name: Run python library tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/ && MAPBOX_API_KEY=$MAPBOX_API_KEY TOMTOM_API_KEY=$TOMTOM_API_KEY GEOCODIO_API_KEY=$GEOCODIO_API_KEY nosetests lib/python/cartodb_services/test"
+        env:
+          MAPBOX_API_KEY: ${{ secrets.MAPBOX_API_KEY }}
+          TOMTOM_API_KEY: ${{ secrets.TOMTOM_API_KEY }}
+          GEOCODIO_API_KEY: ${{ secrets.GEOCODIO_API_KEY }}
+        timeout-minutes: 5
+
+      - name: Run server tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "cd /dataservices-api/server/extension/ && sudo make clean all install installcheck || (cat /dataservices-api/server/extension/test/regression.diffs && false)"
+        timeout-minutes: 5
+
+      - name: Run client tests
+        run: docker-compose -f docker-compose.yaml exec -T postgres-server bash -c "sudo createuser publicuser --no-createrole --no-createdb --no-superuser -U postgres && cd /dataservices-api/client/ && sudo make clean all install installcheck || (cat /dataservices-api/client/test/regression.diffs && false)"
+        timeout-minutes: 5

.gitignore

@@ -5,3 +5,5 @@ cartodb_services.egg-info/
 build/
 dist/
 .vscode/
+.idea/
+venv/

NEWS.md

@@ -1,3 +1,152 @@
+Sep 5th, 2022
+==============
+* Version `0.24.2` of the Python library
+    * Fix error calculating isolines with HERE
+
+May 26th, 2021
+==============
+* Version `0.40.1` of the server extension
+* Version `0.24.1` of the Python library
+    * Support new HERE API keys
+
+Mar 4th, 2021
+=============
+* Version `0.31.0` of the client extension
+* Version `0.40.0` of the server extension
+* Version `0.24.0` of the Python library
+    * Remove Data Observatory
+
+Jan 26th, 2021
+==============
+* Version `0.39.3` of the server extension
+* Version `0.23.5` of the Python library
+    * Country name conversion to ISO code for TomTom bulk geocoding.
+
+Nov 19th, 2020
+==============
+* Version `0.39.2` of the server extension
+* Version `0.23.4` of the Python library
+    * Better messages on remote errors.
+    * Fix error on unknown status codes.
+    * Here bulk geocoder: Return also errors and no-matches.
+
+Sep 23th, 2020
+==============
+* Version `0.39.1` of the server extension
+    * Allow isolines services for google-geocoding users.
+
+Sep 23th, 2020
+==============
+* Version `0.23.3` of the Python library
+    * Change HERE batch geocoding URL
+
+Apr 7th, 2020
+=============
+* Version `0.30.0` of the client extension
+    * PG12: Use postgis_raster instead of custom geomval.
+* Version `0.39.0` of the server extension
+    * PG12: Use postgis_raster instead of custom geomval.
+
+Mar 17th, 2020
+==============
+* Version `0.29.0` of the client extension
+    * Adapted client to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Version `0.38.0` of the server extension
+    * Adapted server to PG12 (https://github.com/CartoDB/cartodb-platform/issues/6237)
+* Update dependencies to fix vulnerabilities
+
+Mar 6th, 2020
+==============
+* Version `0.23.1` of the Python library
+    * Added query per second throttle to TomTom bulk API calls
+
+Feb 12th, 2020
+==============
+* Version `0.23.0` of the Python library
+    * Added Mapbox true isolines
+* Version `0.37.0` of the server extension
+    * Added Mapbox true isolines
+
+Jan 20th, 2020
+==============
+* Version `0.22.0` of the Python library
+    * Added geocodio geocoder
+* Version `0.36.0` of the server extension
+    * Added geocodio geocoder
+* Version `0.28.0` of the client extension
+    * Added geocodio geocoder
+
+Oct 9th, 2019
+==============
+* Version `0.27.0` of the client extension
+    * Changes in search_path
+
+Mar 13rd, 2019
+==============
+* Version `0.21.4` of the python library
+    * Fix TomTom bulk geocoder bug (#551)
+
+Mar 5th, 2019
+==============
+* Version `0.21.3` of the python library
+    * Fixed TomTom Qps respondes part 2 (#546)
+
+Mar 4th, 2019
+==============
+* Version `0.21.2` of the python library
+    * Fixed TomTom Qps responses (#546)
+
+Feb 25th, 2019
+==============
+* Version `0.26.2` of the client extension
+    * Fixed bulk geocoding soft limit (#543)
+
+Dec 31st, 2018
+==============
+* Version `0.35.1` of the server extension and `0.26.1` of the client
+    * Now the bulk geocoder returns `Point` geometries instead of MultiPolygons (#538)
+
+Nov 30th, 2018
+==============
+* Version `0.35.0` of the server extension
+    * Now the `mode_type` is working the providers that support it (#536)
+
+Nov 27th, 2018
+==============
+* Version `0.34.2` of the server extension
+    * Fixed bulk geocoding log metrics in order to fix problems in ELK due to the JSON not being standard (#530)
+
+Nov 22th, 2018
+==============
+* Version `0.34.1` of the server extension
+    * Fixed isochrones metrics that increase `success_rows` field for every isochrone generated (#533)
+
+Oct 31th, 2018
+==============
+* Version `0.20.2` of the python library
+    * Added missing provider property to the QPS decorator in other TomTom services
+    * Now we only retry with the properly header coming from TomTom
+
+Oct 3rd, 2018
+==============
+* Version `0.20.1` of the python library
+    * Fix QPS manager to retry with 403 status codes coming from TomTom
+
+Sep 13th, 2018
+==============
+* Version `0.34.0` of the server, and `0.26.0` of the client.
+    * Use dataservices with non-master keys
+
+Aug 27th, 2018
+==============
+* Version `0.33.0` of the server, and `0.20.0` of the Python library.
+    * Remove the obs_snapshot quota and now the snapshot functions uses obs_general quota
+
+Jul 19th, 2018
+==============
+* Version `0.25.0` of the client, `0.32.0` of the server, and `0.19.1` of the Python library.
+    * Support for batch street-level geocoding.
+
 May 7th, 2018
 =============
 * Version `0.24.0` of the client, `0.31.0` of the server, and `0.18.0` of the python library.

README.md

@@ -40,6 +40,7 @@ Steps to deploy a new Data Services API version :
     # in dataservices-api repo root path:
     cd server/lib/python/cartodb_services && sudo pip install -r requirements.txt && sudo pip install . --upgrade
     ```
+    **CLOUD DEPLOY NOTE**: we were not installing automatically `requirements.txt`, so we fixed it in https://github.com/CartoDB/cartodb-platform/pull/6187 . Please, be aware that in some corner cases scenarios, rolling back to a previous version might require to manually force-install some dependency versions that were upgraded previously in this step.
 
 - Create a database to hold all the server part and a user for it
 
@@ -79,28 +80,6 @@ Steps to deploy a new Data Services API version :
      psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_geocoder; COMMIT" -e
      ```
 
-- [optional] install data observatory extension
-   - Make the extension available in postgresql to be installed
-     ```
-     git clone https://github.com/CartoDB/observatory-extension.git
-     cd observatory
-     sudo make install
-     ```
-   - This extension needs data, dumps are not available so we're going to use the test fixtures to make it work. Execute:
-     ```
-     psql -U postgres -d dataservices_db -f src/pg/test/fixtures/load_fixtures.sql
-     ```
-   - Give permission to execute and select to the `geocoder_api` user:
-     ```
-     psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatory VERSION 'dev'; COMMIT" -e
-     psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO geocoder_api; COMMIT" -e
-     psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO geocoder_api; COMMIT" -e
-     psql -U postgres -d dataservices_db -c "BEGIN;GRANT USAGE ON SCHEMA cdb_observatory TO geocoder_api; COMMIT" -e
-     psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO geocoder_api; COMMIT" -e
-     psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA observatory TO geocoder_api; COMMIT" -e
-     psql -U postgres -d dataservices_db -c "BEGIN;GRANT USAGE ON SCHEMA observatory TO geocoder_api; COMMIT" -e
-     ```
-
 ### Server configuration
 
 Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`.
@@ -153,6 +132,15 @@ SELECT CDB_Conf_SetConf(
 );
 ```
 
+Use HERE apikeys 
+
+```sql
+SELECT CDB_Conf_SetConf(
+    'heremaps_conf',
+    '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "apikey": "here_geocoder_apikey", "use_apikey": true, "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code", "apikey": "here_geocoder_apikey", "use_apikey": true}}'
+);
+```
+
 #### Mapzen configuration
 
 ```sql
@@ -180,12 +168,12 @@ SELECT CDB_Conf_SetConf(
 );
 ```
 
-#### Data Observatory
+#### Geocod.io configuration
 
 ```sql
 SELECT CDB_Conf_SetConf(
-    'data_observatory_conf',
-    '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}'
+    'geocodio_conf',
+    '{"geocoder": {"api_keys": ["your_api_key"], "monthly_quota": 999999}}'
 );
 ```
 

client/.gitignore

@@ -9,3 +9,4 @@ cdb_geocoder_client--0.0.1.sql
 cdb_geocoder_client--0.1.0.sql
 cdb_geocoder_client--0.2.0.sql
 cdb_geocoder_client--0.3.0.sql
+cdb_dataservices_client.control

client/Makefile

@@ -1,19 +1,30 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_client
-EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.31.0
 
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+POSTGIS := postgis
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
+endif
+
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 # OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # DATA =  $(NEW_EXTENSION_ARTIFACT) \
 #   $(OLD_VERSIONS) \
@@ -25,9 +36,9 @@ DATA = $(NEW_EXTENSION_ARTIFACT) \
 		 $(OLD_VERSIONS)
 SOURCES_DATA_DIR = sql/
 
-
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
-TEST_DIR = test/
+REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -43,7 +54,6 @@ TEMPLATE_FILES = $(wildcard $(TEMPLATE_DIR)/*.erb)
 GENERATED_SQL_FILES = $(patsubst $(TEMPLATE_DIR)/%.erb, $(SOURCES_DATA_DIR)/%.sql, $(TEMPLATE_FILES))
 
 # postgres build stuff
-PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
@@ -55,27 +65,20 @@ SOURCES_DATA = $(wildcard $(SOURCES_DATA_DIR)/*.sql) $(GENERATED_SQL_FILES)
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
 	cat $(SOURCES_DATA_DIR)/*.sql >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
 
 .PHONY: all
 all: $(DATA)
 
 .PHONY: release
-release: $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control
-	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
+	$(SED) -i -e 's/^EXTVERSION =.*/EXTVERSION = $(NEW_VERSION)/g' Makefile
+	git add Makefile
 	@echo
 	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
 	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
@@ -87,19 +90,29 @@ devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
 	rm -f $(GENERATED_SQL_FILES)
 
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
+
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
+.PHONY: install
 install: deploy
+	@echo "Install overriden"

client/README.md

@@ -37,7 +37,7 @@ psql -U postgres cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db
 and then:
 
 ```sql
-CREATE EXTENSION cdb_dataservices_client;
+CREATE EXTENSION cdb_dataservices_client CASCADE;
 ```
 
 The extension creation in the user's db requires **superuser** privileges.

client/carto-package.json

@@ -0,0 +1,12 @@
+{
+  "name": "dataservices-api-client-extension",
+  "current_version": {
+    "requires": {
+      "postgresql": "^11.0.0",
+      "postgis": "^2.5.0.0",
+      "carto_postgresql_ext": "^0.36.0"
+    },
+    "works_with": {
+    }
+  }
+}

client/cdb_dataservices_client--0.30.0--0.31.0.sql

@@ -0,0 +1,187 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.31.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_demographic_snapshot(geom public.geometry(Geometry, 4326), time_span text, geometry_level text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe(geom public.geometry(Geometry, 4326), time_span text, geometry_level text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_get_segment_snapshot(geom public.geometry(Geometry, 4326), geometry_level text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_get_segment_snapshot_exception_safe(geom public.geometry(Geometry, 4326), geometry_level text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdemographicsnapshot(geom public.geometry(Geometry, 4326), time_span text, geometry_level text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe(geom public.geometry(Geometry, 4326), time_span text, geometry_level text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getsegmentsnapshot(geom public.geometry(Geometry, 4326), geometry_level text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe(geom public.geometry(Geometry, 4326), geometry_level text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundary(geom public.geometry(Geometry, 4326), boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundary_exception_safe(geom public.geometry(Geometry, 4326), boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundaryid(geom public.geometry(Geometry, 4326), boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundaryid_exception_safe(geom public.geometry(Geometry, 4326), boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundarybyid(geometry_id text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundarybyid_exception_safe(geometry_id text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbygeometry(geom public.geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe(geom public.geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getboundariesbypointandradius(geom public.geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe(geom public.geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbygeometry(geom public.geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbygeometry_exception_safe(geom public.geometry(Geometry, 4326), boundary_id text, time_span text, overlap_type text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpointsbypointandradius(geom public.geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe(geom public.geometry(Geometry, 4326), radius numeric, boundary_id text, time_span text, overlap_type text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasure(geom public.Geometry, measure_id text, normalize text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasure_exception_safe(geom public.Geometry, measure_id text, normalize text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeasurebyid(geom_ref text, measure_id text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeasurebyid_exception_safe(geom_ref text, measure_id text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomvals geomval[], params json, merge boolean) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomvals geomval[], params json, merge boolean )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getdata(geomrefs text[], params json) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getdata_exception_safe(geomrefs text[], params json )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getmeta(geom_ref public.Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getmeta_exception_safe(geom_ref public.Geometry(Geometry, 4326), params json, max_timespan_rank integer, max_score_rank integer, target_geoms integer )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_metadatavalidation(geom_extent public.Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_metadatavalidation_exception_safe(geom_extent public.Geometry(Geometry, 4326), geom_type text, params json, target_geoms integer )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getcategory(geom public.Geometry, category_id text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getcategory_exception_safe(geom public.Geometry, category_id text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensusmeasure(geom public.Geometry, name text, normalize text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensusmeasure_exception_safe(geom public.Geometry, name text, normalize text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getuscensuscategory(geom public.Geometry, name text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getuscensuscategory_exception_safe(geom public.Geometry, name text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getpopulation(geom public.Geometry, normalize text, boundary_id text, time_span text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getpopulation_exception_safe(geom public.Geometry, normalize text, boundary_id text, time_span text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_search(search_term text, relevant_boundary text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_search_exception_safe(search_term text, relevant_boundary text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailableboundaries(geom public.Geometry, timespan text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailableboundaries_exception_safe(geom public.Geometry, timespan text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_dumpversion() FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_dumpversion_exception_safe( )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablenumerators(bounds public.geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablenumerators_exception_safe(bounds public.geometry(Geometry, 4326), filter_tags text[], denom_id text, geom_id text, timespan text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getnumerators(bounds public.geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.__obs_getnumerators_exception_safe(bounds public.geometry(Geometry, 4326), section_tags text[], subsection_tags text[], other_tags text[], ids text[], name text, denom_id text, geom_id text, timespan text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabledenominators(bounds public.geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabledenominators_exception_safe(bounds public.geometry(Geometry, 4326), filter_tags text[], numer_id text, geom_id text, timespan text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailablegeometries(bounds public.geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailablegeometries_exception_safe(bounds public.geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, timespan text, number_geometries integer )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_getavailabletimespans(bounds public.geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_getavailabletimespans_exception_safe(bounds public.geometry(Geometry, 4326), filter_tags text[], numer_id text, denom_id text, geom_id text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client.obs_legacybuildermetadata(aggregate_type text) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._obs_legacybuildermetadata_exception_safe(aggregate_type text )  FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) FROM publicuser;
+REVOKE EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) FROM publicuser;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_get_demographic_snapshot (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_get_segment_snapshot (geometry(Geometry, 4326), TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getdemographicsnapshot (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getsegmentsnapshot (geometry(Geometry, 4326), TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getboundary (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getboundaryid (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getboundarybyid (TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getboundariesbygeometry (geometry(Geometry, 4326), TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getboundariesbypointandradius (geometry(Geometry, 4326), NUMERIC, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getpointsbygeometry (geometry(Geometry, 4326), TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getpointsbypointandradius (geometry(Geometry, 4326), NUMERIC, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getmeasure (geometry, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getmeasurebyid (TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getdata (geomval[], JSON, BOOLEAN);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getdata (TEXT[], JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getmeta (geometry(Geometry, 4326), JSON, INTEGER, INTEGER, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_metadatavalidation (geometry(Geometry, 4326), TEXT, JSON, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getcategory (geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getuscensusmeasure (geometry, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getuscensuscategory (geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getpopulation (geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_search (TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getavailableboundaries (geometry, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_dumpversion ();
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getavailablenumerators (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getnumerators (geometry(Geometry, 4326), TEXT[], TEXT[], TEXT[], TEXT[], TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getavailabledenominators (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getavailablegeometries (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_getavailabletimespans (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.obs_legacybuildermetadata (TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure (TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure (TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure (TEXT, TEXT, TEXT, TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure (TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_ConnectUserTable (TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_GetReturnMetadata (TEXT, TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_FetchJoinFdwTableData (TEXT, TEXT, TEXT, TEXT, TEXT, JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._DST_DisconnectUserTable (TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot_exception_safe (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot_exception_safe (geometry(Geometry, 4326), TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot_exception_safe (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot_exception_safe (geometry(Geometry, 4326), TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary_exception_safe (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid_exception_safe (geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid_exception_safe (TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry_exception_safe (geometry(Geometry, 4326), TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius_exception_safe (geometry(Geometry, 4326), NUMERIC, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry_exception_safe (geometry(Geometry, 4326), TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius_exception_safe (geometry(Geometry, 4326), NUMERIC, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure_exception_safe (geometry, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid_exception_safe (TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata_exception_safe (geomval[], JSON, BOOLEAN);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata_exception_safe (TEXT[], JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta_exception_safe (geometry(Geometry, 4326), JSON, INTEGER, INTEGER, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation_exception_safe (geometry(Geometry, 4326), TEXT, JSON, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory_exception_safe (geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure_exception_safe (geometry, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory_exception_safe (geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation_exception_safe (geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search_exception_safe (TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries_exception_safe (geometry, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion_exception_safe ();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators_exception_safe (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators_exception_safe (geometry(Geometry, 4326), TEXT[], TEXT[], TEXT[], TEXT[], TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators_exception_safe (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries_exception_safe (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans_exception_safe (geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata_exception_safe (TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_demographic_snapshot (TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_get_segment_snapshot (TEXT, TEXT, geometry(Geometry, 4326), TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdemographicsnapshot (TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getsegmentsnapshot (TEXT, TEXT, geometry(Geometry, 4326), TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundary (TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundaryid (TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundarybyid (TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbygeometry (TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getboundariesbypointandradius (TEXT, TEXT, geometry(Geometry, 4326), NUMERIC, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbygeometry (TEXT, TEXT, geometry(Geometry, 4326), TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpointsbypointandradius (TEXT, TEXT, geometry(Geometry, 4326), NUMERIC, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasure (TEXT, TEXT, geometry, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeasurebyid (TEXT, TEXT, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (TEXT, TEXT, geomval[], JSON, BOOLEAN);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getdata (TEXT, TEXT, TEXT[], JSON);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getmeta (TEXT, TEXT, geometry(Geometry, 4326), JSON, INTEGER, INTEGER, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_metadatavalidation (TEXT, TEXT, geometry(Geometry, 4326), TEXT, JSON, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getcategory (TEXT, TEXT, geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensusmeasure (TEXT, TEXT, geometry, TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getuscensuscategory (TEXT, TEXT, geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getpopulation (TEXT, TEXT, geometry, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_search (TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailableboundaries (TEXT, TEXT, geometry, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_dumpversion (TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablenumerators (TEXT, TEXT, geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__obs_getnumerators (TEXT, TEXT, geometry(Geometry, 4326), TEXT[], TEXT[], TEXT[], TEXT[], TEXT, TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabledenominators (TEXT, TEXT, geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailablegeometries (TEXT, TEXT, geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT, INTEGER);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_getavailabletimespans (TEXT, TEXT, geometry(Geometry, 4326), TEXT[], TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._obs_legacybuildermetadata (TEXT, TEXT, TEXT);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_PreCheck (TEXT, JSON);
+
+DROP TYPE IF EXISTS cdb_dataservices_client.obs_meta_numerator;
+DROP TYPE IF EXISTS cdb_dataservices_client.obs_meta_denominator;
+DROP TYPE IF EXISTS cdb_dataservices_client.obs_meta_geometry;
+DROP TYPE IF EXISTS cdb_dataservices_client.obs_meta_timespan;
+DROP TYPE IF EXISTS cdb_dataservices_client.ds_fdw_metadata;
+DROP TYPE IF EXISTS cdb_dataservices_client.ds_return_metadata;
+
+DELETE FROM pg_enum
+WHERE enumlabel = 'observatory'
+AND enumtypid = (
+  SELECT pg_type.oid
+  FROM pg_type INNER JOIN pg_namespace ON (pg_type.typnamespace = pg_namespace.oid)
+  WHERE pg_type.typname = 'service_type' AND pg_namespace.nspname = 'cdb_dataservices_client'
+);

client/cdb_dataservices_client.control.in

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.24.0'
-requires = 'plproxy, cartodb'
+default_version = '@@EXTVERSION@@'
+requires = 'plproxy, cartodb, @@plpythonu@@, @@postgis@@'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.24.0--0.25.0.sql

@@ -0,0 +1,276 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
+$func$
+DECLARE
+    rec   record;
+    ROWS  INTEGER;
+BEGIN
+    FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        EXIT WHEN ROWS IS NOT NULL;
+    END LOOP;
+
+    RETURN ROWS;
+END
+$func$ LANGUAGE plpgsql;
+
+-- Taken from https://stackoverflow.com/a/48013356/351721
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
+    SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
+$f$ LANGUAGE sql IMMUTABLE;--
+
+CREATE TYPE cdb_dataservices_client.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Multipolygon,4326),
+    metadata jsonb
+);
+
+CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
+    service cdb_dataservices_client.service_type,
+    monthly_quota NUMERIC,
+    used_quota NUMERIC,
+    soft_limit BOOLEAN,
+    provider TEXT,
+    max_batch_size NUMERIC
+);
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point (searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
+RETURNS SETOF service_quota_info_batch AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+BEGIN
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF remaining_quota < query_row_count THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;
+
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client.__cdb_bulk_geocode_street_point(username, orgname, searches);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+        
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+--
+-- Exception-safe private DataServices API function
+--
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ()
+RETURNS SETOF service_quota_info_batch AS $$
+DECLARE
+
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    RETURN QUERY SELECT * FROM cdb_dataservices_client._cdb_service_quota_info_batch(username, orgname);
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+
+  END;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server._cdb_bulk_geocode_street_point (username, orgname, searches);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch (username text, orgname text)
+RETURNS SETOF service_quota_info_batch AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_service_quota_info_batch (username, orgname);
+  
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe(searches jsonb )  TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe( )  TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.__cdb_bulk_geocode_street_point (username text, orgname text, searches jsonb) TO publicuser;

client/old_versions/cdb_dataservices_client--0.25.0--0.24.0.sql

@@ -0,0 +1,29 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '<%= version %>'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_count_estimate(query text);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_bulk_geocode_street_point (jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_service_quota_info_batch();
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_bulk_geocode_street_point (text, text, text, text, text, integer);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point_exception_safe (jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch_exception_safe ();
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__cdb_bulk_geocode_street_point (text, text, jsonb);
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_service_quota_info_batch (text, text);
+
+DROP TYPE IF EXISTS cdb_dataservices_client.service_quota_info_batch;
+
+DROP TYPE IF EXISTS cdb_dataservices_client.geocoding;

client/old_versions/cdb_dataservices_client--0.26.0--0.26.1.sql

@@ -0,0 +1,99 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.1'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(Point,4326);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF remaining_quota < query_row_count THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;

client/old_versions/cdb_dataservices_client--0.26.1--0.26.0.sql

@@ -0,0 +1,99 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+ALTER TYPE cdb_dataservices_client.geocoding ALTER ATTRIBUTE the_geom TYPE geometry(MultiPolygon,4326);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF remaining_quota < query_row_count THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Multipolygon,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;

client/old_versions/cdb_dataservices_client--0.26.1--0.26.2.sql

@@ -0,0 +1,96 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.2'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF enough_quota IS NULL OR NOT enough_quota THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;

client/old_versions/cdb_dataservices_client--0.26.2--0.26.1.sql

@@ -0,0 +1,96 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.1'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF remaining_quota < query_row_count THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom geometry(Point,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;

client/old_versions/cdb_dataservices_client--0.26.2--0.27.0.sql

@@ -0,0 +1,8 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.27.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade

client/old_versions/cdb_dataservices_client--0.27.0--0.26.2.sql

@@ -0,0 +1,8 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.26.2'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade

client/old_versions/cdb_dataservices_client--0.27.0--0.28.0.sql

@@ -0,0 +1,83 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS public.Geometry AS $$
+DECLARE
+  ret public.Geometry;
+  username text;
+  orgname text;
+  _returned_sqlstate TEXT;
+  _message_text TEXT;
+  _pg_exception_context TEXT;
+  apikey_permissions json;
+BEGIN
+  
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied';
+  END IF;
+  
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+
+  BEGIN
+    SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+  EXCEPTION
+    WHEN OTHERS THEN
+        GET STACKED DIAGNOSTICS _returned_sqlstate = RETURNED_SQLSTATE,
+                                _message_text = MESSAGE_TEXT,
+                                _pg_exception_context = PG_EXCEPTION_CONTEXT;
+        RAISE WARNING USING ERRCODE = _returned_sqlstate, MESSAGE = _message_text, DETAIL = _pg_exception_context;
+         RETURN ret;
+  END;
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text, state_province text, country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point (username text, orgname text, searchtext text, city text DEFAULT NULL, state_province text DEFAULT NULL, country text DEFAULT NULL)
+RETURNS Geometry AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT cdb_dataservices_server.cdb_geocodio_geocode_street_point (username, orgname, searchtext, city, state_province, country);
+
+$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text ,state_province text ,country text);
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point (searchtext text ,city text DEFAULT NULL ,state_province text DEFAULT NULL ,country text DEFAULT NULL)
+RETURNS public.Geometry AS $$
+DECLARE
+  ret public.Geometry;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+  
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT cdb_dataservices_client._cdb_geocodio_geocode_street_point(username, orgname, searchtext, city, state_province, country) INTO ret; RETURN ret;
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_geocodio_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe(searchtext text, city text, state_province text, country text )  TO publicuser;
+

client/old_versions/cdb_dataservices_client--0.28.0--0.27.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.27.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point_exception_safe;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_geocodio_geocode_street_point;
+
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_geocodio_geocode_street_point;

client/old_versions/cdb_dataservices_client--0.28.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/old_versions/cdb_dataservices_client--0.29.0--0.28.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.28.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+-- PG12_DEPRECATED
+-- Create geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/old_versions/cdb_dataservices_client--0.29.0--0.30.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.30.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+-- In 0.30.0 we removed cdb_dataservices_client.geomval and rely on postgis_raster if necessary
+DO $$
+BEGIN
+  DROP TYPE IF EXISTS cdb_dataservices_client.geomval RESTRICT;
+END$$;

client/old_versions/cdb_dataservices_client--0.30.0--0.29.0.sql

@@ -0,0 +1,19 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.29.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+-- HERE goes your code to upgrade/downgrade
+
+-- Create cdb_dataservices_client.geomval if it doesn't exist (in postgis 3+ it only exists in postgis_raster)
+DO $$
+BEGIN
+    IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'geomval') THEN
+        CREATE TYPE cdb_dataservices_client.geomval AS (
+            geom geometry,
+            val double precision
+        );
+    END IF;
+END$$;

client/renderer/interface.yaml

@@ -1,77 +1,126 @@
 ---
 - name: cdb_geocode_admin0_polygon
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: country_name, type: text }
 
 - name: cdb_geocode_admin1_polygon
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: admin1_name, type: text }
 
 - name: cdb_geocode_admin1_polygon
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: admin1_name,  type: text }
     - { name: country_name, type: text }
 
 - name: cdb_geocode_namedplace_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: city_name, type: text}
 
 - name: cdb_geocode_namedplace_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: city_name,    type: text}
     - { name: country_name, type: text}
 
 - name: cdb_geocode_namedplace_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: city_name,    type: text}
     - { name: admin1_name,  type: text}
     - { name: country_name, type: text}
 
 - name: cdb_geocode_postalcode_polygon
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: postal_code,  type: text}
     - { name: country_name, type: text}
 
 - name: cdb_geocode_postalcode_polygon
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: postal_code,  type: double precision}
     - { name: country_name, type: text}
 
 - name: cdb_geocode_postalcode_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: postal_code,  type: text}
     - { name: country_name, type: text}
 
 - name: cdb_geocode_postalcode_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: postal_code,  type: double precision}
     - { name: country_name, type: text}
 
 - name: cdb_geocode_ipaddress_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: ip_address,  type: text}
 
 - name: cdb_geocode_street_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: searchtext,  type: text}
     - { name: city,  type: text, default: 'NULL'}
     - { name: state_province,  type: text, default: 'NULL'}
     - { name: country,  type: text, default: 'NULL'}
 
+- name: _cdb_bulk_geocode_street_point
+  return_type: SETOF cdb_dataservices_client.geocoding
+  multi_row: true
+  multi_field: true
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
+  params:
+    - { name: searches,  type: jsonb } # Array of JSON objects with id, address, city, state and country fields
+
 - name: cdb_here_geocode_street_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: searchtext,  type: text}
     - { name: city,  type: text, default: 'NULL'}
@@ -79,7 +128,10 @@
     - { name: country,  type: text, default: 'NULL'}
 
 - name: cdb_google_geocode_street_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: searchtext,  type: text}
     - { name: city,  type: text, default: 'NULL'}
@@ -87,7 +139,10 @@
     - { name: country,  type: text, default: 'NULL'}
 
 - name: cdb_mapbox_geocode_street_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: searchtext,  type: text}
     - { name: city,  type: text, default: 'NULL'}
@@ -95,7 +150,21 @@
     - { name: country,  type: text, default: 'NULL'}
 
 - name: cdb_tomtom_geocode_street_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
+  params:
+    - { name: searchtext,  type: text}
+    - { name: city,  type: text, default: 'NULL'}
+    - { name: state_province,  type: text, default: 'NULL'}
+    - { name: country,  type: text, default: 'NULL'}
+
+- name: cdb_geocodio_geocode_street_point
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: searchtext,  type: text}
     - { name: city,  type: text, default: 'NULL'}
@@ -103,7 +172,10 @@
     - { name: country,  type: text, default: 'NULL'}
 
 - name: cdb_mapzen_geocode_street_point
-  return_type: Geometry
+  return_type: public.Geometry
+  requires_permission: true
+  permission_name: geocoding
+  permission_error: Geocoding permission denied
   params:
     - { name: searchtext,  type: text}
     - { name: city,  type: text, default: 'NULL'}
@@ -114,8 +186,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -124,8 +199,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -134,8 +212,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -144,8 +225,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -154,8 +238,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -164,8 +251,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -174,8 +264,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -184,8 +277,11 @@
   return_type: SETOF cdb_dataservices_client.isoline
   multi_row: true
   multi_field: true
+  requires_permission: true
+  permission_name: isolines
+  permission_error: Isolines permission denied
   params:
-    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: source,  type: "public.geometry(Geometry, 4326)" }
     - { name: mode, type: text }
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
@@ -193,9 +289,12 @@
 - name: cdb_route_point_to_point
   return_type: cdb_dataservices_client.simple_route
   multi_field: true
+  requires_permission: true
+  permission_name: routing
+  permission_error: Routing permission denied
   params:
-    - { name: origin,  type: "geometry(Point, 4326)" }
-    - { name: destination,  type: "geometry(Point, 4326)" }
+    - { name: origin,  type: "public.geometry(Point, 4326)" }
+    - { name: destination,  type: "public.geometry(Point, 4326)" }
     - { name: mode, type: text }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
     - { name: units, type: "text", default: "'kilometers'"}
@@ -203,308 +302,24 @@
 - name: cdb_route_with_waypoints
   return_type: cdb_dataservices_client.simple_route
   multi_field: true
+  requires_permission: true
+  permission_name: routing
+  permission_error: Routing permission denied
   params:
-    - { name: waypoints,  type: "geometry(Point, 4326)[]" }
+    - { name: waypoints,  type: "public.geometry(Point, 4326)[]" }
     - { name: mode, type: text }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
     - { name: units, type: "text", default: "'kilometers'"}
 
-- name: obs_get_demographic_snapshot
-  return_type: json
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: time_span,  type: "text", default: "'2009 - 2013'::text" }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_get_segment_snapshot
-  return_type: json
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_getdemographicsnapshot
-  return_type: SETOF JSON
-  multi_row: true
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: time_span,  type: "text", default: 'NULL' }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_getsegmentsnapshot
-  return_type: SETOF JSON
-  multi_row: true
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: geometry_level, type: text, default: 'NULL' }
-
-- name: obs_getboundary
-  return_type: Geometry
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getboundaryid
-  return_type: text
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getboundarybyid
-  return_type: Geometry
-  params:
-    - { name: geometry_id,  type: text }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getboundariesbygeometry
-  return_type: TABLE(the_geom geometry, geom_refs text)
+- name: cdb_service_quota_info
+  return_type: SETOF service_quota_info
   multi_row: true
   multi_field: true
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getboundariesbypointandradius
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: radius, type: numeric }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getpointsbygeometry
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getpointsbypointandradius
-  return_type: TABLE(the_geom geometry, geom_refs text)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: the_geom, type: geometry }
-    - { name: geom_refs, type: text }
-  params:
-    - { name: geom,  type: "geometry(Geometry, 4326)" }
-    - { name: radius, type: numeric }
-    - { name: boundary_id, type: text }
-    - { name: time_span, type: text, default: 'NULL'}
-    - { name: overlap_type, type: text, default: 'NULL'}
-
-- name: obs_getmeasure
-  return_type: numeric
-  params:
-    - { name: geom,  type: Geometry }
-    - { name: measure_id, type: text }
-    - { name: normalize, type: text, default: 'NULL'}
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getmeasurebyid
-  return_type: numeric
-  params:
-    - { name: geom_ref,  type: text }
-    - { name: measure_id, type: text }
-    - { name: boundary_id, type: text}
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getdata
-  return_type: TABLE(id int, data json)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: id, type: integer }
-    - { name: data, type: json }
-  params:
-    - { name: geomvals,  type: "geomval[]" }
-    - { name: params, type: json }
-    - { name: merge, type: boolean, default: true }
-
-- name: obs_getdata
-  return_type: TABLE(id text, data json)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: id, type: text }
-    - { name: data, type: json }
-  params:
-    - { name: geomrefs,  type: "text[]" }
-    - { name: params, type: json }
-
-- name: obs_getmeta
-  return_type: json
-  params:
-    - { name: geom_ref,  type: "Geometry(Geometry, 4326)" }
-    - { name: params, type: json }
-    - { name: max_timespan_rank, type: integer, default: 'NULL' }
-    - { name: max_score_rank, type: integer, default: 'NULL' }
-    - { name: target_geoms, type: integer, default: 'NULL' }
-
-- name: obs_metadatavalidation
-  return_type: TABLE(valid boolean, errors text[])
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: geom_extent,  type: "Geometry(Geometry, 4326)" }
-    - { name: geom_type,  type: text }
-    - { name: params, type: json }
-    - { name: target_geoms, type: integer, default: 'NULL' }
-
-- name: obs_getcategory
-  return_type: text
-  params:
-    - { name: geom,  type: Geometry }
-    - { name: category_id, type: text }
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getuscensusmeasure
-  return_type: numeric
-  params:
-    - { name: geom,  type: Geometry }
-    - { name: name, type: text }
-    - { name: normalize, type: text, default: 'NULL'}
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getuscensuscategory
-  return_type: text
-  params:
-    - { name: geom,  type: Geometry }
-    - { name: name, type: text }
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_getpopulation
-  return_type: numeric
-  params:
-    - { name: geom,  type: Geometry }
-    - { name: normalize, type: text, default: 'NULL'}
-    - { name: boundary_id, type: text, default: 'NULL' }
-    - { name: time_span, type: text, default: 'NULL'}
-
-- name: obs_search
-  return_type: TABLE(id text, description text, name text, aggregate text, source text)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: id, type: text }
-    - { name: description, type: text }
-    - { name: name, type: text }
-    - { name: aggregate, type: text }
-    - { name: source, type: text }
-  params:
-    - { name: search_term, type: text }
-    - { name: relevant_boundary, type: text, default: 'NULL' }
-
-- name: obs_getavailableboundaries
-  return_type: TABLE(boundary_id text, description text, time_span text, tablename text)
-  multi_row: true
-  multi_field: true
-  table_fields:
-    - { name: boundary_id, type: text }
-    - { name: description, type: text }
-    - { name: time_span, type: text }
-    - { name: tablename, type: text }
-  params:
-    - { name: geom,  type: Geometry }
-    - { name: timespan, type: text, default: 'NULL'}
-
-- name: obs_dumpversion
-  return_type: text
   params:
     - {}
 
-- name: obs_getavailablenumerators
-  return_type: SETOF cdb_dataservices_client.obs_meta_numerator
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: denom_id, type: text, default: 'NULL' }
-    - { name: geom_id, type: text, default: 'NULL' }
-    - { name: timespan, type: text, default: 'NULL'}
-
-- name: _obs_getnumerators
-  return_type: SETOF cdb_dataservices_client.obs_meta_numerator
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: section_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: subsection_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: other_tags, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: ids, type: "text[]", default: 'ARRAY[]::TEXT[]' }
-    - { name: name, type: text, default: 'NULL' }
-    - { name: denom_id, type: text, default: "''" }
-    - { name: geom_id, type: text, default: "''" }
-    - { name: timespan, type: text, default: "''"}
-
-- name: obs_getavailabledenominators
-  return_type: SETOF cdb_dataservices_client.obs_meta_denominator
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: numer_id, type: text, default: 'NULL' }
-    - { name: geom_id, type: text, default: 'NULL' }
-    - { name: timespan, type: text, default: 'NULL'}
-
-- name: obs_getavailablegeometries
-  return_type: SETOF cdb_dataservices_client.obs_meta_geometry
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: numer_id, type: text, default: 'NULL' }
-    - { name: denom_id, type: text, default: 'NULL' }
-    - { name: timespan, type: text, default: 'NULL'}
-    - { name: number_geometries, type: integer, default: 'NULL'}
-
-- name: obs_getavailabletimespans
-  return_type: SETOF cdb_dataservices_client.obs_meta_timespan
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: bounds, type: "geometry(Geometry, 4326)", default: 'NULL' }
-    - { name: filter_tags, type: "text[]", default: 'NULL' }
-    - { name: numer_id, type: text, default: 'NULL' }
-    - { name: denom_id, type: text, default: 'NULL' }
-    - { name: geom_id, type: text, default: 'NULL'}
-
-- name: obs_legacybuildermetadata
-  return_type: TABLE(name text, subsection json)
-  multi_row: true
-  multi_field: true
-  params:
-    - { name: aggregate_type, type: text, default: 'NULL' }
-
-- name: cdb_service_quota_info
-  return_type: SETOF service_quota_info
+- name: cdb_service_quota_info_batch
+  return_type: SETOF service_quota_info_batch
   multi_row: true
   multi_field: true
   params:

client/renderer/sql-template-renderer

@@ -28,6 +28,18 @@ class SqlTemplateRenderer
     @function_signature['return_type']
   end
 
+  def requires_permission
+    @function_signature['requires_permission']
+  end
+
+  def permission_name
+    @function_signature['permission_name']
+  end
+
+  def permission_error
+    @function_signature['permission_error']
+  end
+
   def multi_field
     @function_signature['multi_field']
   end

client/renderer/templates/20_public_functions.erb

@@ -9,11 +9,16 @@ RETURNS <%= return_type %> AS $$
 DECLARE
   <%= return_declaration if not multi_row %>
   <%= user_org_declaration %>
+  <% unless superuser_function? -%>apikey_permissions json;<% end %>
 BEGIN
-  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+  <% unless requires_permission %>IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
     RAISE EXCEPTION 'The api_key must be provided';
-  END IF;
-  <% unless superuser_function? -%>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %>
+  END IF;<% end %>
+  <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN
+    RAISE EXCEPTION '<%= permission_error %>' USING ERRCODE = '01007';
+  END IF;<% end %>
+  <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %>
   -- JSON value stored "" is taken as literal
   IF username IS NULL OR username = '' OR username = '""' THEN
     RAISE EXCEPTION 'Username is a mandatory argument, check it out';
@@ -21,4 +26,5 @@ BEGIN
 
   <% return_statement do %><%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(<%= params(_with_user_org=true).join(', ') %>)<% end %>
 END;
-$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;

client/renderer/templates/25_exception_safe_private_functions.erb

@@ -10,11 +10,16 @@ DECLARE
   _returned_sqlstate TEXT;
   _message_text TEXT;
   _pg_exception_context TEXT;
+  <% unless superuser_function? -%>apikey_permissions json;<% end %>
 BEGIN
-  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+  <% unless requires_permission %>IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
     RAISE EXCEPTION 'The api_key must be provided';
-  END IF;
-  <% unless superuser_function? -%>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text);<% end %>
+  END IF;<% end %>
+  <% unless superuser_function? -%>SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  <% if requires_permission %>IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? '<%= permission_name %>' THEN
+    RAISE EXCEPTION '<%= permission_error %>';
+  END IF;<% end %>
+  <% else %>SELECT u, o INTO username, orgname FROM <%= DATASERVICES_CLIENT_SCHEMA %>._cdb_entity_config() AS (u text, o text, p json);<% end %>
   -- JSON value stored "" is taken as literal
   IF username IS NULL OR username = '' OR username = '""' THEN
     RAISE EXCEPTION 'Username is a mandatory argument, check it out';
@@ -32,4 +37,5 @@ BEGIN
         <%= return_statement %>
   END;
 END;
-$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE
+    SET search_path = pg_temp;

client/sql/05_utils.sql

@@ -0,0 +1,20 @@
+-- Taken from https://wiki.postgresql.org/wiki/Count_estimate
+CREATE FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) RETURNS INTEGER AS
+$func$
+DECLARE
+    rec   record;
+    ROWS  INTEGER;
+BEGIN
+    FOR rec IN EXECUTE 'EXPLAIN ' || query LOOP
+        ROWS := SUBSTRING(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)');
+        EXIT WHEN ROWS IS NOT NULL;
+    END LOOP;
+
+    RETURN ROWS;
+END
+$func$ LANGUAGE plpgsql;
+
+-- Taken from https://stackoverflow.com/a/48013356/351721
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) RETURNS text[] AS $f$
+    SELECT array_agg(x) || ARRAY[]::text[] FROM jsonb_array_elements_text($1) t(x);
+$f$ LANGUAGE sql IMMUTABLE;

client/sql/15_config_management.sql

@@ -1,6 +1,7 @@
 CREATE TYPE cdb_dataservices_client._entity_config AS (
     username text,
-    organization_name text
+    organization_name text,
+    apikey_permissions json
 );
 
 --
@@ -13,25 +14,23 @@ CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_entity_config()
 RETURNS record AS $$
 DECLARE
     result cdb_dataservices_client._entity_config;
+    apikey_config json;
     is_organization boolean;
-    username text;
-    organization_name text;
+    organization_name text DEFAULT NULL;
 BEGIN
+    SELECT cartodb.cdb_conf_getconf('api_keys_'||session_user) INTO apikey_config;
+
     SELECT cartodb.cdb_conf_getconf('user_config')->'is_organization' INTO is_organization;
     IF is_organization IS NULL THEN
         RAISE EXCEPTION 'User must have user configuration in the config table';
     ELSIF is_organization = TRUE THEN
-        SELECT nspname
-        FROM pg_namespace s
-        LEFT JOIN pg_roles r ON s.nspowner = r.oid
-        WHERE r.rolname = session_user INTO username;
         SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO organization_name;
-    ELSE
-        SELECT cartodb.cdb_conf_getconf('user_config')->>'entity_name' INTO username;
-        organization_name = NULL;
     END IF;
-    result.username = username;
+    result.username = apikey_config->>'username';
     result.organization_name = organization_name;
+    result.apikey_permissions = apikey_config->'permissions';
     RETURN result;
 END;
-$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL SAFE
+    SET search_path = pg_temp;
+

client/sql/16_custom_types.sql

@@ -4,28 +4,23 @@ CREATE TYPE cdb_dataservices_client.isoline AS (
     the_geom geometry(Multipolygon,4326)
 );
 
+CREATE TYPE cdb_dataservices_client.geocoding AS (
+    cartodb_id integer,
+    the_geom geometry(Point,4326),
+    metadata jsonb
+);
+
 CREATE TYPE cdb_dataservices_client.simple_route AS (
     shape geometry(LineString,4326),
     length real,
     duration integer
 );
 
--- For the OBS_Meta functions
-CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_name text, numer_description text, numer_weight text, numer_license text, numer_source text, numer_type text, numer_aggregate text, numer_extra jsonb, numer_tags jsonb, valid_denom boolean, valid_geom boolean, valid_timespan boolean);
-
-CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean);
-
-CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
-
-CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
-
-
 -- For quotas and services configuration
 CREATE TYPE cdb_dataservices_client.service_type AS ENUM (
     'isolines',
     'hires_geocoder',
-    'routing',
-    'observatory'
+    'routing'
 );
 
 CREATE TYPE cdb_dataservices_client.service_quota_info AS (
@@ -35,3 +30,12 @@ CREATE TYPE cdb_dataservices_client.service_quota_info AS (
     soft_limit BOOLEAN,
     provider TEXT
 );
+
+CREATE TYPE cdb_dataservices_client.service_quota_info_batch AS (
+    service cdb_dataservices_client.service_type,
+    monthly_quota NUMERIC,
+    used_quota NUMERIC,
+    soft_limit BOOLEAN,
+    provider TEXT,
+    max_batch_size NUMERIC
+);

client/sql/20_table_augmentation.sql

@@ -1,275 +0,0 @@
-CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
-CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-DECLARE
-  username text;
-  user_db_role text;
-  orgname text;
-  user_schema text;
-  result boolean;
-BEGIN
-  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
-    RAISE EXCEPTION 'The api_key must be provided';
-  END IF;
-
-  SELECT session_user INTO user_db_role;
-
-  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-  -- JSON value stored "" is taken as literal
-  IF username IS NULL OR username = '' OR username = '""' THEN
-    RAISE EXCEPTION 'Username is a mandatory argument';
-  END IF;
-
-  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
-    user_schema := 'public';
-  ELSE
-    user_schema := username;
-  END IF;
-
-  SELECT cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
-      username,
-      orgname,
-      user_db_role,
-      user_schema,
-      output_table_name,
-      params
-  ) INTO result;
-
-  RETURN result;
-END;
-$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(
-    table_name text,
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-DECLARE
-  username text;
-  user_db_role text;
-  orgname text;
-  dbname text;
-  user_schema text;
-  result boolean;
-BEGIN
-  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
-    RAISE EXCEPTION 'The api_key must be provided';
-  END IF;
-
-  SELECT session_user INTO user_db_role;
-
-  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
-  -- JSON value stored "" is taken as literal
-  IF username IS NULL OR username = '' OR username = '""' THEN
-    RAISE EXCEPTION 'Username is a mandatory argument';
-  END IF;
-
-  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
-    user_schema := 'public';
-  ELSE
-    user_schema := username;
-  END IF;
-
-  SELECT current_database() INTO dbname;
-
-  SELECT cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
-      username,
-      orgname,
-      user_db_role,
-      user_schema,
-      dbname,
-      table_name,
-      output_table_name,
-      params
-  ) INTO result;
-
-  RETURN result;
-END;
-$$ LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE;
-
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PrepareTableOBS_GetMeasure(
-    username text,
-    orgname text,
-    user_db_role text,
-    user_schema text,
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-    function_name = 'OBS_GetMeasure'
-    # Obtain return types for augmentation procedure
-    ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
-        "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
-        .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            function_name=plpy.quote_literal(function_name),
-            params=plpy.quote_literal(params)
-            )
-        )
-    if ds_return_metadata[0]["colnames"]:
-        colnames_arr = ds_return_metadata[0]["colnames"]
-        coltypes_arr = ds_return_metadata[0]["coltypes"]
-    else:
-        raise Exception('Error retrieving OBS_GetMeasure metadata')
-
-
-    # Prepare column and type strings required in the SQL queries
-    columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
-    columns_with_types = ','.join(columns_with_types_arr)
-
-    # Create a new table with the required columns
-    plpy.execute('CREATE TABLE "{schema}".{table_name} ( '
-        'cartodb_id int, the_geom geometry, {columns_with_types} '
-        ');'
-        .format(schema=user_schema, table_name=output_table_name, columns_with_types=columns_with_types)
-        )
-
-    plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
-        .format(schema=user_schema, table_name=output_table_name, user=user_db_role)
-        )
-
-    return True
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client.__DST_PopulateTableOBS_GetMeasure(
-    username text,
-    orgname text,
-    user_db_role text,
-    user_schema text,
-    dbname text,
-    table_name text,
-    output_table_name text,
-    params json
-) RETURNS boolean AS $$
-    function_name = 'OBS_GetMeasure'
-    # Obtain return types for augmentation procedure
-    ds_return_metadata = plpy.execute(
-        "SELECT colnames, coltypes "
-        "FROM cdb_dataservices_client._DST_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);" .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            function_name=plpy.quote_literal(function_name),
-            params=plpy.quote_literal(params)))
-
-    if ds_return_metadata[0]["colnames"]:
-        colnames_arr = ds_return_metadata[0]["colnames"]
-        coltypes_arr = ds_return_metadata[0]["coltypes"]
-    else:
-        raise Exception('Error retrieving OBS_GetMeasure metadata')
-
-    # Prepare column and type strings required in the SQL queries
-    columns_with_types_arr = [
-        colnames_arr[i] +
-        ' ' +
-        coltypes_arr[i] for i in range(
-            0,
-            len(colnames_arr))]
-    columns_with_types = ','.join(columns_with_types_arr)
-    aliased_colname_list = ','.join(
-        ['result.' + name for name in colnames_arr])
-
-    # Instruct the OBS server side to establish a FDW
-    # The metadata is obtained as well in order to:
-    #   - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
-    #   - (b) be able to tell OBS to free resources when done.
-    ds_fdw_metadata = plpy.execute(
-        "SELECT schemaname, tabname, servername "
-        "FROM cdb_dataservices_client._DST_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, "
-        "{schema}::text, {dbname}::text, {table_name}::text);" .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            user_db_role=plpy.quote_literal(user_db_role),
-            schema=plpy.quote_literal(user_schema),
-            dbname=plpy.quote_literal(dbname),
-            table_name=plpy.quote_literal(table_name)))
-
-    if ds_fdw_metadata[0]["schemaname"]:
-        server_schema = ds_fdw_metadata[0]["schemaname"]
-        server_table_name = ds_fdw_metadata[0]["tabname"]
-        server_name = ds_fdw_metadata[0]["servername"]
-    else:
-        raise Exception('Error connecting dataset via FDW')
-
-    # Create a new table with the required columns
-    plpy.execute(
-        'INSERT INTO "{schema}".{analysis_table_name} '
-        'SELECT ut.cartodb_id, ut.the_geom, {colname_list} '
-        'FROM "{schema}".{table_name} ut '
-        'LEFT JOIN _DST_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, '
-        '{function_name}::text, {params}::json) '
-        'AS result ({columns_with_types}, cartodb_id int)  '
-        'ON result.cartodb_id = ut.cartodb_id;' .format(
-            schema=user_schema,
-            analysis_table_name=output_table_name,
-            colname_list=aliased_colname_list,
-            table_name=table_name,
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            server_schema=plpy.quote_literal(server_schema),
-            server_table_name=plpy.quote_literal(server_table_name),
-            function_name=plpy.quote_literal(function_name),
-            params=plpy.quote_literal(params),
-            columns_with_types=columns_with_types))
-
-    # Wipe user FDW data from the server
-    wiped = plpy.execute(
-        "SELECT cdb_dataservices_client._DST_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, "
-        "{server_table_name}::text, {fdw_server}::text)" .format(
-            username=plpy.quote_nullable(username),
-            orgname=plpy.quote_nullable(orgname),
-            server_schema=plpy.quote_literal(server_schema),
-            server_table_name=plpy.quote_literal(server_table_name),
-            fdw_server=plpy.quote_literal(server_name)))
-
-    return True
-$$ LANGUAGE plpythonu VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_ConnectUserTable(
-    username text,
-    orgname text,
-    user_db_role text,
-    user_schema text,
-    dbname text,
-    table_name text
-)RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_ConnectUserTable;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_GetReturnMetadata(
-    username text,
-    orgname text,
-    function_name text,
-    params json
-) RETURNS cdb_dataservices_client.ds_return_metadata AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_GetReturnMetadata;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_FetchJoinFdwTableData(
-    username text,
-    orgname text,
-    table_schema text,
-    table_name text,
-    function_name text,
-    params json
-) RETURNS SETOF record AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_FetchJoinFdwTableData;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._DST_DisconnectUserTable(
-    username text,
-    orgname text,
-    table_schema text,
-    table_name text,
-    server_name text
-) RETURNS boolean AS $$
-    CONNECT cdb_dataservices_client._server_conn_str();
-    TARGET cdb_dataservices_server._DST_DisconnectUserTable;
-$$ LANGUAGE plproxy VOLATILE PARALLEL UNSAFE;

client/sql/21_bulk_geocoding_functions.sql

@@ -0,0 +1,90 @@
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point (query text,
+    street_column text, city_column text default null, state_column text default null, country_column text default null, batch_size integer DEFAULT NULL)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+DECLARE
+  query_row_count integer;
+  enough_quota boolean;
+  remaining_quota integer;
+  max_batch_size integer;
+
+  cartodb_id_batch integer;
+  batches_n integer;
+  DEFAULT_BATCH_SIZE CONSTANT numeric := 100;
+  MAX_SAFE_BATCH_SIZE CONSTANT numeric := 5000;
+
+  temp_table_name text;
+  username text;
+  orgname text;
+  apikey_permissions json;
+BEGIN
+  SELECT u, o, p INTO username, orgname, apikey_permissions FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text, p json);
+  IF apikey_permissions IS NULL OR NOT apikey_permissions::jsonb ? 'geocoding' THEN
+    RAISE EXCEPTION 'Geocoding permission denied' USING ERRCODE = '01007';
+  END IF;
+
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+
+  SELECT csqi.monthly_quota - csqi.used_quota AS remaining_quota, csqi.max_batch_size
+  INTO remaining_quota, max_batch_size
+  FROM cdb_dataservices_client.cdb_service_quota_info_batch() csqi
+  WHERE service = 'hires_geocoder';
+  RAISE DEBUG 'remaining_quota: %; max_batch_size: %', remaining_quota, max_batch_size;
+
+  IF batch_size IS NULL THEN
+    batch_size := max_batch_size;
+  ELSIF batch_size > max_batch_size THEN
+    RAISE EXCEPTION 'batch_size must be lower than %', max_batch_size + 1;
+  END IF;
+
+  IF batch_size > MAX_SAFE_BATCH_SIZE THEN
+    batch_size := MAX_SAFE_BATCH_SIZE;
+  END IF;
+
+  EXECUTE format('SELECT count(1), ceil(count(1)::float/%s) FROM (%s) _x', batch_size, query)
+  INTO query_row_count, batches_n;
+
+  RAISE DEBUG 'cdb_bulk_geocode_street_point --> query_row_count: %; query: %; country: %; state: %; city: %; street: %',
+      query_row_count, query, country_column, state_column, city_column, street_column;
+  SELECT cdb_dataservices_client.cdb_enough_quota('hires_geocoder', query_row_count) INTO enough_quota;
+  IF enough_quota IS NULL OR NOT enough_quota THEN
+    RAISE EXCEPTION 'Remaining quota: %. Estimated cost: %', remaining_quota, query_row_count;
+  END IF;
+
+  RAISE DEBUG 'batches_n: %', batches_n;
+
+  temp_table_name := 'bulk_geocode_street_' || md5(random()::text);
+
+  EXECUTE format('CREATE TEMPORARY TABLE %s ' ||
+   '(cartodb_id integer, the_geom public.geometry(Point,4326), metadata jsonb)',
+   temp_table_name);
+
+  select
+    coalesce(street_column, ''''''), coalesce(city_column, ''''''),
+    coalesce(state_column, ''''''), coalesce(country_column, '''''')
+  into street_column, city_column, state_column, country_column;
+
+  IF batches_n > 0 THEN
+    FOR cartodb_id_batch in 0..(batches_n - 1)
+    LOOP
+      EXECUTE format(
+        'WITH geocoding_data as (' ||
+        '   SELECT ' ||
+        '      json_build_object(''id'', cartodb_id, ''address'', %s, ''city'', %s, ''state'', %s, ''country'', %s) as data , ' ||
+        '      floor((row_number() over () - 1)::float/$1) as batch' ||
+        '   FROM (%s) _x' ||
+        ') ' ||
+        'INSERT INTO %s SELECT (cdb_dataservices_client._cdb_bulk_geocode_street_point(jsonb_agg(data))).* ' ||
+        'FROM geocoding_data ' ||
+        'WHERE batch = $2', street_column, city_column, state_column, country_column, query, temp_table_name)
+      USING batch_size, cartodb_id_batch;
+
+    END LOOP;
+  END IF;
+
+  RETURN QUERY EXECUTE 'SELECT * FROM ' || quote_ident(temp_table_name);
+END;
+$$  LANGUAGE 'plpgsql' SECURITY DEFINER VOLATILE PARALLEL UNSAFE
+    SET search_path = pg_temp;

client/sql/35_precheck_functions.sql

@@ -1,54 +0,0 @@
-CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_PreCheck(
-    source_query text,
-    parameters json
-) RETURNS boolean AS $$
-DECLARE
-    errors text[];
-    validator_errors text[];
-    validator_error text;
-    valid boolean;
-    geoms record;
-BEGIN
-    errors := (ARRAY[])::TEXT[];
-    FOR geoms IN EXECUTE format('SELECT ST_GeometryType(the_geom) as geom_type,
-                                 bool_and(st_isvalid(the_geom)) as valid,
-                                 avg(st_npoints(the_geom)) as avg_vertex,
-                                 ST_SetSRID(ST_Extent(the_geom), 4326) as extent,
-                                 count(*)::INT as numgeoms
-                                 FROM (%s) as _source
-                                 GROUP BY ST_GeometryType(the_geom)', source_query)
-    LOOP
-        IF geoms.geom_type NOT IN ('ST_Polygon', 'ST_MultiPolygon', 'ST_Point') THEN
-            errors := array_append(errors, format($data$'Geometry type %s not supported'$data$, geoms.geom_type));
-        END IF;
-
-        IF geoms.valid IS FALSE THEN
-            errors := array_append(errors, 'There are invalid geometries in the input data, please try to fix them');
-        END IF;
-
-        -- 1000 vertex for a geometry is a limit we have in the obs_getdata function. You can check here
-        -- https://github.com/CartoDB/observatory-extension/blob/1.6.0/src/pg/sql/41_observatory_augmentation.sql#L813
-        IF geoms.avg_vertex > 1000 THEN
-            errors := array_append(errors, 'The average number of vertices per geometry is greater than 1000, please try to simplify them');
-        END IF;
-
-        -- OBS specific part
-        EXECUTE 'SELECT valid, errors
-        FROM cdb_dataservices_client.OBS_MetadataValidation($1, $2, $3, $4)'
-        INTO valid, validator_errors
-        USING geoms.extent, geoms.geom_type, parameters, geoms.numgeoms;
-        IF valid is FALSE THEN
-            FOR validator_error IN EXECUTE 'SELECT unnest($1)' USING validator_errors
-            LOOP
-                errors := array_append(errors, validator_error);
-            END LOOP;
-        END IF;
-    END LOOP;
-
-    IF CARDINALITY(errors) > 0 THEN
-        RAISE EXCEPTION '%', errors;
-    END IF;
-
-    RETURN TRUE;
-END;
-$$ LANGUAGE 'plpgsql' VOLATILE PARALLEL UNSAFE;

client/sql/95_grant_execute_manual.sql

@@ -1,3 +1,4 @@
-GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure(output_table_name text, params json) TO publicuser;
-GRANT EXECUTE ON FUNCTION cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure(table_name text, output_table_name text, params json) TO publicuser;
-GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_PreCheck(source_query text, params JSON) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_bulk_geocode_street_point(query text, street_column text, city_column text, state_column text, country_column text, batch_size integer) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_count_estimate(query text) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_jsonb_array_casttext(jsonb) TO publicuser;

client/test/expected/00_installation_test.out

@@ -1,30 +1,17 @@
 -- Only show warning or error messages in the tests output
-SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION plproxy;
--- Install the extension
-CREATE EXTENSION cdb_dataservices_client;
--- Mock the server connection to point to this very test db
-SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');
+\set ECHO none
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
--- Mock the user configuration
-SELECT cartodb.cdb_conf_setconf('user_config', '{"is_organization": false, "entity_name": "test_user"}');
  cdb_conf_setconf 
 ------------------
  
 (1 row)
 
--- Mock the server schema
-CREATE SCHEMA cdb_dataservices_server;
--- Create a test user to check permissions
-DROP ROLE IF EXISTS test_regular_user;
-CREATE ROLE test_regular_user;
-GRANT publicuser TO test_regular_user;
-ALTER ROLE test_regular_user SET search_path TO public,cartodb,cdb_dataservices_client;
+CREATE SCHEMA
+DROP ROLE
+CREATE ROLE
+GRANT ROLE
+ALTER ROLE

client/test/expected/10_admin0_test.out

@@ -9,7 +9,26 @@ BEGIN
   RETURN NULL;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_geocode_admin0_polygon('Spain');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_geocode_admin0_polygon('Spain');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_geocode_admin0_polygon('Spain');
 NOTICE:  cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, <NULL>, Spain)
  cdb_geocode_admin0_polygon 
@@ -17,3 +36,10 @@ NOTICE:  cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regres
  
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/20_admin1_test.out

@@ -16,7 +16,30 @@ BEGIN
   RETURN NULL;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_geocode_admin1_polygon('California');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_geocode_admin1_polygon('California');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_geocode_admin1_polygon('California');
 NOTICE:  cdb_dataservices_client._cdb_geocode_admin1_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin1_polygon invoked with params (test_user, <NULL>, California)
  cdb_geocode_admin1_polygon 
@@ -31,3 +54,10 @@ NOTICE:  cdb_dataservices_client._cdb_geocode_admin1_polygon(4): [contrib_regres
  
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/21_bulk_geocoding_functions_test.out

@@ -0,0 +1,76 @@
+\set VERBOSITY terse
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
+RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$
+  SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size;
+$$ LANGUAGE SQL;
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
+RETURNS BOOLEAN as $$
+  SELECT FALSE;
+$$ LANGUAGE SQL;
+ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point_mocked;
+CREATE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+BEGIN
+  RAISE NOTICE 'called with this searches: %', searches;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE SET search_path = pg_temp;
+-- No permissions granted
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+ERROR:  Geocoding permission denied
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+ERROR:  Geocoding permission denied
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+ERROR:  Remaining quota: 0. Estimated cost: 1
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+ERROR:  Remaining quota: 0. Estimated cost: 1
+-- Check that when cdb_enough_quota returns true (ie. when soft_limit is set to true, even if not enough quota)
+-- it is able to proceed with the bulk geocode
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
+RETURNS BOOLEAN as $$
+  SELECT TRUE;
+$$ LANGUAGE SQL;
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+NOTICE:  called with this searches: [{"id": 1, "city": "", "state": "", "address": "Valladolid, Spain", "country": ""}]
+ cdb_bulk_geocode_street_point 
+-------------------------------
+(0 rows)
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+
+DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
+DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
+DROP FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point;
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;
+ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_mocked(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point;

client/test/expected/25_exception_safe_private_functions_test.out

@@ -27,9 +27,39 @@ BEGIN
   RETURN ret;
 END;
 $$ LANGUAGE 'plpgsql';
--- Use regular user role
-SET ROLE test_regular_user;
 -- Exercise the exception safe and the proxied functions
+-- No permissions granted
+SET ROLE test_regular_user; -- Use regular user role
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+ERROR:  Geocoding permission denied
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+ERROR:  Isolines permission denied
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+ERROR:  Routing permission denied
+-- No permissions granted but conf created
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": []}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SET ROLE test_regular_user; -- Use regular user role
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+ERROR:  Geocoding permission denied
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+ERROR:  Isolines permission denied
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+ERROR:  Routing permission denied
+-- Grant geocoding permissions
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SET ROLE test_regular_user; -- Use regular user role
 SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
 WARNING:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE ERROR: Not enough quota or any other exception whatsoever.
  _cdb_geocode_street_point_exception_safe 
@@ -50,3 +80,11 @@ WARNING:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regress
        |        |         
 (1 row)
 
+-- Remove permissions
+SET ROLE postgres;
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/30_namedplaces_test.out

@@ -23,7 +23,34 @@ BEGIN
   RETURN NULL;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_geocode_namedplace_point('Elx');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_geocode_namedplace_point('Elx');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_geocode_namedplace_point('Elx');
 NOTICE:  cdb_dataservices_client._cdb_geocode_namedplace_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_namedplace_point invoked with params (test_user, <NULL>, Elx)
  cdb_geocode_namedplace_point 
@@ -45,3 +72,10 @@ NOTICE:  cdb_dataservices_client._cdb_geocode_namedplace_point(5): [contrib_regr
  
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/40_postalcodes_test.out

@@ -16,7 +16,30 @@ BEGIN
   RETURN NULL;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
 NOTICE:  cdb_dataservices_client._cdb_geocode_postalcode_polygon(4): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_postalcode_polygon invoked with params (test_user, <NULL>, 03204, Spain)
  cdb_geocode_postalcode_polygon 
@@ -31,3 +54,10 @@ NOTICE:  cdb_dataservices_client._cdb_geocode_postalcode_point(4): [contrib_regr
  
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/50_ipaddresses_test.out

@@ -9,7 +9,26 @@ BEGIN
   RETURN NULL;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_geocode_ipaddress_point('8.8.8.8');
 NOTICE:  cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_ipaddress_point invoked with params (test_user, <NULL>, 8.8.8.8)
  cdb_geocode_ipaddress_point 
@@ -17,3 +36,10 @@ NOTICE:  cdb_dataservices_client._cdb_geocode_ipaddress_point(3): [contrib_regre
  
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/60_street_test.out

@@ -9,7 +9,62 @@ BEGIN
   RETURN NULL;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_geocode_street_point('One street, 1');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street, 1');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+ERROR:  Geocoding permission denied
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_geocode_street_point('One street, 1');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street, 1');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+ERROR:  Geocoding permission denied
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_geocode_street_point('One street, 1');
 NOTICE:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_geocoder_street_point invoked with params (test_user, <NULL>, One street, 1, <NULL>, <NULL>, <NULL>)
  cdb_geocode_street_point 
@@ -80,3 +135,10 @@ NOTICE:  cdb_dataservices_client._cdb_geocode_street_point(6): [contrib_regressi
  
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/80_route_point_to_point_test.out

@@ -22,7 +22,46 @@ BEGIN
   RETURN ret;
 END;
 $$ LANGUAGE 'plpgsql';
--- Exercise the public and the proxied functions
+-- -- Exercise the public and the proxied function
+-- No permissions granted
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+ERROR:  Routing permission denied
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+ERROR:  Routing permission denied
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles');
+ERROR:  Routing permission denied
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car');
+ERROR:  Routing permission denied
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]);
+ERROR:  Routing permission denied
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles');
+ERROR:  Routing permission denied
+-- Grant other permissions but routing
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+ERROR:  Routing permission denied
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+ERROR:  Routing permission denied
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles');
+ERROR:  Routing permission denied
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car');
+ERROR:  Routing permission denied
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]);
+ERROR:  Routing permission denied
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles');
+ERROR:  Routing permission denied
+-- Grant routing permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
 SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
 NOTICE:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_route_point_to_point invoked with params (test_user, <NULL>, 0101000000D53E1D8F19F455C0185B087250F24440, 0101000000465F419AB1F255C0D8B628B341EE4440, car, {}, kilometers)
  cdb_route_point_to_point 
@@ -65,3 +104,10 @@ NOTICE:  cdb_dataservices_client._cdb_route_with_waypoints(6): [contrib_regressi
  (,2.22,500)
 (1 row)
 
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/expected/95_data_observatory_tables_test.out

@@ -1,65 +0,0 @@
--- Add to the search path the schema
-SET search_path TO public,cartodb,cdb_dataservices_client;
-CREATE TABLE my_table(cartodb_id int);
-INSERT INTO my_table (cartodb_id) VALUES (1);
--- Mock the server functions
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
-RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
-BEGIN
-  RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
-END;
-$$ LANGUAGE 'plpgsql';
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
-RETURNS cdb_dataservices_client.ds_return_metadata AS $$
-BEGIN
-  RETURN (Array['total_pop'], Array['double precision']);
-END;
-$$ LANGUAGE 'plpgsql';
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
-RETURNS RECORD AS $$
-BEGIN
-  RETURN (23.4::double precision, 1::int);
-END;
-$$ LANGUAGE 'plpgsql';
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
-RETURNS boolean AS $$
-BEGIN
-  RETURN true;
-END;
-$$ LANGUAGE 'plpgsql';
--- Create a sample user table
-CREATE TABLE user_table (cartodb_id int, the_geom geometry);
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (2, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (3, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
--- Prepare a table with the total_pop column
-SELECT cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure('my_table_dst', '{"dummy":"dummy"}'::json);
- _dst_preparetableobs_getmeasure 
----------------------------------
- t
-(1 row)
-
--- The table should now exist and be empty
-SELECT * FROM my_table_dst;
- cartodb_id | the_geom | total_pop 
-------------+----------+-----------
-(0 rows)
-
--- Populate the table with measurement data
-SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'my_table_dst', '{"dummy":"dummy"}'::json);
- _dst_populatetableobs_getmeasure 
-----------------------------------
- t
-(1 row)
-
--- The table should now show the results
-SELECT * FROM my_table_dst;
- cartodb_id |                      the_geom                      | total_pop 
-------------+----------------------------------------------------+-----------
-          1 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |      23.4
-          2 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |          
-          3 | 0101000020E6100000F74FC902E07D52C05FE24CC7654B4440 |          
-(3 rows)
-
--- Clean tables
-DROP TABLE my_table_dst;

client/test/expected/999_permissions_test.out

@@ -3,8 +3,94 @@
 SET ROLE test_regular_user;
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
--- Exercise the public function
--- it is public, it shall work
+-- Check the regular user has no permissions on private functions
+SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell');
+ERROR:  permission denied for function _cdb_geocode_admin0_polygon
+SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell');
+ERROR:  permission denied for function _cdb_geocode_admin1_polygon
+SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell');
+ERROR:  permission denied for function _cdb_geocode_admin1_polygon
+SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol');
+ERROR:  permission denied for function _cdb_geocode_namedplace_point
+SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell');
+ERROR:  permission denied for function _cdb_geocode_namedplace_point
+SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world');
+ERROR:  permission denied for function _cdb_geocode_namedplace_point
+SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell');
+ERROR:  permission denied for function _cdb_geocode_postalcode_polygon
+SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell');
+ERROR:  permission denied for function _cdb_geocode_postalcode_point
+SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
+ERROR:  permission denied for function _cdb_geocode_ipaddress_point
+SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1');
+ERROR:  permission denied for function _cdb_geocode_street_point
+SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+ERROR:  permission denied for function _cdb_route_point_to_point
+-- -- Exercise the public function
+-- -- it is public, it shall work
+-- No permissions granted
+SELECT cdb_geocode_admin0_polygon('Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_admin1_polygon('California');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('one street, 1');
+ERROR:  Geocoding permission denied
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+ERROR:  Routing permission denied
+-- Grant other permissions but geocoding and routing
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["isolines"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SET ROLE test_regular_user; -- Use regular user role
+SELECT cdb_geocode_admin0_polygon('Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_admin1_polygon('California');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+ERROR:  Geocoding permission denied
+SELECT cdb_geocode_street_point('one street, 1');
+ERROR:  Geocoding permission denied
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+ERROR:  Routing permission denied
+-- Grant geocoding permissions
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing"]}');
+ cdb_conf_setconf 
+------------------
+ 
+(1 row)
+
+SET ROLE test_regular_user; -- Use regular user role
 SELECT cdb_geocode_admin0_polygon('Spain');
 NOTICE:  cdb_dataservices_client._cdb_geocode_admin0_polygon(3): [contrib_regression] REMOTE NOTICE: cdb_dataservices_server.cdb_geocode_admin0_polygon invoked with params (test_user, <NULL>, Spain)
  cdb_geocode_admin0_polygon 
@@ -82,26 +168,11 @@ NOTICE:  cdb_dataservices_client._cdb_route_point_to_point(7): [contrib_regressi
  (,5.33,100)
 (1 row)
 
--- Check the regular user has no permissions on private functions
-SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell');
-ERROR:  permission denied for function _cdb_geocode_admin0_polygon
-SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell');
-ERROR:  permission denied for function _cdb_geocode_admin1_polygon
-SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Sheol', 'Hell');
-ERROR:  permission denied for function _cdb_geocode_admin1_polygon
-SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol');
-ERROR:  permission denied for function _cdb_geocode_namedplace_point
-SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell');
-ERROR:  permission denied for function _cdb_geocode_namedplace_point
-SELECT _cdb_geocode_namedplace_point('evil_user', 'evil_orgname', 'Sheol', 'Hell', 'Ugly world');
-ERROR:  permission denied for function _cdb_geocode_namedplace_point
-SELECT _cdb_geocode_postalcode_polygon('evil_user', 'evil_orgname', '66666', 'Hell');
-ERROR:  permission denied for function _cdb_geocode_postalcode_polygon
-SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell');
-ERROR:  permission denied for function _cdb_geocode_postalcode_point
-SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
-ERROR:  permission denied for function _cdb_geocode_ipaddress_point
-SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1');
-ERROR:  permission denied for function _cdb_geocode_street_point
-SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
-ERROR:  permission denied for function _cdb_route_point_to_point
+-- Remove permissions
+SET ROLE postgres;
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+ cdb_conf_removeconf 
+---------------------
+ 
+(1 row)
+

client/test/sql/00_installation_test.sql

@@ -1,13 +1,13 @@
 -- Only show warning or error messages in the tests output
+\set ECHO none
+\set QUIET on
 SET client_min_messages TO WARNING;
--- Install dependencies
-CREATE EXTENSION postgis;
-CREATE EXTENSION plpythonu;
-CREATE EXTENSION cartodb;
-CREATE EXTENSION plproxy;
 
 -- Install the extension
-CREATE EXTENSION cdb_dataservices_client;
+CREATE EXTENSION cdb_dataservices_client CASCADE;
+
+\unset ECHO
+\unset QUIET
 
 -- Mock the server connection to point to this very test db
 SELECT cartodb.cdb_conf_setconf('geocoder_server_config', '{"connection_str": "dbname=contrib_regression host=127.0.0.1 user=postgres"}');

client/test/sql/10_admin0_test.sql

@@ -11,6 +11,18 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
+-- -- Exercise the public and the proxied function
 
--- Exercise the public and the proxied function
+-- No permissions granted
 SELECT cdb_geocode_admin0_polygon('Spain');
+
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+SELECT cdb_geocode_admin0_polygon('Spain');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+SELECT cdb_geocode_admin0_polygon('Spain');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/20_admin1_test.sql

@@ -19,7 +19,21 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
+-- -- Exercise the public and the proxied function
 
--- Exercise the public and the proxied function
+-- No permissions granted
 SELECT cdb_geocode_admin1_polygon('California');
 SELECT cdb_geocode_admin1_polygon('California', 'United States');
+
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+SELECT cdb_geocode_admin1_polygon('California');
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+SELECT cdb_geocode_admin1_polygon('California');
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/21_bulk_geocoding_functions_test.sql

@@ -0,0 +1,60 @@
+\set VERBOSITY terse
+
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch() RENAME TO cdb_service_quota_info_batch_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch ()
+RETURNS SETOF cdb_dataservices_client.service_quota_info_batch AS $$
+  SELECT 'hires_geocoder'::cdb_dataservices_client.service_type AS service, 0::NUMERIC AS monthly_quota, 0::NUMERIC AS used_quota, FALSE AS soft_limit, 'google' AS provider, 1::NUMERIC AS max_batch_size;
+$$ LANGUAGE SQL;
+
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota_mocked;
+CREATE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
+RETURNS BOOLEAN as $$
+  SELECT FALSE;
+$$ LANGUAGE SQL;
+
+ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point_mocked;
+CREATE FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point(searches jsonb)
+RETURNS SETOF cdb_dataservices_client.geocoding AS $$
+BEGIN
+  RAISE NOTICE 'called with this searches: %', searches;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER STABLE PARALLEL UNSAFE SET search_path = pg_temp;
+
+-- No permissions granted
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+-- Test bulk size not mandatory (it will get the optimal)
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''', null, null, null, null);
+-- Test quota check by mocking quota 0
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+
+-- Check that when cdb_enough_quota returns true (ie. when soft_limit is set to true, even if not enough quota)
+-- it is able to proceed with the bulk geocode
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_enough_quota (service TEXT ,input_size NUMERIC)
+RETURNS BOOLEAN as $$
+  SELECT TRUE;
+$$ LANGUAGE SQL;
+SELECT cdb_dataservices_client.cdb_bulk_geocode_street_point('select 1 as cartodb_id', '''Valladolid, Spain''');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');
+
+DROP FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch;
+DROP FUNCTION cdb_dataservices_client.cdb_enough_quota;
+DROP FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point;
+
+ALTER FUNCTION cdb_dataservices_client.cdb_enough_quota_mocked (service TEXT ,input_size NUMERIC) RENAME TO cdb_enough_quota;
+ALTER FUNCTION cdb_dataservices_client.cdb_service_quota_info_batch_mocked() RENAME TO cdb_service_quota_info_batch;
+ALTER FUNCTION cdb_dataservices_client._cdb_bulk_geocode_street_point_mocked(searches jsonb) RENAME TO _cdb_bulk_geocode_street_point;

client/test/sql/25_exception_safe_private_functions_test.sql

@@ -32,11 +32,30 @@ END;
 $$ LANGUAGE 'plpgsql';
 
 
-
--- Use regular user role
-SET ROLE test_regular_user;
-
 -- Exercise the exception safe and the proxied functions
+
+-- No permissions granted
+SET ROLE test_regular_user; -- Use regular user role
 SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
 SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
 SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+
+-- No permissions granted but conf created
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": []}');
+SET ROLE test_regular_user; -- Use regular user role
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+
+-- Grant geocoding permissions
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing", "isolines"]}');
+SET ROLE test_regular_user; -- Use regular user role
+SELECT _cdb_geocode_street_point_exception_safe('One street, 1');
+SELECT * FROM _cdb_isodistance_exception_safe('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300]::integer[]);
+SELECT * FROM _cdb_route_point_to_point_exception_safe('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+
+-- Remove permissions
+SET ROLE postgres;
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/30_namedplaces_test.sql

@@ -27,8 +27,24 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+
+-- No permissions granted
 SELECT cdb_geocode_namedplace_point('Elx');
 SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
 SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
 
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+SELECT cdb_geocode_namedplace_point('Elx');
+SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+SELECT cdb_geocode_namedplace_point('Elx');
+SELECT cdb_geocode_namedplace_point('Elx', 'Spain');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/40_postalcodes_test.sql

@@ -19,6 +19,21 @@ BEGIN
 END;
 $$ LANGUAGE 'plpgsql';
 
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+
+-- No permissions granted
 SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
 SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/50_ipaddresses_test.sql

@@ -12,5 +12,18 @@ END;
 $$ LANGUAGE 'plpgsql';
 
 
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+
+-- No permissions granted
 SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/60_street_test.sql

@@ -12,7 +12,9 @@ END;
 $$ LANGUAGE 'plpgsql';
 
 
--- Exercise the public and the proxied function
+-- -- Exercise the public and the proxied function
+
+-- No permissions granted
 SELECT cdb_geocode_street_point('One street, 1');
 SELECT cdb_geocode_street_point('One street', 'city');
 SELECT cdb_geocode_street_point('One street', 'city', 'state');
@@ -23,3 +25,32 @@ SELECT cdb_geocode_street_point('One street', 'city');
 SELECT cdb_geocode_street_point('One street', 'city', 'state');
 SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
 SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+
+-- Grant other permissions but geocoding
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing", "isolines"]}');
+SELECT cdb_geocode_street_point('One street, 1');
+SELECT cdb_geocode_street_point('One street', 'city');
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+SELECT cdb_geocode_street_point('One street, 1');
+SELECT cdb_geocode_street_point('One street', 'city');
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+
+-- Grant geocoding permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding"]}');
+SELECT cdb_geocode_street_point('One street, 1');
+SELECT cdb_geocode_street_point('One street', 'city');
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+SELECT cdb_geocode_street_point('One street, 1');
+SELECT cdb_geocode_street_point('One street', 'city');
+SELECT cdb_geocode_street_point('One street', 'city', 'state');
+SELECT cdb_geocode_street_point('One street', 'city', 'state', 'country');
+SELECT cdb_geocode_street_point('One street', 'city', NULL, 'country');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/80_route_point_to_point_test.sql

@@ -28,12 +28,33 @@ END;
 $$ LANGUAGE 'plpgsql';
 
 
--- Exercise the public and the proxied functions
+-- -- Exercise the public and the proxied function
+
+-- No permissions granted
 SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
 SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
 SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles');
-
 SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car');
 SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]);
 SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles');
 
+-- Grant other permissions but routing
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "isolines"]}');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles');
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car');
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]);
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles');
+
+-- Grant routing permissions
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["routing"]}');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]);
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car', ARRAY[]::text[], 'miles');
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car');
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY['mode_type=shortest']::text[]);
+SELECT cdb_route_with_waypoints(Array['POINT(-87.81406 41.89308)'::geometry,'POINT(-87.80406 41.87308)'::geometry,'POINT(-87.79209 41.86138)'::geometry], 'car', ARRAY[]::text[], 'miles');
+
+-- Remove permissions
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

client/test/sql/95_data_observatory_tables_test.sql

@@ -1,56 +0,0 @@
--- Add to the search path the schema
-SET search_path TO public,cartodb,cdb_dataservices_client;
-
-CREATE TABLE my_table(cartodb_id int);
-
-INSERT INTO my_table (cartodb_id) VALUES (1);
-
--- Mock the server functions
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
-RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
-BEGIN
-  RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
-END;
-$$ LANGUAGE 'plpgsql';
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_GetReturnMetadata(username text, orgname text, function_name text, params json)
-RETURNS cdb_dataservices_client.ds_return_metadata AS $$
-BEGIN
-  RETURN (Array['total_pop'], Array['double precision']);
-END;
-$$ LANGUAGE 'plpgsql';
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
-RETURNS RECORD AS $$
-BEGIN
-  RETURN (23.4::double precision, 1::int);
-END;
-$$ LANGUAGE 'plpgsql';
-
-CREATE OR REPLACE FUNCTION cdb_dataservices_server._DST_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
-RETURNS boolean AS $$
-BEGIN
-  RETURN true;
-END;
-$$ LANGUAGE 'plpgsql';
-
--- Create a sample user table
-CREATE TABLE user_table (cartodb_id int, the_geom geometry);
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (1, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (2, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-INSERT INTO user_table(cartodb_id, the_geom) VALUES (3, '0101000020E6100000F74FC902E07D52C05FE24CC7654B4440');
-
--- Prepare a table with the total_pop column
-SELECT cdb_dataservices_client._DST_PrepareTableOBS_GetMeasure('my_table_dst', '{"dummy":"dummy"}'::json);
-
--- The table should now exist and be empty
-SELECT * FROM my_table_dst;
-
--- Populate the table with measurement data
-SELECT cdb_dataservices_client._DST_PopulateTableOBS_GetMeasure('user_table', 'my_table_dst', '{"dummy":"dummy"}'::json);
-
--- The table should now show the results
-SELECT * FROM my_table_dst;
-
--- Clean tables
-DROP TABLE my_table_dst;

client/test/sql/999_permissions_test.sql

@@ -5,20 +5,6 @@ SET ROLE test_regular_user;
 -- Add to the search path the schema
 SET search_path TO public,cartodb,cdb_dataservices_client;
 
--- Exercise the public function
--- it is public, it shall work
-SELECT cdb_geocode_admin0_polygon('Spain');
-SELECT cdb_geocode_admin1_polygon('California');
-SELECT cdb_geocode_admin1_polygon('California', 'United States');
-SELECT cdb_geocode_namedplace_point('Elx');
-SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
-SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
-SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
-SELECT cdb_geocode_postalcode_point('03204', 'Spain');
-SELECT cdb_geocode_ipaddress_point('8.8.8.8');
-SELECT cdb_geocode_street_point('one street, 1');
-SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
-
 -- Check the regular user has no permissions on private functions
 SELECT _cdb_geocode_admin0_polygon('evil_user', 'evil_orgname', 'Hell');
 SELECT _cdb_geocode_admin1_polygon('evil_user', 'evil_orgname', 'Hell');
@@ -31,3 +17,54 @@ SELECT _cdb_geocode_postalcode_point('evil_user', 'evil_orgname', '66666', 'Hell
 SELECT _cdb_geocode_ipaddress_point('evil_user', 'evil_orgname', '8.8.8.8');
 SELECT _cdb_geocode_street_point('evil_user', 'evil_orgname', 'one street, 1');
 SELECT _cdb_route_point_to_point('evil_user', 'evil_orgname', 'POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+
+-- -- Exercise the public function
+-- -- it is public, it shall work
+-- No permissions granted
+SELECT cdb_geocode_admin0_polygon('Spain');
+SELECT cdb_geocode_admin1_polygon('California');
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+SELECT cdb_geocode_namedplace_point('Elx');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+SELECT cdb_geocode_street_point('one street, 1');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+
+-- Grant other permissions but geocoding and routing
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["isolines"]}');
+SET ROLE test_regular_user; -- Use regular user role
+SELECT cdb_geocode_admin0_polygon('Spain');
+SELECT cdb_geocode_admin1_polygon('California');
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+SELECT cdb_geocode_namedplace_point('Elx');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+SELECT cdb_geocode_street_point('one street, 1');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+
+-- Grant geocoding permissions
+SET ROLE postgres;
+SELECT CDB_Conf_SetConf('api_keys_postgres', '{"username": "test_user", "permissions": ["geocoding", "routing"]}');
+SET ROLE test_regular_user; -- Use regular user role
+SELECT cdb_geocode_admin0_polygon('Spain');
+SELECT cdb_geocode_admin1_polygon('California');
+SELECT cdb_geocode_admin1_polygon('California', 'United States');
+SELECT cdb_geocode_namedplace_point('Elx');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia');
+SELECT cdb_geocode_namedplace_point('Elx', 'Valencia', 'Spain');
+SELECT cdb_geocode_postalcode_polygon('03204', 'Spain');
+SELECT cdb_geocode_postalcode_point('03204', 'Spain');
+SELECT cdb_geocode_ipaddress_point('8.8.8.8');
+SELECT cdb_geocode_street_point('one street, 1');
+SELECT cdb_route_point_to_point('POINT(-87.81406 41.89308)'::geometry,'POINT(-87.79209 41.86138)'::geometry, 'car');
+
+-- Remove permissions
+SET ROLE postgres;
+SELECT CDB_Conf_RemoveConf('api_keys_postgres');

doc/API.md

@@ -1,13 +0,0 @@
-# Data Services API
-
-The CARTO Data Services API offers a set of location based services that can be used programatically to empower your geospatial applications.
-
-## Documentation
-
-* [Overview](overview.md)
-* [Geocoding Functions](geocoding_functions.md)
-* [Isoline Functions](isoline_functions.md)
-* [Routing Functions](routing_functions.md)
-* [Demographic Functions](demographic_functions.md)
-* [Segmentation Functions](segmentation_functions.md)
-* [Quota Information](quota_information.md)

doc/demographic_functions.md

@@ -1,155 +0,0 @@
-# Demographic Functions
-
-The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
-
-_**Note:** The Demographic Snapshot functions are only available for the United States._
-
-## OBS_GetDemographicSnapshot( point geometry )
-
-Fields returned include information about income, education, transportation, race, and more. Not all fields will have information for every coordinate queried.
-
-### Arguments
-
-Name | Description | Example Values
---- | --- | ---
-point geometry | A point geometry. You can use the helper function, `CDB_LatLng` to quickly generate one from latitude and longitude | `CDB_LatLng(40.760410,-73.964242)`
-
-### Returns
-
-The Demographic Snapshot contains a broad subset of demographic measures in the Data Observatory. Over 80 measurements are returned by a single API request. For each demographic measure, the API returns the following values.
-
-Value | Name | Tablename | Aggregate | Type | Description
------ | ---- | --------- | --------- | ---- |------------
-The value of the measure at the point you requested | The name of the measure | The table it was drawn from | Indicated if the measure is a count or median. | postgresql | A description of the measure
-
-For example the "Female Population" measure returns
-
-```json
-obs_getdemographicsnapshot: {
-   "value": 32.5395066379175,
-   "name": "Female Population",
-   "tablename": "obs_1a098da56badf5f32e336002b0a81708c40d29cd",
-   "aggregate": "sum",
-   "type": "Numeric",
-   "description": "The number of people within each geography who are female."
-}
-```
-
-**For details, see the [Glossary of Demographic Measures](#glossary-of-demographic-measures).**
-
-### Examples
-
-```bash
-https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
-OBS_GetDemographicSnapshot({{point geometry}})
-```
-
-##### Get the Geographic Snapshot of a Demographic
-
-__Get the Demographic Snapshot at Camp David__
-
-```bash
-https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
-OBS_GetDemographicSnapshot(CDB_LatLng(39.648333, -77.465))
-```
-
-__Get the Demographic Snapshot in the Upper West Side__
-
-```bash
-https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
-OBS_GetDemographicSnapshot(CDB_LatLng(40.80, -73.960))
-```
-
-## Glossary of Demographic Measures
-
-This list contains the demographic measures and response names for results from the ```OBS_GetDemographicSnapshot``` function.
-
- Measure name | Measure Description | Response Mame | Response Units
---- | --- | --- | ---
-Total Population | The total number of all people living in a given geographic area. This is a very useful catch-all denominator when calculating rates. | total_pop | Count per sq. km
-Male Population | The number of people within each geography who are male. | male_pop | Count per sq. km
-Female Population | The number of people within each geography who are female.| female_pop | Count per sq. km
-Population not Hispanic	| The number of people not identifying as Hispanic or Latino in each geography. | not_hispanic_pop | Count per sq. km
-White Population | The number of people identifying as white, non-Hispanic in each geography. | white_pop | Count per sq. km
-Black or African American Population| The number of people identifying as black or African American, non-Hispanic in each geography. | black_pop | Count per sq. km
-American Indian and Alaska Native Population | The number of people identifying as American Indian or Alaska native in each geography.| amerindian_pop| Count per sq. km
-Asian Population | The number of people identifying as Asian, non-Hispanic in each geography.| asian_pop | Count per sq. km
-Other Race population | The number of people identifying as another race in each geography. | other_race_pop | Count per sq. km
-Two or more races population| The number of people identifying as two or more races in each geography | two_or_more_races_pop | Count per sq. km
-Hispanic Population | The number of people identifying as Hispanic or Latino in each geography. | hispanic_pop | Count per sq. km
-Not a U.S. Citizen Population | The number of people within each geography who indicated that they are not U.S. citizens. | not_us_citizen_pop | Count per sq. km
-Median Age | The median age of all people in a given geographic area.| median_age | Years
-Children under 18 Years of Age | The number of people within each geography who are under 18 years of age.| children | Count per sq. km
-Population 15 Years and Over | The number of people in a geographic area who are over the age of 15. This is used mostly as a denominator of marital status. | pop_15_and_over | Count per sq. km
-Population 3 Years and Over | The total number of people in each geography age 3 years and over. This denominator is mostly used to calculate rates of school enrollment. | population_3_years_over | Count per sq. km
-Population 5 Years and Over | The number of people in a geographic area who are over the age of 5. This is primarily used as a denominator of measures of language spoken at home.| pop_5_years_over | Count per sq. km
-Workers over the Age of 16 | The number of people in each geography who work. Workers include those employed at private for-profit companies, the self-employed, government workers and non-profit employees. | workers_16_and_over | Count per sq. km
-Workers age 16 and over who do not work from home| The number of workers over the age of 16 who do not work from home in a geographic area| commuters_16_over | Count per sq. km
-Commuters by Car, Truck, or Van | The number of workers age 16 years and over within a geographic area who primarily traveled to work by car, truck or  van. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. | commuters_by_car_truck_van | Count per sq. km
-Commuters who drove alone | The number of workers age 16 years and over within a geographic area who primarily traveled by car driving alone. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. | commuters_drove_alone | Count per sq. km
-Commuters by Carpool| The number of workers age 16 years and over within a geographic area who primarily traveled to work by carpool. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. | commuters_by_carpool | Count per sq. km
-Commuters by Public Transportation 	| The number of workers age 16 years and over within a geographic area who primarily traveled to work by public transportation.  This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. | commuters_by_public_transportation 	| Count per sq. km 	|
-Commuters by Bus | The number of workers age 16 years and over within a geographic area who primarily traveled to work by bus.  This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. This is a subset of workers who commuted by public transport. | commuters_by_bus| Count per sq. km
-Commuters by Subway or Elevated | The number of workers age 16 years and over within a geographic area who primarily traveled to work by subway or elevated train. This is the principal mode of travel or type of conveyance, by distance rather than time, that the worker usually used to get from home to work. This is a subset of workers who commuted by public transport. | commuters_by_subway_or_elevated | Count per sq. km
-Walked to Work | The number of workers age 16 years and over within a geographic area who primarily walked to work. This would mean that of any way of getting to work, they travelled the most distance walking. | walked_to_work | Count per sq. km
-Worked at Home | The count within a geographical area of workers over the age of 16 who worked at home. | worked_at_home | Count per sq. km
-Number of workers with less than 10 minute commute | The number of workers over the age of 16 who do not work from home and commute in less than 10 minutes in a geographic area. | commute_less_10_mins | Count per sq. km
-Number of workers with a commute between 10 and 14 minutes| The number of workers over the age of 16 who do not work from home and commute in between 10 and 14 minutes in a geographic area. | commute_10_14_mins | Count per sq. km
-Number of workers with a commute between 15 and 19 minutes | The number of workers over the age of 16 who do not work from home and commute in between 15 and 19 minutes in a geographic area. | commute_15_19_mins | Count per sq. km
-Number of workers with a commute between 20 and 24 minutes | The number of workers over the age of 16 who do not work from home and commute in between 20 and 24 minutes in a geographic area. | commute_20_24_mins | Count per sq. km
-Number of workers with a commute between 25 and 29 minutes | The number of workers over the age of 16 who do not work from home and commute in between 25 and 29 minutes in a geographic area. | commute_25_29_mins| Count per sq. km
-Number of workers with a commute between 30 and 34 minutes | The number of workers over the age of 16 who do not work from home and commute in between 30 and 34 minutes in a geographic area. | commute_30_34_mins | Count per sq. km
-Number of workers with a commute between 35 and 44 minutes | The number of workers over the age of 16 who do not work from home and commute in between 35 and 44 minutes in a geographic area. | commute_35_44_mins | Count per sq. km
-Number of workers with a commute between 45 and 59 minutes | The number of workers over the age of 16 who do not work from home and commute in between 45 and 59 minutes in a geographic area. | commute_45_59_mins | Count per sq. km
-Number of workers with a commute of over 60 minutes | The number of workers over the age of 16 who do not work from home and commute in over 60 minutes in a geographic area.| commute_60_more_mins | Count per sq. km
-Aggregate travel time to work | The total number of minutes every worker over the age of 16 who did not work from home spent spent commuting to work in one day in a geographic area. | aggregate_travel_time_to_work | Minutes
-Households | A count of the number of households in each geography. A household consists of one or more people who live in the same dwelling and also share at meals or living accommodation, and may consist of a single family or some other grouping of people. | households | Count per sq. km
-Never Married | The number of people in a geographic area who have never been married. | pop_never_married | Count per sq. km
-Currently married| The number of people in a geographic area who are currently married. | pop_now_married | Count per sq. km
-Married but separated | The number of people in a geographic area who are married but separated.| pop_separated | Count per sq. km
-Widowed | The number of people in a geographic area who are widowed.| pop_widowed | Count per sq. km
-Divorced | The number of people in a geographic area who are divorced. | pop_divorced | Count per sq. km
-Students Enrolled in School | The total number of people in each geography currently enrolled at any level of school, from nursery or pre-school to advanced post-graduate education. Only includes those over the age of 3. | in_school | Count per sq. km
-Students Enrolled in Grades 1 to 4 | The total number of people in each geography currently enrolled in grades 1 through 4 inclusive.  This corresponds roughly to elementary school. | in_grades_1_to_4 | Count per sq. km
-Students Enrolled in Grades 5 to 8 | The total number of people in each geography currently enrolled in grades 5 through 8 inclusive.  This corresponds roughly to middle school. | in_grades_5_to_8 | Count per sq. km
-Students Enrolled in Grades 9 to 12 | The total number of people in each geography currently enrolled in grades 9 through 12 inclusive.  This corresponds roughly to high school. | in_grades_9_to_12 | Count per sq. km
-Students Enrolled as Undergraduate in College | The number of people in a geographic area who are enrolled in college at the undergraduate level. Enrollment refers to being registered or listed as a student in an educational program leading to a college degree. This may be a public school or college, a private school or college. | in_undergrad_college | Count per sq. km
-Population 25 Years and Over | The number of people in a geographic area who are over the age of 25. This is used mostly as a denominator of educational attainment. | pop_25_years_over | Count per sq. km
-Population Completed High School | The number of people in a geographic area over the age of 25 who completed high school, and did not complete a more advanced degree.	| high_school_diploma| Count per sq. km
-Population completed less than one year of college, no degree | The number of people in a geographic area over the age of 25 who attended college for less than one year and no further. | less_one_year_college | Count per sq. km
-Population completed more than one year of college, no degree | The number of people in a geographic area over the age of 25 who attended college for more than one year but did not obtain a degree. | one_year_more_college | Count per sq. km
-Population Completed Associate's Degree | The number of people in a geographic area over the age of 25 who obtained a associate's degree, and did not complete a more advanced degree.| associates_degree | Count per sq. km
-Population Completed Bachelor's Degree| The number of people in a geographic area over the age of 25 who obtained a bachelor's degree, and did not complete a more advanced degree. | bachelors_degree| Count per sq. km
-Population Completed Master's Degree | The number of people in a geographic area over the age of 25 who obtained a master's degree, but did not complete a more advanced degree. | masters_degree | Count per sq. km
-Speaks only English at Home | The number of people in a geographic area over age 5 who speak only English at home. | speak_only_english_at_home | Count per sq. km
-Speaks Spanish at Home | The number of people in a geographic area over age 5 who speak Spanish at home, possibly in addition to other languages. | speak_spanish_at_home | Count per sq. km
-Population for Whom Poverty Status Determined | The number of people in each geography who could be identified as either living in poverty or not.  This should be used as the denominator when calculating poverty rates, as it excludes people for whom it was not possible to determine poverty. | pop_determined_poverty_status | Count per sq. km
-Income In The Past 12 Months Below Poverty Level | The number of people in a geographic area who are part of a family (which could be just them as an individual) determined to be "in poverty" following the [Office of Management and Budget's Directive 14](https://www.census.gov/hhes/povmeas/methodology/ombdir14.html). | poverty | Count per sq. km
-Households with income less than $10,000 | The number of households in a geographic area whose annual income was less than $10,000. | income_less_10000 | Count per sq. km
-Households with income of $10,000 to $14,999 | The number of households in a geographic area whose annual income was between $10,000 and $14,999. | income_10000_14999 | Count per sq. km
-Households with income of $15,000 to $19,999 | The number of households in a geographic area whose annual income was between $15,000 and $19,999. | income_15000_19999 | Count per sq. km
-Households with income of $20,000 To $24,999 | The number of households in a geographic area whose annual income was between $20,000 and $24,999. | income_20000_24999 | Count per sq. km
-Households with income of $25,000 To $29,999 | The number of households in a geographic area whose annual income was between $20,000 and $24,999. | income_25000_29999 | Count per sq. km
-Households with income of $30,000 To $34,999 | The number of households in a geographic area whose annual income was between $30,000 and $34,999. | income_30000_34999 | Count per sq. km
-Households with income of $35,000 To $39,999 | The number of households in a geographic area whose annual income was between $35,000 and $39,999. | income_35000_39999 | Count per sq. km
-Households with income of $40,000 To $44,999 | The number of households in a geographic area whose annual income was between $40,000 and $44,999. | income_40000_44999| Count per sq. km
-Households with income of $45,000 To $49,999 | The number of households in a geographic area whose annual income was between $45,000 and $49,999. | income_45000_49999 | Count per sq. km
-Households with income of $50,000 To $59,999 | The number of households in a geographic area whose annual income was between $50,000 and $59,999. | income_50000_59999 | Count per sq. km
-Households with income of $60,000 To $74,999 | The number of households in a geographic area whose annual income was between $60,000 and $74,999. | income_60000_74999 | Count per sq. km
-Households with income of $75,000 To $99,999 | The number of households in a geographic area whose annual income was between $75,000 and $99,999. | income_75000_99999 | Count per sq. km
-Households with income of $100,000 To $124,999 | The number of households in a geographic area whose annual income was between $100,000 and $124,999. | income_100000_124999 | Count per sq. km
-Households with income of $125,000 To $149,999 | The number of households in a geographic area whose annual income was between $125,000 and $149,999. | income_125000_149999 | Count per sq. km
-Households with income of $150,000 To $199,999 | The number of households in a geographic area whose annual income was between $150,000 and $1999,999. | income_150000_199999 | Count per sq. km
-Households with income of $200,000 Or More | The number of households in a geographic area whose annual income was more than $200,000. | income_200000_or_more | Count per sq. km
-Median Household Income in the past 12 Months | Within a geographic area, the median income received by every household on a regular basis before payments for personal income taxes, social security, union dues, medicare deductions, etc.  It includes income received from wages, salary, commissions, bonuses, and tips; self-employment income from own nonfarm or farm businesses, including proprietorships and partnerships; interest, dividends, net rental income, royalty income, or income from estates and trusts; Social Security or Railroad Retirement income; Supplemental Security Income (SSI); any cash public assistance or welfare payments from the state or local welfare office; retirement, survivor, or disability benefits; and any other sources of income received regularly such as Veterans' (VA) payments, unemployment and/or worker's compensation, child support, and alimony. | median_income | USD
-Per Capita Income in the past 12 Months |  	| income_per_capita | USD
-Gini Index | A measurement of the income distribution of a country's residents. | gini_index | None
-Housing Units | A count of housing units in each geography. A housing unit is a house, an apartment, a mobile home or trailer, a group of rooms, or a single room occupied as separate living quarters, or if vacant, intended for occupancy as separate living quarters. | housing_units | Count per sq. km
-Vacant Housing Units | The count of vacant housing units in a geographic area. A housing unit is vacant if no one is living in it at the time of enumeration, unless its occupants are only temporarily absent. Units temporarily occupied at the time of enumeration entirely by people who have a usual residence elsewhere are also classified as vacant. | vacant_housing_units | Count per sq. km
-Vacant Housing Units for Rent | The count of vacant housing units in a geographic area that are for rent. A housing unit is vacant if no one is living in it at the time of enumeration, unless its occupants are only temporarily absent. Units temporarily occupied at the time of enumeration entirely by people who have a usual residence elsewhere are also classified as vacant. | vacant_housing_units_for_rent | Count per sq. km
-Vacant Housing Units for Sale| The count of vacant housing units in a geographic area that are for sale. A housing unit is vacant if no one is living in it at the time of enumeration, unless its occupants are only temporarily absent. Units temporarily occupied at the time of enumeration entirely by people who have a usual residence elsewhere are also classified as vacant. | vacant_housing_units_for_sale | Count per sq. km
-Owner-occupied Housing Units | The count of owner occupied housing units in a geographic area. | owner_occupied_housing_units | Count per sq. km
-Owner-occupied Housing Units valued at $1,000,000 or more. | The count of owner occupied housing units in a geographic area that are valued at $1,000,000 or more. Value is the respondent's estimate of how much the property (house and lot, mobile home and lot, or condominium unit) would sell for if it were for sale. | million_dollar_housing_units | Count per sq. km
-Owner-occupied Housing Units with a Mortgage | The count of housing units within a geographic area that are mortagaged. "Mortgage" refers to all forms of debt where the property is pledged as security for repayment of the debt, including deeds of trust, trust deed, contracts to purchase, land contracts, junior mortgages, and home equity loans. | mortgaged_housing_units | Count per sq. km
-Median Rent | The median contract rent within a geographic area. The contract rent is the monthly rent agreed to or contracted for, regardless of any furnishings, utilities, fees, meals, or services that may be included. For vacant units, it is the monthly rent asked for the rental unit at the time of interview.| median_rent | USD
-Percent of Household Income Spent on Rent | Within a geographic area, the median percentage of household income which was spent on gross rent.  Gross rent is the amount of the contract rent plus the estimated average monthly cost of utilities (electricity, gas, water, sewer etc.) and fuels (oil, coal, wood, etc.) if these are paid by the renter. Household income is the sum of the income of all people 15 years and older living in the household. | percent_income_spent_on_rent | Percent

doc/segmentation_functions.md

@@ -1,181 +0,0 @@
-# Segmentation Functions
-
-The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
-
-_**Note:** The Segmentation Snapshot functions are only available for the United States. Our first release (May 18, 2016) is derived from Census 2010 variables. Our next release will be based on Census 2014 data. For the latest information, see the [Open Segments](https://github.com/CartoDB/open-segments) project repository._
-
-## OBS_GetSegmentSnapshot( Point Geometry )
-
-### Arguments
-
-Name | Description | Example Values
---- |  --- | ---
-point geometry | A point geometry. You can use the helper function, `CDB_LatLng` to quickly generate one from latitude and longitude | `CDB_LatLng(40.760410,-73.964242)`
-
-### Returns
-
-The segmentation function returns two segment levels for the point you requests, the x10\_segment and x55\_segment. These segmentation levels contain different classifications of population within with each segment. The function also returns the quantile of a number of census variables. For example, if total_poulation is at 90% quantile level then this tract has a higher total population than 90% of the other tracts. 
-
-Name | Type | Description
----- | ---- | -----------
-x10\_segment | text | The demographic segment location at the 10 segment level, containing populations at high-levels, broken down into 10 broad categories
-x55\_segment | text | The demographic segment location at the 55 segment level, containing more granular sub-levels to categorize the population
-
-An example response appears as follows:
-
-```json 
-obs_getsegmentsnapshot: {
-  "x10_segment": "Wealthy, urban without Kids",
-  "x55_segment": "Wealthy city commuters",
-  "us.census.acs.B01001001_quantile": "0.0180540540540541",
-  "us.census.acs.B01001002_quantile": "0.0279864864864865",
-  "us.census.acs.B01001026_quantile": "0.016527027027027",
-  "us.census.acs.B01002001_quantile": "0.507297297297297",
-  "us.census.acs.B03002003_quantile": "0.133162162162162",
-  "us.census.acs.B03002004_quantile": "0.283743243243243",
-  "us.census.acs.B03002006_quantile": "0.683945945945946",
-  "us.census.acs.B03002012_quantile": "0.494594594594595",
-  "us.census.acs.B05001006_quantile": "0.670972972972973",
-  "us.census.acs.B08006001_quantile": "0.0607567567567568",
-  "us.census.acs.B08006002_quantile": "0.0684324324324324",
-  "us.census.acs.B08006008_quantile": "0.565135135135135",
-  "us.census.acs.B08006009_quantile": "0.638081081081081",
-  "us.census.acs.B08006011_quantile": "0",
-  "us.census.acs.B08006015_quantile": "0.900932432432432",
-  "us.census.acs.B08006017_quantile": "0.186648648648649",
-  "us.census.acs.B09001001_quantile": "0.0193513513513514",
-  "us.census.acs.B11001001_quantile": "0.0617972972972973",
-  "us.census.acs.B14001001_quantile": "0.0179594594594595",
-  "us.census.acs.B14001002_quantile": "0.0140405405405405",
-  "us.census.acs.B14001005_quantile": "0",
-  "us.census.acs.B14001006_quantile": "0",
-  "us.census.acs.B14001007_quantile": "0",
-  "us.census.acs.B14001008_quantile": "0.0609054054054054",
-  "us.census.acs.B15003001_quantile": "0.0314594594594595",
-  "us.census.acs.B15003017_quantile": "0.0403378378378378",
-  "us.census.acs.B15003022_quantile": "0.285972972972973",
-  "us.census.acs.B15003023_quantile": "0.214567567567568",
-  "us.census.acs.B16001001_quantile": "0.0181621621621622",
-  "us.census.acs.B16001002_quantile": "0.0463108108108108",
-  "us.census.acs.B16001003_quantile": "0.540540540540541",
-  "us.census.acs.B17001001_quantile": "0.0237567567567568",
-  "us.census.acs.B17001002_quantile": "0.155972972972973",
-  "us.census.acs.B19013001_quantile": "0.380662162162162",
-  "us.census.acs.B19083001_quantile": "0.986891891891892",
-  "us.census.acs.B19301001_quantile": "0.989594594594595",
-  "us.census.acs.B25001001_quantile": "0.998418918918919",
-  "us.census.acs.B25002003_quantile": "0.999824324324324",
-  "us.census.acs.B25004002_quantile": "0.999986486486486",
-  "us.census.acs.B25004004_quantile": "0.999662162162162",
-  "us.census.acs.B25058001_quantile": "0.679054054054054",
-  "us.census.acs.B25071001_quantile": "0.569716216216216",
-  "us.census.acs.B25075001_quantile": "0.0415",
-  "us.census.acs.B25075025_quantile": "0.891702702702703"
-}
-```
-
-The possible segments are:
-
-<table>
-  <tr><th> X10 segment</th> <th> X55 Segment </th></tr>
-  
-  <tr><td> Hispanic and kids</td><td></td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #99C945'></div> Middle Class, Educated, Suburban, Mixed Race </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #a3ce57'></div> Low Income on Urban Periphery</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #add468'></div> Suburban, Young and Low-income  </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #b7d978'></div> low-income, urban, young, unmarried  </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #c1df88'></div> Low education, mainly suburban </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #cbe598'></div> Young, working class and rural  </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #d5eba8'></div> Low-Income with gentrification  </td></tr>
-    
-  <tr><td>Low Income and Diverse</td><td></td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #52BCA3'></div> High school education Long Commuters, Black, White Hispanic mix</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #66c5ae'></div> Rural, Bachelors or college degree, Rent owned mix</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #79cdb7'></div> Rural,High School Education, Owns property</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #8bd5c1'></div> Young, City based renters in Sparse neighborhoods, Low poverty   </td></tr>
-    
-  <tr><td>Low income, minority mix</td><td></td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #5D69B1'></div> Predominantly black, high high school attainment, home owners </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #7b83c6'></div> White and minority mix multilingual, mixed income / education. Married </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #9095d2'></div> Hispanic Black mix multilingual, high poverty, renters, uses public transport</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #a3a7df'></div>  Predominantly black renters, rent own mix </td></tr>
-    
-  <tr><td>Middle income, single family homes</td><td></td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #E58606'></div> Lower Middle Income with higher rent burden </td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #f0983b'></div> Black and mixed community with rent burden</td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #f4a24e'></div> Lower Middle Income with affordable housing</td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #f8ab5f'></div> Relatively affordable, satisfied lower middle class</td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #fcb470'></div> Satisfied Lower Middle Income Higher Rent Costs</td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #ffbe81'></div> Suburban/Rural Satisfied, decently educated lower middle class</td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #ffc792'></div> Struggling lower middle class with rent burden</td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #ffd0a3'></div> Older white home owners, less comfortable financially </td></tr>
-    <tr><td></td> <td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #ffdab4'></div> Older home owners, more financially comfortable, some diversity</td></tr>
-    
-  <tr><td>Native American</td><td></td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #2F8AC4'></div>Younger, poorer,single parent family Native Americans</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background: #77b8ee'></div>Older, middle income Native Americans once married and Educated </td></tr>
-    
-  <tr><td>Old Wealthy, White</td><td></td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#24796C'></div> Older, mixed race professionals</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#388d7e'></div> Works from home, Highly Educated, Super Wealthy </td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#4ca191'></div> Retired Grandparents</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#60b5a5'></div> Wealthy and Rural Living</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#73c9b8'></div> Wealthy, Retired Mountains/Coasts</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#87decc'></div> Wealthy Diverse Suburbanites On the Coasts</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#9bf3e1'></div> Retirement Communitties</td></tr>
-    
-  <tr><td>Low Income African American</td><td></td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#c23a7e'></div>Urban - Inner city</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#d86298'></div>Rural families</td></tr>
-  <tr><td>Residential institutions, young people</td><td></td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#764e9f'></div>College towns</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#8a64b1'></div>College town with poverty</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#9e7ac3'></div>University campus wider area</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#b491d5'></div>City Outskirt University Campuses</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#c9a8e8'></div>City Center University Campuses</td></tr>
-    
-  <tr><td>Wealthy Nuclear Families</td><td></td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ed645a'></div>Lower educational attainment, Homeowner, Low rent</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ee7655'></div>Younger, Long Commuter in dense neighborhood</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#f38060'></div>Long commuters White black mix</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#f98a6b'></div>Low rent in built up neighborhoods</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#fe9576'></div>Renters within cities, mixed income areas, White/Hispanic mix, Unmarried</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ff9f82'></div>Older Home owners with high income</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ffa98d'></div>Older home owners and very high income</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ffb399'></div>White Asian Mix Big City Burbs Dwellers</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ffbda5'></div>Bachelors degree Mid income With Mortgages</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ffc8b1'></div>Asian Hispanic Mix, Mid income</td></tr>
-    <tr><td></td><td><div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ffd2bd'></div>Bachelors degree Higher income Home Owners</td></tr>
-    
-  <tr><td>Wealthy, urban, and kid-free</td><td></td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#CC61B0'></div>Wealthy city commuters </td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#d975bd'></div>New Developments</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#e488c9'></div>Very wealthy, multiple million dollar homes</td></tr>
-    <tr><td></td> <td> <div  style='float:left;margin-right:10px;width:20px; height:20px; border-radius:20px;background:#ee9ad4'></div>High rise, dense urbanites</td></tr>
-</table>
-
-
-### Examples
-
-```bash
-https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
-OBS_GetSegmentSnapshot({{point geometry}})
-```
-
-##### Get the Geographic Snapshot of a Segmentation
-
-__Get the Segmentation Snapshot around the MGM Grand__
-
-
-```bash
-https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
-OBS_GetSegmentSnapshot(CDB_LatLng(36.10222, -115.169516))
-```
-
-__Get the Segmentation Snapshot at CARTO's NYC HQ__
-
-
-```bash
-https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
-OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669))
-```

docs/developer-center/guides/01-overview.md

@@ -1,34 +1,34 @@
-# Overview
+## Overview
 
 By using CARTO libraries and the SQL API, you can apply location data services to your maps with unique data services functions. These functions are integrated with a number of internal and external services, enabling you to programatically customize subsets of data for your visualizations. These features are useful for geospatial analysis and the results can be saved, and stored, for additional location data service operations.
 
-**Note:** Based on your account plan, some of these data services are subject to different [quota limitations](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-information).
+**Note:** Based on your account plan, some of these data services are subject to different [quota limitations]({{site.dataservicesapi_docs}}/support/quota-information/).
 
-_In order to supply the best location data services from within our CARTO Engine, the Data Services API collaborates with [Mapbox](https://www.mapbox.com/) and several other geospatial service providers. [Contact us](mailto:sales@carto.com) if you have any specific questions or requirements about the location data service provider being used with your account._
+_In order to supply the best location data services from within our CARTO Engine, the Data Services API collaborates with [TomTom](https://www.tomtom.com/) and several other geospatial service providers. [Contact us](mailto:sales@carto.com) if you have any specific questions or requirements about the location data service provider being used with your account._
 
-## Data Services Integration
+### Data Services Integration
 
 By using the SQL API to query the Data Services API functions, you can manage specific operations and the corresponding geometries (a `polygon` or a `point`), according to the input information.
 
-The Data Services API also exposes its services directly through CARTO Builder. For example, you can geocode data (from single rows, complete datasets, or simple inputs) and perform trade areas analysis (computing isodistances or isochrones) programatically, through authenticated SQL requests, or by using the ANALYSIS options. 
+The Data Services API also exposes its services directly through CARTO Builder. For example, you can geocode data (from single rows, complete datasets, or simple inputs) and perform trade areas analysis (computing isodistances or isochrones) programatically, through authenticated SQL requests, or by using the ANALYSIS options.
 
 The geometries provided by this API are projected in the projection [WGS 84 SRID 4326](http://spatialreference.org/ref/epsg/wgs-84/).
 
-**Note:** The Data Services API [geocoding functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/#geocoding-functions) return different types of geometries (points or polygons) as result of different geocoding processes. The CARTO Engine does not support multi-geometry layers or datasets, therefore you must confirm that you are using consistent geometry types inside a table, to avoid future conflicts in your map visualization.
+**Note:** The Data Services API [geocoding functions]({{site.dataservicesapi_docs}}/reference/#geocoding-functions) return different types of geometries (points or polygons) as result of different geocoding processes. The CARTO Engine does not support multi-geometry layers or datasets, therefore you must verify that you are using consistent geometry types inside a table, to avoid future conflicts in your map visualization.
 
-### Best Practices
+#### Best Practices
 
 _Be mindful of the following usage notes when using the Data Services functions with the SQL API:_
 
-It is discouraged to use the SELECT operation with the Data Services API functions in your map layers, as these type of queries consume quota when rendering tiles for your live map views. It may also result in sync performance issues, due to executing multiple requests to the API each time your map is viewed. See details about [Quota Consumption](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-consumption).
+It is discouraged to use the SELECT operation with the Data Services API functions in your map layers, as these type of queries consume quota when rendering tiles for your live map views. It may also result in sync performance issues, due to executing multiple requests to the API each time your map is viewed. See details about [Quota Consumption]({{site.dataservicesapi_docs}}/support/quota-information/#quota-consumption).
 
 The Data Services API is **recommended** to be used with INSERT or UPDATE operations, for applying location data to your tables. While SELECT (retrieve) is standard for SQL API requests, be mindful of quota consumption and use INSERT (to insert a new record) or UPDATE (to update an existing record), for best practices.
 
-## Authentication
+### Authentication
 
-All requests performed to the CARTO Data Services API must be authenticated with the user API Key. For more information about where to find your API Key, and how to authenticate your SQL API requests, view the [SQL API authentication](/carto-engine/sql-api/authentication/) documentation.
+All requests performed to the CARTO Data Services API must be authenticated with the user API Key. For more information about where to find your API Key, and how to authenticate your SQL API requests, view the [Auth API]({{site.authapi_docs}}/) documentation.
 
-## Errors
+### Errors
 
 Errors are described in the response of the request. An example is as follows:
 
@@ -40,10 +40,10 @@ Errors are described in the response of the request. An example is as follows:
 }
 ```
 
-Since the Data Services API is used on top of the CARTO SQL API, you can refer to the [Making calls to the SQL API](https://carto.com/docs/carto-engine/sql-api/making-calls/) documentation for help debugging your SQL errors.
+Since the Data Services API is used on top of the CARTO SQL API, you can refer to the [Making calls to the SQL API]({{site.sqlapi_docs}}/guides/making-calls/) documentation for help debugging your SQL errors.
 
 If the requested information is not in the CARTO geocoding database, or if CARTO is unable to recognize your input and match it with a result, the geocoding function returns `null` as a result.
 
-## Limits
+### Limits
 
 Usage of the Data Services API is subject to the CARTO SQL API limits, stated in our [Terms of Service](https://carto.com/terms/#excessive).

docs/developer-center/reference/01-introduction.md

@@ -0,0 +1,5 @@
+## Introduction
+
+The CARTO Data Services API offers a set of location based services that can be used to programatically customize subsets of data for your visualizations.
+
+The contents described in this document are subject to CARTO's [Terms of Service](https://carto.com/legal/)

docs/developer-center/reference/02-authentication.md

@@ -0,0 +1,9 @@
+## Authentication
+
+Data Services API, like any other [CARTO platform's component]({{site.fundamental_docs}}/components/), requires using an API Key. From your CARTO dashboard, click _[Your API keys](https://carto.com/login)_ from the avatar drop-down menu to view your uniquely generated API Key for managing data with CARTO Engine.
+
+![Your API Keys](../img/avatar.gif)
+
+Learn more about the [basics of authorization]({{site.fundamental_docs}}/authorization/), or dig into the details of [Auth API]({{site.authapi_docs}}/), if you want to know more about this part of CARTO platform.
+
+The examples in this documentation may include a placeholder for the API Key. Ensure that you modify any placeholder parameters with your own credentials.

docs/developer-center/reference/03-versioning.md

@@ -0,0 +1,3 @@
+## Versioning
+
+Data Services API uses [Semantic Versioning](http://semver.org/). View our Github repository to find tags for each [release](https://github.com/CartoDB/data-services-api/releases).

docs/developer-center/reference/04-error-handling.md

@@ -0,0 +1,5 @@
+## Error handling
+
+Most of the errors fired by the API are handled by the API itself. It triggers a `CartoError` every time an error happens.
+
+A cartoError is an object containing a single `message` field with a string explaining the error.

docs/developer-center/reference/05-geocoding-functions.md

@@ -1,8 +1,8 @@
-# Geocoding Functions
+## Geocoding Functions
 
 The [geocoder](https://carto.com/data/geocoder-api/) functions allow you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CARTO SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes, IP addresses and street addresses. CARTO provides functions for several different categories of geocoding through the Data Services API.
 
-_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
+**Warning:** This service is subject to quota limitations and extra fees may apply. View the [Quota Information]({{site.dataservicesapi_docs}}/support/quota-information/) section for details and recommendations about to quota consumption.
 
 The following example displays how to geocode a single country:
 
@@ -25,169 +25,168 @@ https://{username}.carto.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = ST_C
 
 The following geocoding functions are available, grouped by categories.
 
-## Country Geocoder
+### Country Geocoder
 
 This function geocodes your data into country border geometries. It recognizes the names of the different countries either by different synonyms (such as their English name or their endonym), or by ISO (ISO2 or ISO3) codes.
 
-### cdb_geocode_admin0_polygon(_country_name text_)
+#### cdb_geocode_admin0_polygon(_country_name text_)
 
 Geocodes the text name of a country into a country_name geometry, displayed as polygon data.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `country_name` | `text` | Name of the country
 
-#### Returns
+##### Returns
 
 Geometry (polygon, EPSG 4326) or null
 
-#### Example
+### Example
 
-##### Update the geometry of a table to geocode it
+#### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_admin0_polygon({country_column})
 ```
 
-##### Insert a geocoded row into a table
-
+#### Insert a geocoded row into a table
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_admin0_polygon('France')
 ```
 
-## Level-1 Administrative Regions Geocoder
+### Level-1 Administrative Regions Geocoder
 
 This function geocodes your data into polygon geometries for [Level 1](https://en.wikipedia.org/wiki/Table_of_administrative_divisions_by_country), or [NUTS-1](https://en.wikipedia.org/wiki/NUTS_1_statistical_regions_of_England), administrative divisions (or units) of countries. For example, a "state" in the United States, "départements" in France, or an autonomous community in Spain.
 
-### cdb_geocode_admin1_polygon(_admin1_name text_)
+#### cdb_geocode_admin1_polygon(_admin1_name text_)
 
 Geocodes the name of the province/state into a Level-1 administrative region, displayed as a polygon geometry.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `admin1_name` | `text` | Name of the province/state
 
-#### Returns
+##### Returns
 
 Geometry (polygon, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column})
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_admin1_polygon('Alicante')
 ```
 
 
-### cdb_geocode_admin1_polygon(_admin1_name text, country_name text_)
+#### cdb_geocode_admin1_polygon(_admin1_name text, country_name text_)
 
 Geocodes the name of the province/state for a specified country into a Level-1 administrative region, displayed as a polygon geometry.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `admin1_name` | `text` | Name of the province/state
 `country_name` | `text` | Name of the country in which the province/state is located
 
-#### Returns
+##### Returns
 
 Geometry (polygon, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_admin1_polygon({province_column}, {country_column})
 ```
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_admin1_polygon('Alicante', 'Spain')
 ```
 
 
-## City Geocoder
+### City Geocoder
 
 This function geocodes your data into point geometries for names of cities. It is recommended to use geocoding functions that require more defined parameters — this returns more accurate results when several cities have the same name. _If there are duplicate results for a city name, the city name with the highest population will be returned._
 
-### cdb_geocode_namedplace_point(_city_name text_)
+#### cdb_geocode_namedplace_point(_city_name text_)
 
 Geocodes the text name of a city into a named place geometry, displayed as point data.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `city_name` | `text` | Name of the city
 
-#### Returns
+##### Returns
 
 Geometry (point, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Select
+###### Select
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column})
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_namedplace_point('Barcelona')
 ```
 
 
-### cdb_geocode_namedplace_point(_city_name text, country_name text_)
+#### cdb_geocode_namedplace_point(_city_name text, country_name text_)
 
 Geocodes the text name of a city for a specified country into a named place point geometry.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `city_name` | `text` | Name of the city
 `country_name` | `text` | Name of the country in which the city is located
 
-#### Returns
+##### Returns
 
 Geometry (point, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, 'Spain')
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_namedplace_point('Barcelona', 'Spain')
 ```
 
 
-### cdb_geocode_namedplace_point(_city_name text, admin1_name text, country_name text_)
+#### cdb_geocode_namedplace_point(_city_name text, admin1_name text, country_name text_)
 
-Geocodes your data into a named place point geometry, containing the text name of a city, for a specified province/state and country. This is recommended for the most accurate geocoding of city data. 
-#### Arguments
+Geocodes your data into a named place point geometry, containing the text name of a city, for a specified province/state and country. This is recommended for the most accurate geocoding of city data.
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
@@ -195,132 +194,132 @@ Name | Type | Description
 `admin1_name` | `text` | Name of the province/state in which the city is located
 `country_name` | `text` | Name of the country in which the city is located
 
-#### Returns
+##### Returns
 
 Geometry (point, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_namedplace_point({city_column}, {province_column}, 'USA')
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_namedplace_point('New York', 'New York', 'USA')
 ```
 
-## Postal Code Geocoder
+### Postal Code Geocoder
 
 These functions geocode your data into point, or polygon, geometries for postal codes. The postal code geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response.
 
 **Note:** For the USA, US Census Zip Code Tabulation Areas (ZCTA) are used to reference geocodes for USPS postal codes service areas. This is not a CARTO restriction, this is a US Government licensing protection of their zip code data source; which is not publicly available. Additionally, zip codes are considered service areas and are not actually geometric areas. As a solution, the US Census provides ZCTA data, which tabulates GIS postal codes for USPS locations by aggregating census blocks. For details about how ZCTAs are created, see [ZIP Code™ Tabulation Areas (ZCTAs™)](https://www.census.gov/geo/reference/zctas.html). If you are geocoding data and your zip codes fail, ensure you are using ZCTAs for the postal code.
 
-### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
+#### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
 
 Geocodes the postal code for a specified country into a **polygon** geometry.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `postal_code` | `text` | Postal code
 `country_name` | `text` | Name of the country in which the postal code is located
 
-#### Returns
+##### Returns
 
 Geometry (polygon, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_polygon({postal_code_column}, 'USA')
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_postalcode_polygon('11211', 'USA')
 ```
 
-### cdb_geocode_postalcode_point(_code text, country_name text_)
+#### cdb_geocode_postalcode_point(_code text, country_name text_)
 
 Geocodes the postal code for a specified country into a **point** geometry.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `postal_code` | `text` | Postal code
 `country_name` | `text` | Name of the country in which the postal code is located
 
-#### Returns
+##### Returns
 
 Geometry (point, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_postalcode_point({postal_code_column}, 'USA')
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_postalcode_point('11211', 'USA')
 ```
 
 
-## IP Addresses Geocoder
+### IP Addresses Geocoder
 
 This function geocodes your data into point geometries for IP addresses. This is useful if you are analyzing location based data, based on a set of user's IP addresses.
 
-### cdb_geocode_ipaddress_point(_ip_address text_)
+#### cdb_geocode_ipaddress_point(_ip_address text_)
 
 Geocodes a postal code from a specified country into an IP address, displayed as a point geometry.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | ---
 `ip_address` | `text` | IPv4 or IPv6 address
 
-#### Returns
+##### Returns
 
 Geometry (point, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_ipaddress_point('102.23.34.1')
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_ipaddress_point('102.23.34.1')
 ```
 
-## Street-Level Geocoder
+### Street-Level Geocoder
 
-This function geocodes your data into a point geometry for a street address. CARTO uses several different service providers for street-level geocoding, depending on your platform. If you access CARTO on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [Mapbox geocoding services](https://www.mapbox.com/). [Contact us](mailto:sales@carto.com) if you have any specific questions or requirements about the location data service provider being used with your account._.
+These functions geocode your data into a point geometry for a street address. CARTO platform uses [TomTom geocoding services](https://www.tomtom.com/) by default as the service provider for street-level geocoding. [Contact us](mailto:sales@carto.com) if you have any specific questions or requirements about the location data service provider being used with your account.
 
-**This service is subject to quota limitations, and extra fees may apply**. View the [Quota information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) for details and recommendations about quota consumption.
+**This service is subject to quota limitations, and extra fees may apply**. View the [Quota information]({{site.dataservicesapi_docs}}/support/quota-information/) for details and recommendations about quota consumption.
 
-### cdb_geocode_street_point(_search_text text, [city text], [state text], [country text]_)
+#### cdb_geocode_street_point(_search_text text, [city text], [state text], [country text]_)
 
 Geocodes a complete address into a single street geometry, displayed as point data.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description
 --- | --- | --- | ---
@@ -329,20 +328,62 @@ Name | Type | Description
 `state` | `text` | (Optional) Name of the state.
 `country` | `text` | (Optional) Name of the country.
 
-#### Returns
+##### Returns
 
 Geometry (point, EPSG 4326) or null
 
-#### Example
+##### Example
 
-##### Update the geometry of a table to geocode it
+###### Update the geometry of a table to geocode it
 
 ```bash
 UPDATE {tablename} SET the_geom = cdb_geocode_street_point({street_name_column})
 ```
 
-##### Insert a geocoded row into a table
+###### Insert a geocoded row into a table
 
 ```bash
 INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_street_point('651 Lombard Street', 'San Francisco', 'California', 'United States')
 ```
+
+#### cdb_bulk_geocode_street_point (_query text, street_column text, [city_column text], [state_column text], [country_column text], [batch_size integer]_)
+
+Geocodes complete street addresses into point data. Similar to `cdb_geocode_street_point`, but using batch services and therefore allowing for several addresses to be geocoded in a single API call.
+
+##### Arguments
+
+Name | Type | Description
+--- | --- | --- | ---
+`query` | `text` | SQL query that returns the addresses to be geocoded. It must include a `cartodb_id` column and another column to get the free-form addresses from. Optionally, it may include other columns to fine-tune the geocoding, such as a city column, a state column and a country column.
+`street_column` | `text` | Name of the free-form address column, must be present in the SQL query.
+`city_column` | `text` | (Optional) Name of the city column, if present in the SQL query.
+`state_column` | `text` | (Optional) Name of the state column, if present in the SQL query.
+`country_column` | `text` | (Optional) Name of the country column, if present in the SQL query.
+`batch_size` | `integer` | (Optional) Geocoding queries are sent in batches. Batch size can be configured, from 1 geocoding query per batch to a maximum value, limited by user quota or other limits. If not specified, it defaults to the maximum size available to the user, which is typically the best option, performance-wise.
+
+##### Returns
+
+Geocoding results are returned in an array. Each array element contains:
+
+Name | Type | Description
+--- | --- | --- | ---
+`cartodb_id` | `integer` | `cartodb_id` from the original query.
+`the_geom` | `Geometry (point, EPSG 4326)` | Point that corresponds to the most accurate match found for this particular address, or `null` if no match was found.
+`metadata` | `JSON` | Information about the geocoding result, empty if no match was found.
+
+The `metadata` JSON type includes the following attributes when geocoding was successful:
+
+Name | Type | Description
+--- | --- | --- | ---
+`precision` | `text` | One of `precise` or `interpolated`.
+`relevance` | `number` | Relevance factor, from 0 to 1, higher being more relevant.
+`match_type` | `text` | Array with one of `point_of_interest`, `country`, `state`, `county`, `locality`, `district`, `street`, `intersection`, `street_number`, `postal_code`. Empty array if match type is unknown.
+
+
+##### Example
+
+###### Update the geometries of an entire table by geocoding all the rows based on a street address
+
+```bash
+WITH geocoding_results AS (SELECT cartodb_id, the_geom FROM cdb_bulk_geocode_street_point('SELECT cartodb_id, {address_column} from {tablename}', '{address_column}')) UPDATE {tablename} tn SET the_geom = geocoding_results.the_geom FROM geocoding_results WHERE tn.cartodb_id = geocoding_results.cartodb_id
+```

docs/developer-center/reference/06-isoline-functions.md

@@ -1,8 +1,8 @@
-# Isoline Functions
+## Isoline Functions
 
 [Isolines](https://carto.com/data/isolines/) are contoured lines that display equally calculated levels over a given surface area. This enables you to view polygon dimensions by forward or reverse measurements. Isoline functions are calculated as the intersection of areas from the origin point, measured by distance (isodistance) or time (isochrone). For example, the distance of a road from a sidewalk. Isoline services through CARTO are available by requesting a single function in the Data Services API.
 
-_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
+_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information]({{site.dataservicesapi_docs}}/support/quota-information/) section for details and recommendations about to quota consumption._
 
 You can use the isoline functions to retrieve, for example, isochrone lines from a certain location, specifying the mode and the ranges that will define each of the isolines. The following query calculates isolines for areas that are 5, 10 and 15 minutes (300, 600 and 900 seconds, respectively) away from the location by following a path defined by car routing and inserts them into a table.
 
@@ -12,10 +12,12 @@ https://{username}.carto.com/api/v2/sql?q=INSERT INTO {table} (the_geom) SELECT
 
 The following functions provide an isoline generator service, based on time or distance. This service uses the isolines service defined for your account. The default service limits the usage of displayed polygons represented on top of [Mapbox](https://www.mapbox.com/) maps.
 
-## cdb_isodistance(_source geometry, mode text, range integer[], [options text[]]_)
+### cdb_isodistance(_source geometry, mode text, range integer[], [options text[]]_)
 
 Displays a contoured line on a map, connecting geometries to a defined area, measured by an equal range of distance (in meters).
 
+Note that not all the providers, for example TomTom, provide us a way to define the isoline limit in distance so we need to make some estimations. Due that estimations the produced isolines could not be 100% precise.
+
 #### Arguments
 
 Name | Type | Description | Accepted values
@@ -26,7 +28,7 @@ Name | Type | Description | Accepted values
 `options` | `text[]` | (Optional) Multiple options to add more capabilities to the analysis. See [Optional isolines parameters](#optional-isoline-parameters) for details.
 
 
-#### Returns
+##### Returns
 
 Name | Type | Description
 --- | --- | ---
@@ -34,9 +36,9 @@ Name | Type | Description
 `data_range` | `integer` | The range that belongs to the generated isoline.
 `the_geom` | `geometry(MultiPolygon)` | MultiPolygon geometry of the generated isoline in the 4326 projection.
 
-#### Examples
+##### Examples
 
-##### Calculate and insert isodistance polygons from a point into another table
+###### Calculate and insert isodistance polygons from a point into another table
 
 ```bash
 INSERT INTO {table} (the_geom) SELECT  the_geom FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300, 600, 900]::integer[])
@@ -48,18 +50,18 @@ or equivalently:
 INSERT INTO {table} (the_geom) SELECT (cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'walk', ARRAY[300, 600, 900]::integer[])).the_geom
 ```
 
-##### Calculate and insert the generated isolines from `points_table` table to another table
+###### Calculate and insert the generated isolines from `points_table` table to another table
 
 ```bash
 INSERT INTO {table} (the_geom) SELECT (cdb_isodistance(the_geom, 'walk', string_to_array(distance, ',')::integer[])).the_geom FROM {points_table}
 ```
 
 
-## cdb_isochrone(_source geometry, mode text, range integer[], [options text[]]_)
+### cdb_isochrone(_source geometry, mode text, range integer[], [options text[]]_)
 
 Displays a contoured line on a map, connecting geometries to a defined area, measured by an equal range of time (in seconds).
 
-#### Arguments
+##### Arguments
 
 This function uses the same parameters and information as the `cdb_isodistance` function, with the exception that the range is measured in seconds instead of meters.
 
@@ -70,9 +72,9 @@ Name | Type | Description | Accepted values
 `range` | `integer[]` | Range of the isoline, in seconds. |
 `options` | `text[]` | (Optional) Multiple options to add more capabilities to the analysis. See [Optional isolines parameters](#optional-isoline-parameters) for details.
 
-#### Examples
+##### Examples
 
-##### Calculate and insert isochrone polygons from a point into another table
+###### Calculate and insert isochrone polygons from a point into another table
 
 ```bash
 INSERT INTO {table} (the_geom) SELECT  the_geom FROM cdb_isochrone('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 900, 12000]::integer[], ARRAY['mode_traffic=enabled','quality=3']::text[])
@@ -84,13 +86,13 @@ or equivalently:
 INSERT INTO {table} (the_geom) SELECT (cdb_isochrone('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 900, 12000]::integer[], ARRAY['mode_traffic=enabled','quality=3']::text[])).the_geom
 ```
 
-##### Calculate and insert the generated isolines from `points_table` table into another table
+###### Calculate and insert the generated isolines from `points_table` table into another table
 
 ```bash
 INSERT INTO {table} (the_geom) SELECT (cdb_isochrone(the_geom, 'walk', string_to_array(time_distance, ',')::integer[])).the_geom FROM {points_table}
 ```
 
-### Optional isoline parameters
+#### Optional isoline parameters
 
 The optional value parameters must be passed using the format: `option=value`.
 

docs/developer-center/reference/07-routing-functions.md

@@ -1,23 +1,23 @@
-# Routing Functions
+## Routing Functions
 
 Routing is the navigation from a defined start location to a defined end location. The calculated results are displayed as turn-by-turn directions on your map, based on the transportation mode that you specified. Routing services through CARTO are available by using the available functions in the Data Services API.
 
-## cdb_route_point_to_point(_origin geometry(Point), destination geometry(Point), mode text, [options text[], units text]_)
+### cdb_route_point_to_point(_origin geometry(Point), destination geometry(Point), mode text, [options text[], units text]_)
 
 Returns a route from origin to destination.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description | Accepted values
 --- | --- | --- | ---
 `origin` | `geometry(Point)` | Origin point, in 4326 projection, which defines the start location. |
 `destination` | `geometry(Point)` | Destination point, in 4326 projection, which defines the end location. |
-`mode` | `text` | Type of transport used to calculate the routes. | `car`, `walk`, `bicycle` or `public_transport`
+`mode` | `text` | Type of transport used to calculate the routes. | `car`, `walk` or `bicycle`
 `options` | `text[]` | (Optional) Multiple options to add more capabilities to the analysis. See [Optional routing parameters](#optional-routing-parameters) for details.
 `units` | `text` | (Optional) Unit used to represent the length of the route. | `kilometers`, `miles`. By default is `kilometers`. This option is not supported by Mapbox provider
 
 
-#### Returns
+##### Returns
 
 Name | Type | Description
 --- | --- | ---
@@ -25,34 +25,34 @@ Name | Type | Description
 `length` | `real` | Length in the defined unit in the `units` field. `meters` by default .
 `the_geom` | `geometry(LineString)` | LineString geometry of the calculated route in the 4326 projection.
 
-#### Examples
+##### Examples
 
-##### Insert the values from the calculated route in your table
+###### Insert the values from the calculated route in your table
 
 ```bash
 INSERT INTO <TABLE> (duration, length, the_geom) SELECT duration, length, shape FROM cdb_route_point_to_point('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car')
 ```
-##### Update the geometry field with the calculated route shape
+###### Update the geometry field with the calculated route shape
 
 ```bash
 UPDATE <TABLE> SET the_geom = (SELECT shape FROM cdb_route_point_to_point('POINT(-3.70237112 40.41706163)'::geometry,'POINT(-3.69909883 40.41236875)'::geometry, 'car', ARRAY['mode_type=shortest']::text[]))
 ```
 
-## cdb_route_with_waypoints(_waypoints geometry(Point)[], mode text, [options text[], units text]_)
+### cdb_route_with_waypoints(_waypoints geometry(Point)[], mode text, [options text[], units text]_)
 
 Returns a route that goes from origin to destination and whose path travels through the defined locations.
 
-#### Arguments
+##### Arguments
 
 Name | Type | Description | Accepted values
 --- | --- | --- | ---
 `waypoints` | `geometry(Point)[]` | Array of ordered points, in 4326 projection, which defines the origin point, one or more locations for the route path to travel through, and the destination. The first element of the array defines the origin and the last element the destination of the route. |
-`mode` | `text` | Type of transport used to calculate the routes. | `car`, `walk`, `bicycle` or `public_transport`
+`mode` | `text` | Type of transport used to calculate the routes. | `car`, `walk` or `bicycle`
 `options` | `text[]` | (Optional) Multiple options to add more capabilities to the analysis. See [Optional routing parameters](#optional-routing-parameters) for details.
 `units` | `text` | (Optional) Unit used to represent the length of the route. | `kilometers`, `miles`. By default is `kilometers`. This option is not supported by Mapbox provider
 
 
-#### Returns
+##### Returns
 
 Name | Type | Description
 --- | --- | ---
@@ -62,23 +62,23 @@ Name | Type | Description
 
 *Note*: A request to the function _cdb\_route\_with\_waypoints(waypoints geometry(Point)[], mode text, [options text[], units text])_ with only two points in the geometry array are automatically defined as origin and destination. It is equivalent to performing the following request with these two locations as parameters: _cdb\_route\_point\_to\_point(origin geometry(Point), destination geometry(Point), mode text, [options text[], units text])_.
 
-#### Examples
+##### Examples
 
-##### Insert the values from the calculated route in your table
+###### Insert the values from the calculated route in your table
 
 ```bash
 INSERT INTO <TABLE> (duration, length, the_geom) SELECT duration, length, shape FROM cdb_route_with_waypoints(Array['POINT(-3.7109 40.4234)'::GEOMETRY, 'POINT(-3.7059 40.4203)'::geometry, 'POINT(-3.7046 40.4180)'::geometry]::geometry[], 'walk')
 ```
-##### Update the geometry field with the calculated route shape
+###### Update the geometry field with the calculated route shape
 
 ```bash
 UPDATE <TABLE> SET the_geom = (SELECT shape FROM cdb_route_with_waypoints(Array['POINT(-3.7109 40.4234)'::GEOMETRY, 'POINT(-3.7059 40.4203)'::geometry, 'POINT(-3.7046 40.4180)'::geometry]::geometry[], 'car', ARRAY['mode_type=shortest']::text[]))
 ```
 
-### Optional routing parameters
+#### Optional routing parameters
 
 The optional value parameters must be passed using the format: `option=value`. Not all are available for all the routing providers
 
 Name | Type | Description | Accepted values
 --- | --- | --- | ---
-`mode_type` | `text` | Type of route calculation | `shortest` (this option only applies to the car transport mode)
+`mode_type` | `text` | Type of route calculation | `shortest` or `fastest` (this option only applies to the car transport mode)

docs/developer-center/support/01-support-options.md

@@ -0,0 +1,36 @@
+## Support Options
+
+Feeling stuck? There are many ways to find help.
+
+* Ask a question on [GIS StackExchange](https://gis.stackexchange.com/questions/tagged/carto) using the `CARTO` tag.
+* [Report an issue](https://github.com/CartoDB/cartodb/issues) in Github.
+* Engine Plan customers have additional access to enterprise-level support through CARTO's support representatives.
+
+If you just want to describe an issue or share an idea, just <a class="typeform-share" href="https://cartohq.typeform.com/to/mH6RRl" data-mode="popup" target="_blank"> send your feedback</a>
+
+### Issues on Github
+
+If you think you may have found a bug, or if you have a feature request that you would like to share with the Data Services API team, please [open an issue](https://github.com/CartoDB/cartodb/issues/new). 
+
+Before opening an issue, review the [contributing guidelines](https://github.com/CartoDB/cartodb/blob/master/CONTRIBUTING.md).
+
+
+### Community support on GIS Stack Exchange
+
+GIS Stack Exchange is the most popular community in the geospatial industry. This is a collaboratively-edited question and answer site for geospatial programmers and technicians. It is a fantastic resource for asking technical questions about developing and maintaining your application.
+
+
+When posting a new question, please consider the following:
+
+* Read the GIS Stack Exchange [help](https://gis.stackexchange.com/help) and [how to ask](https://gis.stackexchange.com/help/how-to-ask) pages for guidelines and tips about posting questions.
+* Be very clear about your question in the subject. A clear explanation helps those trying to answer your question, as well as those who may be looking for information in the future.
+* Be informative in your post. Details, code snippets, logs, screenshots, etc. help others to understand your problem.
+* Use code that demonstrates the problem. It is very hard to debug errors without sample code to reproduce the problem.
+
+### Engine Plan Customers
+
+Engine Plan customers have additional support options beyond general community support. As per your account Terms of Service, you have access to enterprise-level support through CARTO's support representatives available at [enterprise-support@carto.com](mailto:enterprise-support@carto.com)
+
+In order to speed up the resolution of your issue, provide as much information as possible (even if it is a link from community support). This allows our engineers to investigate your problem as soon as possible.
+
+If you are not yet CARTO customer, browse our [plans & pricing](https://carto.com/pricing/) and find the right plan for you.

docs/developer-center/support/02-contribute.md

@@ -0,0 +1,36 @@
+## Contribute
+
+CARTO platform is an open-source ecosystem. You can read about the [fundamentals]({{site.fundamental_docs}}/components/) of CARTO architecture and its components.
+We are more than happy to receive your contributions to the code and the documentation as well.
+
+## Filling a ticket
+
+If you want to open a new issue in our repository, please follow these instructions:
+
+1. Descriptive title.
+2. Write a good description, it always helps.
+3. Specify the steps to reproduce the problem.
+4. Try to add an example showing the problem.
+
+## Contributing code
+
+Best part of open source, collaborate in Data Services API code!. We like hearing from you, so if you have any bug fixed, or a new feature ready to be merged, those are the steps you should follow:
+
+1. Fork the repository.
+2. Create a new branch in your forked repository.
+3. Commit your changes. Add new tests if it is necessary.
+4. Open a pull request.
+5. Any of the maintainers will take a look.
+6. If everything works, it will merged and released \o/.
+
+If you want more detailed information, this [GitHub guide](https://guides.github.com/activities/contributing-to-open-source/) is a must.
+
+## Completing documentation
+
+Data Services API documentation is located in ```docs/```. That folder is the content that appears in the [Developer Center](http://carto.com/developers/data-services-api/). Just follow the instructions described in [contributing code](#contributing-code) and after accepting your pull request, we will make it appear online :).
+
+**Tip:** A convenient, easy way of proposing changes in documentation is by using the GitHub editor directly on the web. You can easily create a branch with your changes and make a PR from there.
+
+## Submitting contributions
+
+You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

docs/developer-center/support/03-quota-information.md

@@ -1,8 +1,8 @@
-# Quota Information
+## Quota Information
 
 **Based on your account plan, some of the Data Services API functions are subject to quota limitations and extra fees may apply.** View our [terms and conditions](https://carto.com/terms/), or [contact us](mailto:sales@carto.com) for details about which functions require service credits to your account.
 
-## Quota Consumption
+### Quota Consumption
 
 Quota consumption is calculated based on the number of request made for each function. Be mindful of the following usage recommendations when using the Data Services API functions:
 
@@ -14,15 +14,15 @@ Quota consumption is calculated based on the number of request made for each fun
 * It is advised to store results of these queries into your datasets, and refresh them as needed. This ensure more control of quota credits for your account
 
 
-## Quota Information Functions
+### Quota Information Functions
 
 There are several SQL functions that you can run to obtain quota information about your services. 
 
-## cdb_service_quota_info()
+### cdb_service_quota_info()
 
 Returns information about per-service quotas (available and used) for the account.
 
-#### Returns
+##### Returns
 
 This function returns a set of service quota information records, one per service.
 
@@ -36,10 +36,9 @@ Name            | Type      | Description
 
 Service Types:
 
-* `'isolines'` [Isoline/Isochrones (isochrone/isodistance lines) service](https://carto.com/docs/carto-engine/dataservices-api/isoline_functions/)
-* `'hires_geocoder'` [Street level geocoding](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions#street-level-geocoder)
-* `'routing'` [Routing functions](https://carto.com/docs/carto-engine/dataservices-api/routing_functions/)
-* `'observatory'` Data Observatory services ([demographic](https://carto.com/docs/carto-engine/dataservices-api/demographic_functions/) and [segmentation](https://carto.com/docs/carto-engine/dataservices-api/segmentation_functions/) functions)
+* `'isolines'` [Isoline/Isochrones (isochrone/isodistance lines) service]({{site.dataservicesapi_docs}}/reference/#isoline_functions/)
+* `'hires_geocoder'` [Street level geocoding]({{site.dataservicesapi_docs}}/reference/#street-level-geocoder)
+* `'routing'` [Routing functions]({{site.dataservicesapi_docs}}/reference/#routing_functions/)
 
 **Notes**
 
@@ -48,7 +47,7 @@ expenses when the regular quota is exceeded.
 
 A zero value of `monthly_quota` indicates that the service has not been activated for the user.
 
-#### Example
+##### Example
 
 ```sql
 SELECT * FROM cdb_service_quota_info();
@@ -59,17 +58,16 @@ Result:
 ```sql
     service     | monthly_quota | used_quota | soft_limit |     provider
 ----------------+---------------+------------+------------+------------------
- isolines       |           100 |          0 | f          | mapbox
- hires_geocoder |           100 |          0 | f          | mapbox
- routing        |            50 |          0 | f          | mapbox
- observatory    |             0 |          0 | f          | data observatory
+ isolines       |           100 |          0 | f          | tomtom
+ hires_geocoder |           100 |          0 | f          | tomtom
+ routing        |            50 |          0 | f          | tomtom
 (4 rows)
 
 ```
 
-In this case, notice that the user has no access to the observatory services. All quotas are *hard-limited* (no soft limits), and no quota has been used in the present period.
+All quotas are *hard-limited* (no soft limits), and no quota has been used in the present period.
 
-## cdb_enough_quota(service text ,input_size numeric)
+### cdb_enough_quota(service text ,input_size numeric)
 
 This function is useful to check if enough quota is available for completing a job.
 
@@ -79,20 +77,20 @@ This is specifically relevant if a number of service calls are to be performed i
 
 Note that some services consume more than one credit per row/call. For example, isolines (with more than one range/track) consume (N rows x M ranges) credits; indicating that the input size should be N x M.
 
-#### Arguments
+##### Arguments
 
 Name         | Type      | Description
 ------------ | --------- | -----------
 `service`    | `text`    | Service to check; see the list of valid services above.
 `input_size` | `numeric` | Number of service calls required, i.e. size of the input to be processed.
 
-#### Returns
+##### Returns
 
 The result is a *boolean* value. A *true* value (`'t'`) indicates that the available quota
 for the service is enough for the input size requested. A *false* value (`'f'`) indicates
 insufficient quota.
 
-#### Example
+##### Example
 
 Suppose you want to geocode a whole table. In order to check that you have enough quota, and avoid a "quota exhausted" exception, first find out how many records you need to geocode:
 

docs/developer-center/support/04-rate-limits.md

@@ -1,4 +1,4 @@
-# Rate limits
+## Rate limits
 
 Services can be rate-limited. (currently only gecoding is limited)
 
@@ -15,72 +15,72 @@ If a service request exceeds the configured rate limits
 (i.e. if more than `limit` calls are performe in a fixed interval of
 duration `period` seconds) the call will fail with an "Rate limit exceeded" error.
 
-## Server-side interface
+### Server-side interface
 
 There's a server-side SQL interface to query or change the configuration.
 
-### cdb_dataservices_server.cdb_service_get_rate_limit(username, orgname, service)
+#### cdb_dataservices_server.cdb_service_get_rate_limit(username, orgname, service)
 
 This function returns the rate limit configuration for a given user and service.
 
-#### Returns
+##### Returns
 
 The result is a JSON object with the configuration (`period` and `limit` attributes as explained above).
 
-### cdb_dataservices_server.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
+#### cdb_dataservices_server.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
 
 This function sets the rate limit configuration for the user. This overrides any other configuration.
 
 The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`.
 
-#### Returns
+##### Returns
 
 This functions doesn't return any value.
 
-### cdb_dataservices_server.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
+#### cdb_dataservices_server.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
 
 This function sets the rate limit configuration for the organization.
 This overrides server level configuration and is overriden by user configuration if present.
 
 The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
 
-#### Returns
+##### Returns
 
 This functions doesn't return any value.
 
-### cdb_dataservices_server.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
+#### cdb_dataservices_server.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
 
 This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration.
 
 The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
 
-#### Returns
+##### Returns
 
 This functions doesn't return any value.
 
-## Client-side interface
+### Client-side interface
 
 For convenience there's also a client-side interface (in the client dataservices-api extension), consisting
 of public functions to get the current configuration and privileged functions to change it.
 
-### Public functions
+#### Public functions
 
 These functions are accesible to non-privileged roles, and should only be executed
 using the role corresponding to a CARTO user, since that will determine the
 user and organization to which the rate limits configuration applies.
 
-### cdb_dataservices_client.cdb_service_get_rate_limit(service)
+#### cdb_dataservices_client.cdb_service_get_rate_limit(service)
 
 This function returns the rate limit configuration in effect for the specified service
 and the user corresponding to the role which makes the calls. The effective configuration
 may come from any of the configuration levels (server/organization/user); only the
 existing configuration with most precedence is returned.
 
-#### Returns
+##### Returns
 
 The result is a JSON object with the configuration (`period` and `limit` attributes as explained above).
 
-#### Example:
+##### Example:
 
 ```
 SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoder');
@@ -92,21 +92,21 @@ SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoder');
 ```
 
 
-### Privileged (superuser) functions
+#### Privileged (superuser) functions
 
 Thes functions are not accessible by regular user roles, and the user and organization names must be provided as parameters.
 
-### cdb_dataservices_client.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
+#### cdb_dataservices_client.cdb_service_set_user_rate_limit(username, orgname, service, rate_limit)
 
 This function sets the rate limit configuration for the user. This overrides any other configuration.
 
 The configuration is provided as a JSON literal. To remove the user-level configuration `NULL` should be passed as the `rate_limit`.
 
-#### Returns
+##### Returns
 
 This functions doesn't return any value.
 
-#### Example
+##### Example
 
 This will configure the geocoder service rate limit for  user  `myusername`, a non-organization user.
 The limit will be set at 1000 requests per day. Since the user doesn't belong to any organization,
@@ -129,18 +129,18 @@ SELECT cdb_dataservices_client.cdb_service_set_user_rate_limit(
 (1 row)
 ```
 
-### cdb_dataservices_client.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
+#### cdb_dataservices_client.cdb_service_set_org_rate_limit(username, orgname, service, rate_limit)
 
 This function sets the rate limit configuration for the organization.
 This overrides server level configuration and is overriden by user configuration if present.
 
 The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
 
-#### Returns
+##### Returns
 
 This functions doesn't return any value.
 
-#### Example
+##### Example
 
 This will configure the geocoder service rate limit for the `myorg` organization.
 The limit will be set at 100 requests per hour.
@@ -162,17 +162,17 @@ SELECT cdb_dataservices_client.cdb_service_set_org_rate_limit(
 (1 row)
 ```
 
-### cdb_dataservices_client.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
+#### cdb_dataservices_client.cdb_service_set_server_rate_limit(username, orgname, service, rate_limit)
 
 This function sets the default rate limit configuration for all users accesing the dataservices server. This is overriden by organization of user configuration.
 
 The configuration is provided as a JSON literal. To remove the organization-level configuration `NULL` should be passed as the `rate_limit`.
 
-#### Returns
+##### Returns
 
 This functions doesn't return any value.
 
-#### Example
+##### Example
 
 This will configure the default geocoder service rate limit for all users
 accesing the data-services server.

docs/internal/quota_management.md

@@ -56,11 +56,9 @@ This operation could be done through the rails console:
 
   - Geocoding (street level): `geocoding_quota`, `soft_geocoding_limit`
   - Isolines: `here_isolines_quota`, `soft_here_isolines_limit`
-  - Data observatory snapshot: `obs_snapshot_quota`, `soft_obs_snapshot_limit`
-  - Data observatory general: `obs_general_quota`, `soft_obs_general_limit`
 
 ### How is the quota spent?###
 
-Almost in all the services: geocoding, data observatory snapshot and general the number of spent credits is calculated per request made (either successful or empty request).
+Almost in all the services: geocoding and general the number of spent credits is calculated per request made (either successful or empty request).
 
 In the case of the isolines service, the number of credits is calculated based on the number of isolines generated by the request. Ie. If your query generates 3 isolines for the request, you've spent 3 isolines credits.

server/extension/.gitignore

@@ -6,4 +6,6 @@ cdb_geocoder_server--0.1.0.sql
 cdb_geocoder_server--0.2.0.sql
 cdb_geocoder_server--0.3.0.sql
 cdb_geocoder_server--0.4.0.sql
-cdb_geocoder_server--0.5.0.sql
+cdb_geocoder_server--0.5.0.sql
+cdb_dataservices_server.control
+./test/sql/366_empty_table_test.sql

server/extension/Makefile

@@ -1,20 +1,35 @@
 # Makefile to generate the extension out of separate sql source files.
 # Once a version is released, it is not meant to be changed. E.g: once version 0.0.1 is out, it SHALL NOT be changed.
 EXTENSION = cdb_dataservices_server
-EXTVERSION = $(shell grep default_version $(EXTENSION).control | sed -e "s/default_version[[:space:]]*=[[:space:]]*'\([^']*\)'/\1/")
+EXTVERSION = 0.40.1
+
 # The new version to be generated from templates
-SED = sed
-ERB = erb
-REPLACEMENTS = -i 's/$(EXTVERSION)/$(NEW_VERSION)/g'
-NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
-AWK = awk
+SED ?= sed
+ERB ?= erb
+AWK ?= awk
+PG_CONFIG ?= pg_config
 
 # Parallel support macros
-PG_CONFIG         = pg_config
+PG_CONFIG        ?= pg_config
 PG_PARALLEL      := $(shell $(PG_CONFIG) --version | ($(AWK) '{$$2*=1000; if ($$2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
 
+# PG12 compatibility
+PG_VERSION := $(shell $(PG_CONFIG) --version | $(AWK) '{split($$2,a,"."); print a[1]}')
+PG_12_GE := $(shell [ $(PG_VERSION) -ge 12 ] && echo true)
+PLPYTHONU := plpythonu
+POSTGIS := postgis
+ifeq ($(PG_12_GE), true)
+PLPYTHONU := plpython3u
+POSTGIS := postgis, postgis_raster
+endif
+
+REPLACEMENTS = -e 's/@@EXTVERSION@@/$(EXTVERSION)/g' -e 's/@@plpythonu@@/$(PLPYTHONU)/g' -e 's/@@postgis@@/$(POSTGIS)/g'
+REPLACEMENTS_TEST_REVERSE = -e 's/$(PLPYTHONU)/@@plpythonu@@/g'
+NEW_EXTENSION_ARTIFACT = $(EXTENSION)--$(EXTVERSION).sql
+
 REGRESS = $(notdir $(basename $(sort $(wildcard test/sql/*test.sql))))
-TEST_DIR = test/
+REGRESS_EXPEC = $(notdir $(basename $(sort $(wildcard test/expected/*test.out))))
+TEST_DIR = test
 REGRESS_OPTS = --inputdir='$(TEST_DIR)' --outputdir='$(TEST_DIR)' --user='postgres'
 
 # DATA is a special variable used by postgres build infrastructure
@@ -29,57 +44,70 @@ SOURCES_DATA_DIR = sql/
 SOURCES_DATA = $(wildcard sql/*.sql)
 
 # postgres build stuff
-PG_CONFIG = pg_config
+
+# The targets listed under REGRESS_PREP are executed before regress
+PYTHON_TESTS = ./test/sql/366_empty_table_test.sql
+REGRESS_PREP = $(PYTHON_TESTS)
+
+./test/sql/366_empty_table_test.sql: ./test/sql/366_empty_table_test.sql.template
+	$(SED) $(REPLACEMENTS) $@.template > $@
+
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
+
 $(NEW_EXTENSION_ARTIFACT): $(SOURCES_DATA)
 	rm -f $@
 	cat $(SOURCES_DATA_DIR)/*.sql >> $@
-ifeq ($(PG_PARALLEL), 0)
-# Remove PARALLEL in aggregates and functions
-	$(eval TMPFILE := $(shell mktemp /tmp/$(basename $0).XXXXXXXX))
-	$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' \
-		-e 's/PARALLEL [A-Z]*/''/g' $@ > $(TMPFILE)
-	mv $(TMPFILE) $@
-endif
 
 .PHONY: all
 all: $(DATA)
 
 .PHONY: release
-release: $(EXTENSION).control $(SOURCES_DATA)
+release: $(SOURCES_DATA)
 	test -n "$(NEW_VERSION)"  # $$NEW_VERSION VARIABLE MISSING. Eg. make release NEW_VERSION=0.x.0
 	git mv *.sql old_versions
-	$(SED) $(REPLACEMENTS) $(EXTENSION).control
-	git add $(EXTENSION).control
 	cat $(SOURCES_DATA_DIR)/*.sql > $(EXTENSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(NEW_VERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql
 	$(ERB) version=$(EXTVERSION) upgrade_downgrade_template.erb > $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql
 	git add $(EXTENSION)--$(NEW_VERSION).sql
+	$(SED) -i -e 's/^EXTVERSION =.*/EXTVERSION = $(NEW_VERSION)/g' Makefile
+	git add Makefile
 	@echo
-	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
-	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
+	@echo "Please review the file $(EXTENSION)--$(EXTVERSION)--$(NEW_VERSION).sql.in and add any code needed to upgrade $(EXTVERSION) to $(NEW_VERSION)"
+	@echo "Please review the file $(EXTENSION)--$(NEW_VERSION)--$(EXTVERSION).sql.in and add any code needed to downgrade $(NEW_VERSION) to $(EXTVERSION)"
 	@echo
 
 # Only meant for development time, do not use once a version is released
 .PHONY: devclean
 devclean:
 	rm -f $(NEW_EXTENSION_ARTIFACT)
+	rm -f $(PYTHON_TESTS)
 
-# If needed remove PARALLEL tags from the release files
-release_remove_parallel_deploy:
-ifeq ($(PG_PARALLEL), 0)
+# Replace variables (and PARALLEL tags if necessary) and deploy files
+.PHONY: replace_variables_and_deploy
+replace_variables_and_deploy: $(NEW_EXTENSION_ARTIFACT)
+	mkdir -p '$(DESTDIR)$(datadir)/extension/'; \
 	for n in $(wildcard old_versions/*.sql *.sql); do \
 		$(eval TMPFILE := $(shell mktemp /tmp/XXXXXXXXXX)) \
-		$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
-		mv $(TMPFILE) $$n; \
+		if [ "$(PG_PARALLEL)" -eq "0" ]; then \
+			$(SED) -e 's/PARALLEL \= [A-Z]*,/''/g' -e 's/PARALLEL [A-Z]*/''/g' $$n > $(TMPFILE); \
+			mv $(TMPFILE) $$n; \
+		fi; \
+		$(SED) $(REPLACEMENTS) $$n > $(TMPFILE); \
+		mv $(TMPFILE) '$(DESTDIR)$(datadir)/extension/'$$(basename $$n); \
 	done
-endif
+	
+
+.PHONY: $(EXTENSION).control
+$(EXTENSION).control:
+	$(SED) $(REPLACEMENTS) $(EXTENSION).control.in > $(EXTENSION).control
 
 # Install the current release into the PostgreSQL extensions directory
-deploy: release_remove_parallel_deploy
+.PHONY: deploy
+deploy: replace_variables_and_deploy $(EXTENSION).control
 	$(INSTALL_DATA) $(EXTENSION).control '$(DESTDIR)$(datadir)/extension/'
-	$(INSTALL_DATA) old_versions/*.sql *.sql '$(DESTDIR)$(datadir)/extension/'
 
+.PHONY: install
 install: deploy
+	@echo "Install overriden"