Merge pull request #226 from CartoDB/release_server_13_1

Release server 0.13.1

README.md

@@ -1,5 +1,5 @@
-# Data Services API
-The CartoDB Data Services SQL API (server and client FTM)
+# CARTO Data Services API
+The CARTO Data Services SQL API
 
 ### Deploy instructions
 Steps to deploy a new Data Services API version :
@@ -49,7 +49,7 @@ Steps to deploy a new Data Services API version :
 - install python library
 
     ```
-    cd server/lib/python/cartodb_services && python setup.py install
+    cd server/lib/python/cartodb_services && sudo pip install --upgrade .
     ```
 
 - install extensions in user database
@@ -62,16 +62,16 @@ Steps to deploy a new Data Services API version :
     create extension cdb_dataservices_client;
     ```
 
-- add configuration for different services in user database
+- add configuration for different services in server database
 
 
     ```
     # If sentinel is used:
       SELECT CDB_Conf_SetConf('redis_metadata_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
-      SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
+      SELECT CDB_Conf_SetConf('redis_metrics_config', '{"sentinel_host": "localhost", "sentinel_port": 26379, "sentinel_master_id": "mymaster", "timeout": 0.1, "redis_db": 5}');
       
     # If sentinel is not used
-      SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 26379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
+      SELECT CDB_Conf_SetConf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
       SELECT CDB_Conf_SetConf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "sentinel_master_id": "", "timeout": 0.1, "redis_db": 5}');
   
     SELECT CDB_Conf_SetConf('heremaps_conf', '{"geocoder": {"app_id": "here_geocoder_app_id", "app_code": "here_geocoder_app_code", "geocoder_cost_per_hit": "1"}, "isolines" : {"app_id": "here_isolines_app_id", "app_code": "here_geocoder_app_code"}}');
@@ -81,10 +81,13 @@ Steps to deploy a new Data Services API version :
     SELECT CDB_Conf_SetConf('data_observatory_conf', '{"connection": {"whitelist": [], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}}')
     ```
 
-- configure plproxy to point to the a database (you can use a specific database for the server or your same user)
+- configure the user DB:
 
-    ```
-     SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
+    ```sql
+    -- Point to the dataservices server DB (you can use a specific database for the server or your same user's):
+    SELECT CDB_Conf_SetConf('geocoder_server_config', '{ "connection_str": "host=localhost port=5432 dbname=<SERVER_DB_NAME> user=postgres"}');
+
+    SELECT CDB_Conf_SetConf('user_config', '{"is_organization": false, "entity_name": "<YOUR_USERNAME>"}');
     ```
 
 - configure the search path in order to be able to execute the functions without using the schema:

client/Makefile

@@ -13,8 +13,8 @@ OLD_VERSIONS = $(wildcard old_versions/*.sql)
 # @see http://www.postgresql.org/docs/current/static/extend-pgxs.html
 DATA =  $(NEW_EXTENSION_ARTIFACT) \
   $(OLD_VERSIONS) \
-  cdb_dataservices_client--0.7.0--0.8.0.sql \
-  cdb_dataservices_client--0.8.0--0.7.0.sql
+  cdb_dataservices_client--0.9.0--0.10.0.sql \
+  cdb_dataservices_client--0.10.0--0.9.0.sql
 
 
 REGRESS = $(notdir $(basename $(wildcard test/sql/*test.sql)))

client/README.md

@@ -1,5 +1,5 @@
-# CartoDB dataservices API client extension
-Postgres extension for the CartoDB dataservices API, client side.
+# CARTO Data Services API client extension
+Postgres extension for the CARTO Data Services API, client side.
 
 ## Dependencies
 This extension is thought to be used on top of CartoDB geocoder extension, for the multiples available geocoders (internal, nokia, etc). 
@@ -9,7 +9,7 @@ The following is a non-comprehensive list of dependencies:
 - Postgres 9.3+
 - Postgis extension
 - Schema triggers extension
-- CartoDB extension
+- cartodb-postgresql CARTO extension
 
 ## Installation into the db cluster
 This requires root privileges
@@ -28,7 +28,7 @@ One-liner:
 sudo PGUSER=postgres make all install installcheck
 ```
 
-## Install onto a cartodb user's database
+## Install onto a CARTO user's database
 
 ```
 psql -U postgres cartodb_dev_user_fe3b850a-01c0-48f9-8a26-a82f09e9b53f_db

client/cdb_dataservices_client--0.10.0--0.9.0.sql

@@ -0,0 +1,16 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.9.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetTable(text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_AugmentTable(text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_AugmentTable(text, text, text, text, text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.__OBS_GetTable(text, text, text, text, text, text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_ConnectUserTable(text, text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_GetReturnMetadata(text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_FetchJoinFdwTableData(text, text, text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_DisconnectUserTable(text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.OBS_DumpVersion();
+DROP FUNCTION IF EXISTS cdb_dataservices_client._OBS_DumpVersion(text, text);
+
+DROP TYPE IF EXISTS cdb_dataservices_client.ds_fdw_metadata;
+DROP TYPE IF EXISTS cdb_dataservices_client.ds_return_metadata;

client/cdb_dataservices_client--0.9.0--0.10.0.sql

@@ -0,0 +1,303 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.10.0'" to load this file. \quit
+
+CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
+CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json)
+RETURNS boolean AS $$
+DECLARE
+  username text;
+  user_db_role text;
+  orgname text;
+  dbname text;
+  user_schema text;
+  result boolean;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+
+  SELECT session_user INTO user_db_role;
+
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument';
+  END IF;
+
+  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
+    user_schema := 'public';
+  ELSE
+    user_schema := username;
+  END IF;
+
+  SELECT current_database() INTO dbname;
+
+  SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result;
+
+  RETURN result;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json)
+RETURNS boolean AS $$
+DECLARE
+  username text;
+  user_db_role text;
+  orgname text;
+  dbname text;
+  user_schema text;
+  result boolean;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+
+  SELECT session_user INTO user_db_role;
+
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument';
+  END IF;
+
+  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
+    user_schema := 'public';
+  ELSE
+    user_schema := username;
+  END IF;
+
+  SELECT current_database() INTO dbname;
+
+  SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result;
+
+  RETURN result;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
+RETURNS boolean AS $$
+    from time import strftime
+    try:
+        server_table_name = None
+        temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
+
+        # Obtain return types for augmentation procedure
+        ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
+            "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
+            )
+
+        colnames_arr = ds_return_metadata[0]["colnames"]
+        coltypes_arr = ds_return_metadata[0]["coltypes"]
+
+        # Prepare column and type strings required in the SQL queries
+        colnames = ','.join(colnames_arr)
+        columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
+        columns_with_types = ','.join(columns_with_types_arr)
+
+
+        # Instruct the OBS server side to establish a FDW
+        # The metadata is obtained as well in order to:
+        #   - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
+        #   - (b) be able to tell OBS to free resources when done.
+        ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
+            "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
+            )
+
+        server_schema = ds_fdw_metadata[0]["schemaname"]
+        server_table_name = ds_fdw_metadata[0]["tabname"]
+        server_name = ds_fdw_metadata[0]["servername"]
+
+        # Create temporary table with the augmented results
+        plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
+            '(SELECT {columns}, cartodb_id '
+            'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
+            '{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
+            'AS results({columns_with_types}, cartodb_id int) )'
+            .format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
+                user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
+                function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
+                temp_table_name=temporary_table_name)
+            )
+
+        # Wipe user FDW data from the server
+        wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+            )
+
+        # Prepare table to receive augmented results in new columns
+        for idx, column in enumerate(colnames_arr):
+            if colnames_arr[idx] is not 'the_geom':
+                plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
+                    .format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
+                    )
+
+        # Populate the user table with the augmented results
+        plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
+            '(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
+            'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
+            .format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
+                user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
+                temporary_table_name=temporary_table_name)
+            )
+
+        plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
+            .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
+            )
+
+        return True
+    except Exception as e:
+        plpy.warning('Error trying to augment table {0}'.format(e))
+        # Wipe user FDW data from the server in case of failure if the table was connected
+        if server_table_name:
+            # Wipe local temporary table
+            plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
+                .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
+                )
+
+            wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+                .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+                )
+        return False
+$$ LANGUAGE plpythonu;
+
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json)
+RETURNS boolean AS $$
+    try:
+        server_table_name = None
+        # Obtain return types for augmentation procedure
+        ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
+            "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
+            )
+
+        colnames_arr = ds_return_metadata[0]["colnames"]
+        coltypes_arr = ds_return_metadata[0]["coltypes"]
+
+        # Prepare column and type strings required in the SQL queries
+        colnames = ','.join(colnames_arr)
+        columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
+        columns_with_types = ','.join(columns_with_types_arr)
+
+
+        # Instruct the OBS server side to establish a FDW
+        # The metadata is obtained as well in order to:
+        #   - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
+        #   - (b) be able to tell OBS to free resources when done.
+        ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
+            "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
+            )
+
+        server_schema = ds_fdw_metadata[0]["schemaname"]
+        server_table_name = ds_fdw_metadata[0]["tabname"]
+        server_name = ds_fdw_metadata[0]["servername"]
+
+        # Get list of user columns to include in the new table
+        user_table_columns = ','.join(
+            plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns '
+                'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass '
+                'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' '
+                'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));'
+                .format(user_schema=user_schema, table_name=table_name, colnames=colnames)
+                )[0]["columns"]
+        )
+
+        # Populate a new table with the augmented results
+        plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS '
+            '(SELECT results.{columns}, {user_table_columns} '
+            'FROM {table_name} AS user_table '
+            'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) '
+            'ON results.cartodb_id = user_table.cartodb_id)'
+            .format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username),
+                orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name),
+                table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types)
+            )
+
+        plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
+            .format(schema=user_schema, table_name=output_table_name, user=user_db_role)
+            )
+
+        # Wipe user FDW data from the server
+        wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+            )
+
+        return True
+    except Exception as e:
+        plpy.warning('Error trying to get table {0}'.format(e))
+        # Wipe user FDW data from the server in case of failure if the table was connected
+        if server_table_name:
+            wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+                .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+                )
+        return False
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_ConnectUserTable;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
+RETURNS cdb_dataservices_client.ds_return_metadata AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_GetReturnMetadata;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS SETOF record AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text)
+RETURNS boolean AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_DisconnectUserTable;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.obs_dumpversion ()
+RETURNS text AS $$
+DECLARE
+  ret text;
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    SELECT * FROM cdb_dataservices_client._obs_dumpversion(username, orgname) INTO ret;
+    RETURN ret;
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._obs_dumpversion (username text, organization_name text)
+
+RETURNS text AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.obs_dumpversion (username, organization_name);
+  
+$$ LANGUAGE plproxy;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_AugmentTable(text, text, json) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._OBS_GetTable(text, text, text, json) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.OBS_DumpVersion() TO publicuser;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.8.0'
+default_version = '0.10.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/old_versions/cdb_dataservices_client--0.8.0--0.9.0.sql

@@ -0,0 +1,90 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.9.0'" to load this file. \quit
+
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    RETURN QUERY
+    SELECT * FROM cdb_dataservices_client._cdb_mapzen_isochrone(username, orgname, source, mode, range, options);
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+--
+-- Public dataservices API function
+--
+-- These are the only ones with permissions to publicuser role
+-- and should also be the only ones with SECURITY DEFINER
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance (source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+DECLARE
+  
+  username text;
+  orgname text;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument, check it out';
+  END IF;
+  
+    RETURN QUERY
+    SELECT * FROM cdb_dataservices_client._cdb_mapzen_isodistance(username, orgname, source, mode, range, options);
+  
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isochrone (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_mapzen_isochrone (username, organization_name, source, mode, range, options);
+  
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._cdb_mapzen_isodistance (username text, organization_name text, source geometry(Geometry, 4326), mode text, range integer[], options text[] DEFAULT ARRAY[]::text[])
+RETURNS SETOF cdb_dataservices_client.isoline AS $$
+  CONNECT cdb_dataservices_client._server_conn_str();
+  
+  SELECT * FROM cdb_dataservices_server.cdb_mapzen_isodistance (username, organization_name, source, mode, range, options);
+  
+$$ LANGUAGE plproxy;
+
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isochrone(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_isodistance(source geometry(Geometry, 4326), mode text, range integer[], options text[]) TO publicuser;
+
+-- Grants missing in previous version
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_here_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_google_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;
+
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client.cdb_mapzen_geocode_street_point(searchtext text, city text, state_province text, country text) TO publicuser;

client/old_versions/cdb_dataservices_client--0.9.0--0.8.0.sql

@@ -0,0 +1,7 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.8.0'" to load this file. \quit
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_isochrone(geometry(Geometry, 4326), text, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client.cdb_mapzen_isodistance(geometry(Geometry, 4326), text, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isochrone(text, text, geometry(Geometry, 4326), text, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_client._cdb_mapzen_isodistance(text, text, geometry(Geometry, 4326), text, integer[], text[]);

client/renderer/interface.yaml

@@ -103,6 +103,26 @@
     - { name: range, type: "integer[]" }
     - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
 
+- name: cdb_mapzen_isochrone
+  return_type: SETOF cdb_dataservices_client.isoline
+  multi_row: true
+  multi_field: true
+  params:
+    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: mode, type: text }
+    - { name: range, type: "integer[]" }
+    - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
+
+- name: cdb_mapzen_isodistance
+  return_type: SETOF cdb_dataservices_client.isoline
+  multi_row: true
+  multi_field: true
+  params:
+    - { name: source,  type: "geometry(Geometry, 4326)" }
+    - { name: mode, type: text }
+    - { name: range, type: "integer[]" }
+    - { name: options, type: "text[]", default: 'ARRAY[]::text[]' }
+
 - name: cdb_route_point_to_point
   return_type: cdb_dataservices_client.simple_route
   multi_field: true
@@ -301,3 +321,9 @@
   params:
     - { name: geom,  type: Geometry }
     - { name: timespan, type: text, default: 'NULL'}
+
+- name: obs_dumpversion
+  return_type: text
+  no_params: true
+  params:
+    - {}

client/renderer/sql-template-renderer

@@ -36,6 +36,10 @@ class SqlTemplateRenderer
     @function_signature['multi_row']
   end
 
+  def no_params
+    @function_signature['no_params']
+  end
+
   def user_config_key
     @function_signature['user_config_key']
   end
@@ -45,15 +49,15 @@ class SqlTemplateRenderer
   end
 
   def params
-    @function_signature['params'].map { |p| p['name'] }.join(', ')
+    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']}"}.join(', ')
   end
 
   def params_with_type
-    @function_signature['params'].map { |p| "#{p['name']} #{p['type']}" }.join(', ')
+    @function_signature['params'].reject(&:empty?).map { |p| "#{p['name']} #{p['type']}" }.join(', ')
   end
 
   def params_with_type_and_default
-    parameters = @function_signature['params'].map do |p|
+    parameters = @function_signature['params'].reject(&:empty?).map do |p|
       if not p['default'].nil?
         "#{p['name']} #{p['type']} DEFAULT #{p['default']}"
       else

client/renderer/templates/20_public_functions.erb

@@ -25,10 +25,12 @@ BEGIN
   <% elsif multi_field %>
     SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret;
     RETURN ret;
+  <% elsif no_params %>
+    SELECT * FROM <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname) INTO ret;
+    RETURN ret;
   <% else %>
     SELECT <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %>(username, orgname, <%= params %>) INTO ret;
     RETURN ret;
   <% end %>
 END;
 $$ LANGUAGE 'plpgsql' SECURITY DEFINER;
-

client/renderer/templates/30_plproxy_functions.erb

@@ -1,10 +1,15 @@
+<% if no_params %>
+CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text)
+<% else %>
 CREATE OR REPLACE FUNCTION <%= DATASERVICES_CLIENT_SCHEMA %>._<%= name %> (username text, organization_name text, <%= params_with_type_and_default %>)
+<% end %>
 RETURNS <%= return_type %> AS $$
   CONNECT <%= DATASERVICES_CLIENT_SCHEMA %>._server_conn_str();
   <% if multi_field %>
   SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>);
+  <% elsif no_params %>
+  SELECT * FROM <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name);
   <% else %>
   SELECT <%= DATASERVICES_SERVER_SCHEMA %>.<%= name %> (username, organization_name, <%= params %>);
   <% end %>
 $$ LANGUAGE plproxy;
-

client/sql/20_table_augmentation.sql

@@ -0,0 +1,263 @@
+CREATE TYPE cdb_dataservices_client.ds_fdw_metadata as (schemaname text, tabname text, servername text);
+CREATE TYPE cdb_dataservices_client.ds_return_metadata as (colnames text[], coltypes text[]);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetTable(table_name text, output_table_name text, function_name text, params json)
+RETURNS boolean AS $$
+DECLARE
+  username text;
+  user_db_role text;
+  orgname text;
+  dbname text;
+  user_schema text;
+  result boolean;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+
+  SELECT session_user INTO user_db_role;
+
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument';
+  END IF;
+
+  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
+    user_schema := 'public';
+  ELSE
+    user_schema := username;
+  END IF;
+
+  SELECT current_database() INTO dbname;
+
+  SELECT cdb_dataservices_client.__OBS_GetTable(username, orgname, user_db_role, user_schema, dbname, table_name, output_table_name, function_name, params) INTO result;
+
+  RETURN result;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_AugmentTable(table_name text, function_name text, params json)
+RETURNS boolean AS $$
+DECLARE
+  username text;
+  user_db_role text;
+  orgname text;
+  dbname text;
+  user_schema text;
+  result boolean;
+BEGIN
+  IF session_user = 'publicuser' OR session_user ~ 'cartodb_publicuser_*' THEN
+    RAISE EXCEPTION 'The api_key must be provided';
+  END IF;
+
+  SELECT session_user INTO user_db_role;
+
+  SELECT u, o INTO username, orgname FROM cdb_dataservices_client._cdb_entity_config() AS (u text, o text);
+  -- JSON value stored "" is taken as literal
+  IF username IS NULL OR username = '' OR username = '""' THEN
+    RAISE EXCEPTION 'Username is a mandatory argument';
+  END IF;
+
+  IF orgname IS NULL OR orgname = '' OR orgname = '""' THEN
+    user_schema := 'public';
+  ELSE
+    user_schema := username;
+  END IF;
+
+  SELECT current_database() INTO dbname;
+
+  SELECT cdb_dataservices_client.__OBS_AugmentTable(username, orgname, user_db_role, user_schema, dbname, table_name, function_name, params) INTO result;
+
+  RETURN result;
+END;
+$$ LANGUAGE 'plpgsql' SECURITY DEFINER;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_AugmentTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, function_name text, params json)
+RETURNS boolean AS $$
+    from time import strftime
+    try:
+        server_table_name = None
+        temporary_table_name = 'ds_tmp_' + str(strftime("%s")) + table_name
+
+        # Obtain return types for augmentation procedure
+        ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
+            "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
+            )
+
+        colnames_arr = ds_return_metadata[0]["colnames"]
+        coltypes_arr = ds_return_metadata[0]["coltypes"]
+
+        # Prepare column and type strings required in the SQL queries
+        colnames = ','.join(colnames_arr)
+        columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
+        columns_with_types = ','.join(columns_with_types_arr)
+
+
+        # Instruct the OBS server side to establish a FDW
+        # The metadata is obtained as well in order to:
+        #   - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
+        #   - (b) be able to tell OBS to free resources when done.
+        ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
+            "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {user_schema}::text, {dbname}::text, {table_name}::text);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), user_schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
+            )
+
+        server_schema = ds_fdw_metadata[0]["schemaname"]
+        server_table_name = ds_fdw_metadata[0]["tabname"]
+        server_name = ds_fdw_metadata[0]["servername"]
+
+        # Create temporary table with the augmented results
+        plpy.execute('CREATE UNLOGGED TABLE "{user_schema}".{temp_table_name} AS '
+            '(SELECT {columns}, cartodb_id '
+            'FROM cdb_dataservices_client._OBS_FetchJoinFdwTableData('
+            '{username}::text, {orgname}::text, {schema}::text, {table_name}::text, {function_name}::text, {params}::json) '
+            'AS results({columns_with_types}, cartodb_id int) )'
+            .format(columns=colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
+                user_schema=user_schema, schema=plpy.quote_literal(server_schema), table_name=plpy.quote_literal(server_table_name),
+                function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types,
+                temp_table_name=temporary_table_name)
+            )
+
+        # Wipe user FDW data from the server
+        wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+            )
+
+        # Prepare table to receive augmented results in new columns
+        for idx, column in enumerate(colnames_arr):
+            if colnames_arr[idx] is not 'the_geom':
+                plpy.execute('ALTER TABLE "{user_schema}".{table_name} ADD COLUMN {column_name} {column_type}'
+                    .format(user_schema=user_schema, table_name=table_name, column_name=colnames_arr[idx], column_type=coltypes_arr[idx])
+                    )
+
+        # Populate the user table with the augmented results
+        plpy.execute('UPDATE "{user_schema}".{table_name} SET {columns} = '
+            '(SELECT {columns} FROM "{user_schema}".{temporary_table_name} '
+            'WHERE "{user_schema}".{temporary_table_name}.cartodb_id = "{user_schema}".{table_name}.cartodb_id)'
+            .format(columns = colnames, username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname),
+                user_schema = user_schema, table_name=table_name, function_name=function_name, params=params, columns_with_types=columns_with_types,
+                temporary_table_name=temporary_table_name)
+            )
+
+        plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
+            .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
+            )
+
+        return True
+    except Exception as e:
+        plpy.warning('Error trying to augment table {0}'.format(e))
+        # Wipe user FDW data from the server in case of failure if the table was connected
+        if server_table_name:
+            # Wipe local temporary table
+            plpy.execute('DROP TABLE IF EXISTS "{user_schema}".{temporary_table_name}'
+                .format(user_schema=user_schema, table_name=table_name, temporary_table_name=temporary_table_name)
+                )
+
+            wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+                .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+                )
+        return False
+$$ LANGUAGE plpythonu;
+
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client.__OBS_GetTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text, output_table_name text, function_name text, params json)
+RETURNS boolean AS $$
+    try:
+        server_table_name = None
+        # Obtain return types for augmentation procedure
+        ds_return_metadata = plpy.execute("SELECT colnames, coltypes "
+            "FROM cdb_dataservices_client._OBS_GetReturnMetadata({username}::text, {orgname}::text, {function_name}::text, {params}::json);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params))
+            )
+
+        colnames_arr = ds_return_metadata[0]["colnames"]
+        coltypes_arr = ds_return_metadata[0]["coltypes"]
+
+        # Prepare column and type strings required in the SQL queries
+        colnames = ','.join(colnames_arr)
+        columns_with_types_arr = [colnames_arr[i] + ' ' + coltypes_arr[i] for i in range(0,len(colnames_arr))]
+        columns_with_types = ','.join(columns_with_types_arr)
+
+
+        # Instruct the OBS server side to establish a FDW
+        # The metadata is obtained as well in order to:
+        #   - (a) be able to write the query to grab the actual data to be executed in the remote server via pl/proxy,
+        #   - (b) be able to tell OBS to free resources when done.
+        ds_fdw_metadata = plpy.execute("SELECT schemaname, tabname, servername "
+            "FROM cdb_dataservices_client._OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text);"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(user_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
+            )
+
+        server_schema = ds_fdw_metadata[0]["schemaname"]
+        server_table_name = ds_fdw_metadata[0]["tabname"]
+        server_name = ds_fdw_metadata[0]["servername"]
+
+        # Get list of user columns to include in the new table
+        user_table_columns = ','.join(
+            plpy.execute('SELECT array_agg(\'user_table.\' || attname) AS columns '
+                'FROM pg_attribute WHERE attrelid = \'"{user_schema}".{table_name}\'::regclass '
+                'AND attnum > 0 AND NOT attisdropped AND attname NOT LIKE \'the_geom_webmercator\' '
+                'AND NOT attname LIKE ANY(string_to_array(\'{colnames}\',\',\'));'
+                .format(user_schema=user_schema, table_name=table_name, colnames=colnames)
+                )[0]["columns"]
+        )
+
+        # Populate a new table with the augmented results
+        plpy.execute('CREATE TABLE "{user_schema}".{output_table_name} AS '
+            '(SELECT results.{columns}, {user_table_columns} '
+            'FROM {table_name} AS user_table '
+            'LEFT JOIN cdb_dataservices_client._OBS_FetchJoinFdwTableData({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {function_name}::text, {params}::json) as results({columns_with_types}, cartodb_id int) '
+            'ON results.cartodb_id = user_table.cartodb_id)'
+            .format(output_table_name=output_table_name, columns=colnames, user_table_columns=user_table_columns, username=plpy.quote_nullable(username),
+                orgname=plpy.quote_nullable(orgname), user_schema=user_schema, server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name),
+                table_name=table_name, function_name=plpy.quote_literal(function_name), params=plpy.quote_literal(params), columns_with_types=columns_with_types)
+            )
+
+        plpy.execute('ALTER TABLE "{schema}".{table_name} OWNER TO "{user}";'
+            .format(schema=user_schema, table_name=output_table_name, user=user_db_role)
+            )
+
+        # Wipe user FDW data from the server
+        wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+            .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+            )
+
+        return True
+    except Exception as e:
+        plpy.warning('Error trying to get table {0}'.format(e))
+        # Wipe user FDW data from the server in case of failure if the table was connected
+        if server_table_name:
+            wiped = plpy.execute("SELECT cdb_dataservices_client._OBS_DisconnectUserTable({username}::text, {orgname}::text, {server_schema}::text, {server_table_name}::text, {fdw_server}::text)"
+                .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), server_schema=plpy.quote_literal(server_schema), server_table_name=plpy.quote_literal(server_table_name), fdw_server=plpy.quote_literal(server_name))
+                )
+        return False
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_ConnectUserTable(username text, orgname text, user_db_role text, user_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_ConnectUserTable;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
+RETURNS cdb_dataservices_client.ds_return_metadata AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_GetReturnMetadata;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS SETOF record AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_FetchJoinFdwTableData;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_client._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, server_name text)
+RETURNS boolean AS $$
+    CONNECT _server_conn_str();
+    TARGET cdb_dataservices_server._OBS_DisconnectUserTable;
+$$ LANGUAGE plproxy;

client/sql/95_grant_execute_manual.sql

@@ -0,0 +1,2 @@
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_augmenttable(table_name text, function_name text, params json) TO publicuser;
+GRANT EXECUTE ON FUNCTION cdb_dataservices_client._obs_gettable(table_name text, output_table_name text, function_name text, params json) TO publicuser;

client/test/expected/95_data_observatory_tables_test.out

@@ -0,0 +1,67 @@
+-- Add to the search path the schema
+SET search_path TO public,cartodb,cdb_dataservices_client;
+CREATE TABLE my_table(cartodb_id int);
+INSERT INTO my_table (cartodb_id) VALUES (1);
+-- Mock the server functions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
+BEGIN
+  RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
+RETURNS cdb_dataservices_client.ds_return_metadata AS $$
+BEGIN
+  RETURN (Array['total_pop'], Array['double precision']);
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS RECORD AS $$
+BEGIN
+  RETURN (23.4::double precision, 1::int);
+END;
+$$ LANGUAGE 'plpgsql';
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
+RETURNS boolean AS $$
+BEGIN
+  RETURN true;
+END;
+$$ LANGUAGE 'plpgsql';
+-- Augment a table with the total_pop column
+SELECT cdb_dataservices_client._OBS_AugmentTable('my_table', 'dummy', '{"dummy":"dummy"}'::json);
+ _obs_augmenttable 
+-------------------
+ t
+(1 row)
+
+-- The results of the table should return the mocked value of 23.4 in the total_pop column
+SELECT * FROM my_table;
+ cartodb_id | total_pop 
+------------+-----------
+          1 |      23.4
+(1 row)
+
+-- Mock again the function for it to return a different value now
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS RECORD AS $$
+BEGIN
+  RETURN (577777.4::double precision, 1::int);
+END;
+$$ LANGUAGE 'plpgsql';
+-- Augment a new table with total_pop
+SELECT cdb_dataservices_client._OBS_GetTable('my_table', 'my_table_new', 'dummy', '{"dummy":"dummy"}'::json);
+ _obs_gettable 
+---------------
+ t
+(1 row)
+
+-- Check that the table contains the new value for total_pop and not the value already existent in the table
+SELECT * FROM my_table_new;
+ total_pop | cartodb_id 
+-----------+------------
+  577777.4 |          1
+(1 row)
+
+-- Clean tables
+DROP TABLE my_table;
+DROP TABLE my_table_new;

client/test/sql/95_data_observatory_tables_test.sql

@@ -0,0 +1,59 @@
+-- Add to the search path the schema
+SET search_path TO public,cartodb,cdb_dataservices_client;
+
+CREATE TABLE my_table(cartodb_id int);
+
+INSERT INTO my_table (cartodb_id) VALUES (1);
+
+-- Mock the server functions
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_client.ds_fdw_metadata AS $$
+BEGIN
+  RETURN ('dummy_schema'::text, 'dummy_table'::text, 'dummy_server'::text);
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
+RETURNS cdb_dataservices_client.ds_return_metadata AS $$
+BEGIN
+  RETURN (Array['total_pop'], Array['double precision']);
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS RECORD AS $$
+BEGIN
+  RETURN (23.4::double precision, 1::int);
+END;
+$$ LANGUAGE 'plpgsql';
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
+RETURNS boolean AS $$
+BEGIN
+  RETURN true;
+END;
+$$ LANGUAGE 'plpgsql';
+
+-- Augment a table with the total_pop column
+SELECT cdb_dataservices_client._OBS_AugmentTable('my_table', 'dummy', '{"dummy":"dummy"}'::json);
+
+-- The results of the table should return the mocked value of 23.4 in the total_pop column
+SELECT * FROM my_table;
+
+-- Mock again the function for it to return a different value now
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS RECORD AS $$
+BEGIN
+  RETURN (577777.4::double precision, 1::int);
+END;
+$$ LANGUAGE 'plpgsql';
+
+-- Augment a new table with total_pop
+SELECT cdb_dataservices_client._OBS_GetTable('my_table', 'my_table_new', 'dummy', '{"dummy":"dummy"}'::json);
+
+-- Check that the table contains the new value for total_pop and not the value already existent in the table
+SELECT * FROM my_table_new;
+
+-- Clean tables
+DROP TABLE my_table;
+DROP TABLE my_table_new;

doc/API.md

@@ -1,6 +1,6 @@
 # Data Services API
 
-The CartoDB Data Services API offers a set of location based services that can be used programatically to empower your geospatial applications.
+The CARTO Data Services API offers a set of location based services that can be used programatically to empower your geospatial applications.
 
 ## Documentation
 

doc/demographic_functions.md

@@ -1,6 +1,6 @@
 # Demographic Functions
 
-The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CartoDB's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/cartodb-platform/dataservices-api/geocoding-functions/) documentation.
+The Demographic Snapshot enables you to collect demographic reports around a point location. For example, you can take the coordinates of a coffee shop and find the average population characteristics, such as total population, educational attainment, housing and income information around that location. You can use raw street addresses by combining the Demographic Snapshot with CARTO's geocoding features. If you need help creating coordinates from addresses, see the [Geocoding Functions](/carto-engine/dataservices-api/geocoding-functions/) documentation.
 
 _**Note:** The Demographic Snapshot functions are only available for the United States._
 
@@ -40,7 +40,7 @@ obs_getdemographicsnapshot: {
 ### Examples
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
+https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
 OBS_GetDemographicSnapshot({{point geometry}})
 ```
 
@@ -49,14 +49,14 @@ OBS_GetDemographicSnapshot({{point geometry}})
 __Get the Demographic Snapshot at Camp David__
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
+https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
 OBS_GetDemographicSnapshot(CDB_LatLng(39.648333, -77.465))
 ```
 
 __Get the Demographic Snapshot in the Upper West Side__
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
+https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
 OBS_GetDemographicSnapshot(CDB_LatLng(40.80, -73.960))
 ```
 

doc/geocoding_functions.md

@@ -1,25 +1,25 @@
 # Geocoding Functions
 
-The [geocoder](https://cartodb.com/data/geocoder-api/) functions allow you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CartoDB SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes, IP addresses and street addresses. CartoDB provides functions for several different categories of geocoding through the Data Services API.
+The [geocoder](https://carto.com/data/geocoder-api/) functions allow you to match your data with geometries on your map. This geocoding service can be used programatically to geocode datasets via the CARTO SQL API. It is fed from _Open Data_ and it serves geometries for countries, provinces, states, cities, postal codes, IP addresses and street addresses. CARTO provides functions for several different categories of geocoding through the Data Services API.
 
-_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
+_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
 
 Here is an example of how to geocode a single country:
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT cdb_geocode_admin0_polygon('USA')&api_key={api_key}
+https://{username}.carto.com/api/v2/sql?q=SELECT cdb_geocode_admin0_polygon('USA')&api_key={api_key}
 ```
 
-In order to geocode an existent CartoDB dataset, an SQL UPDATE statement must be used to populate the geometry column in the dataset with the results of the Data Services API. For example, if the column where you are storing the country names for each one of our rows is called `country_column`, run the following statement in order to geocode the dataset:
+In order to geocode an existent CARTO dataset, an SQL UPDATE statement must be used to populate the geometry column in the dataset with the results of the Data Services API. For example, if the column where you are storing the country names for each one of our rows is called `country_column`, run the following statement in order to geocode the dataset:
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = cdb_geocode_admin0_polygon('USA')&api_key={api_key}
+https://{username}.carto.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = cdb_geocode_admin0_polygon('USA')&api_key={api_key}
 ```
 
 Notice that you can make use of Postgres or PostGIS functions in your Data Services API requests, as the result is a geometry that can be handled by the system. For example, suppose you need to retrieve the centroid of a specific country, you can wrap the resulting geometry from the geocoder functions inside the PostGIS `ST_Centroid` function:
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = ST_Centroid(cdb_geocode_admin0_polygon('USA'))&api_key={api_key}
+https://{username}.carto.com/api/v2/sql?q=UPDATE {tablename} SET the_geom = ST_Centroid(cdb_geocode_admin0_polygon('USA'))&api_key={api_key}
 ```
 
 
@@ -217,11 +217,11 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_namedplace_point('New York
 
 This function geocodes your data into point, or polygon, geometries for postal codes. The postal code polygon geocoder covers the United States, France, Australia and Canada; a request for a different country will return an empty response.
 
-**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](http://docs.cartodb.com/faqs/datasets-and-data/#why-does-cartodb-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details.
+**Note:** For the USA, US Census [Zip Code Tabulation Areas](https://www.census.gov/geo/reference/zctas.html) (ZCTA) are used to reference geocodes for USPS postal codes service areas. See the [FAQs](https://carto.com/docs/faqs/datasets-and-data/#why-does-carto-use-census-bureau-zctas-and-not-usps-zip-codes-for-postal-codes) about datasets and data for details.
 
 ### cdb_geocode_postalcode_polygon(_postal_code text, country_name text_)
 
-Goecodes the postal code for a specified country into a **polygon** geometry.
+Geocodes the postal code for a specified country into a **polygon** geometry.
 
 #### Arguments
 
@@ -250,7 +250,7 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_postalcode_polygon('11211'
 
 ### cdb_geocode_postalcode_point(_code text, country_name text_)
 
-Goecodes the postal code for a specified country into a **point** geometry.
+Geocodes the postal code for a specified country into a **point** geometry.
 
 #### Arguments
 
@@ -313,9 +313,9 @@ INSERT INTO {tablename} (the_geom) SELECT cdb_geocode_ipaddress_point('102.23.34
 
 ## Street-Level Geocoder
 
-This function geocodes your data into a point geometry for a street address. CartoDB uses several different service providers for street-level geocoding, depending on your platform. If you access CartoDB on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [HERE geocoding services](https://developer.here.com/rest-apis/documentation/geocoder/topics/quick-start.html). Additional service providers will be implemented in the future.
+This function geocodes your data into a point geometry for a street address. CARTO uses several different service providers for street-level geocoding, depending on your platform. If you access CARTO on a Google Cloud Platform, [Google Maps geocoding](https://developers.google.com/maps/documentation/geocoding/intro) is applied. All other platform users are provided with [HERE geocoding services](https://developer.here.com/rest-apis/documentation/geocoder/topics/quick-start.html). Additional service providers will be implemented in the future.
 
-**This service is subject to quota limitations, and extra fees may apply**. View the [Quota information](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) for details and recommendations about quota consumption.
+**This service is subject to quota limitations, and extra fees may apply**. View the [Quota information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) for details and recommendations about quota consumption.
 
 ### cdb_geocode_street_point(_search_text text, [city text], [state text], [country text]_)
 

doc/isoline_functions.md

@@ -1,13 +1,13 @@
 # Isoline Functions
 
-[Isolines](https://cartodb.com/data/isolines/) are contoured lines that display equally calculated levels over a given surface area. This enables you to view polygon dimensions by forward or reverse measurements. Isoline functions are calculated as the intersection of areas from the origin point, measured by distance (isodistance) or time (isochrone). For example, the distance of a road from a sidewalk. Isoline services through CartoDB are available by requesting a single function in the Data Services API.
+[Isolines](https://carto.com/data/isolines/) are contoured lines that display equally calculated levels over a given surface area. This enables you to view polygon dimensions by forward or reverse measurements. Isoline functions are calculated as the intersection of areas from the origin point, measured by distance (isodistance) or time (isochrone). For example, the distance of a road from a sidewalk. Isoline services through CARTO are available by requesting a single function in the Data Services API.
 
-_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
+_**This service is subject to quota limitations and extra fees may apply**. View the [Quota Information](https://carto.com/docs/carto-engine/dataservices-api/quota-information/) section for details and recommendations about to quota consumption._
 
 You can use the isoline functions to retrieve, for example, isochrone lines from a certain location, specifying the mode and the ranges that will define each of the isolines. The following query calculates isolines for areas that are 5, 10 and 15 minutes (300, 600 and 900 seconds, respectively) away from the location by following a path defined by car routing and inserts them into a table.
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=INSERT INTO {table} (the_geom) SELECT  the_geom FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 600, 900]::integer[])&api_key={api_key}
+https://{username}.carto.com/api/v2/sql?q=INSERT INTO {table} (the_geom) SELECT  the_geom FROM cdb_isodistance('POINT(-3.70568 40.42028)'::geometry, 'car', ARRAY[300, 600, 900]::integer[])&api_key={api_key}
 ```
 
 The following functions provide an isoline generator service, based on time or distance. This service uses the isolines service defined for your account. The default service limits the usage of displayed polygons represented on top of [HERE](https://developer.here.com/coverage-info) maps.

doc/overview.md

@@ -1,32 +1,32 @@
 # Overview
 
-By using CartoDB libraries and the SQL API, you can apply location data services to your maps with unique data services functions. These functions are integrated with a number of internal and external services, enabling you to programatically customize subsets of data for your visualizations. These features are useful for geospatial analysis and the results can be saved, and stored, for additional location data service operations.
+By using CARTO libraries and the SQL API, you can apply location data services to your maps with unique data services functions. These functions are integrated with a number of internal and external services, enabling you to programatically customize subsets of data for your visualizations. These features are useful for geospatial analysis and the results can be saved, and stored, for additional location data service operations.
 
-**Note:** Based on your account plan, some of these data services are subject to different [quota limitations](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/#quota-information).
+**Note:** Based on your account plan, some of these data services are subject to different [quota limitations](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-information).
 
-_The Data Services API is collaborating with [Mapzen](https://mapzen.com/), and several other geospatial service providers, in order to supply the best location data services from within our CartoDB Platform._
+_The Data Services API is collaborating with [Mapzen](https://mapzen.com/), and several other geospatial service providers, in order to supply the best location data services from within our CARTO Engine._
 
 ## Data Services Integration
 
 By using the SQL API to query the Data Services API functions, you can manage specific operations and the corresponding geometries (a `polygon` or a `point`), according to the input information.
 
-The Data Services API decouples the geocoding and isoline services from the CartoDB Editor. The API allows you to geocode data (from single rows, complete datasets, or simple inputs) and to perform trade areas analysis (computing isodistances or isochrones) programatically, through authenticated requests.
+The Data Services API decouples the geocoding and isoline services from the CARTO Editor. The API allows you to geocode data (from single rows, complete datasets, or simple inputs) and to perform trade areas analysis (computing isodistances or isochrones) programatically, through authenticated requests.
 
 The geometries provided by this API are projected in the projection [WGS 84 SRID 4326](http://spatialreference.org/ref/epsg/wgs-84/).
 
-**Note:** The Data Services API [geocoding functions](http://docs.cartodb.com/cartodb-platform/dataservices-api/geocoding-functions/#geocoding-functions) return different types of geometries (points or polygons) as result of different geocoding processes. The CartoDB Platform does not support multi-geometry layers or datasets, therefore you must confirm that you are using consistent geometry types inside a table, to avoid future conflicts in your map visualization.
+**Note:** The Data Services API [geocoding functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/#geocoding-functions) return different types of geometries (points or polygons) as result of different geocoding processes. The CARTO Engine does not support multi-geometry layers or datasets, therefore you must confirm that you are using consistent geometry types inside a table, to avoid future conflicts in your map visualization.
 
 ### Best Practices
 
 _Be mindful of the following usage notes when using the Data Services functions with the SQL API:_
 
-It is discouraged to use the SELECT operation with the Data Services API functions in your map layers, as these type of queries consume quota when rendering tiles for your live map views. It may also result in sync performance issues, due to executing multiple requests to the API each time your map is viewed. See details about [Quota Consumption](http://docs.cartodb.com/cartodb-platform/dataservices-api/quota-information/#quota-consumption).
+It is discouraged to use the SELECT operation with the Data Services API functions in your map layers, as these type of queries consume quota when rendering tiles for your live map views. It may also result in sync performance issues, due to executing multiple requests to the API each time your map is viewed. See details about [Quota Consumption](https://carto.com/docs/carto-engine/dataservices-api/quota-information/#quota-consumption).
 
 The Data Services API is **recommended** to be used with INSERT or UPDATE operations, for applying location data to your tables. While SELECT (retrieve) is standard for SQL API requests, be mindful of quota consumption and use INSERT (to insert a new record) or UPDATE (to update an existing record), for best practices.
 
 ## Authentication
 
-All requests performed to the CartoDB Data Services API must be authenticated with the user API Key. For more information about where to find your API Key, and how to authenticate your SQL API requests, view the [SQL API authentication](/cartodb-platform/sql-api/authentication/) documentation.
+All requests performed to the CARTO Data Services API must be authenticated with the user API Key. For more information about where to find your API Key, and how to authenticate your SQL API requests, view the [SQL API authentication](/carto-engine/sql-api/authentication/) documentation.
 
 ## Errors
 
@@ -40,10 +40,10 @@ Errors are described in the response of the request. An example is as follows:
 }
 ```
 
-Since the Data Services API is used on top of the CartoDB SQL API, you can refer to the [Making calls to the SQL API](/cartodb-platform/sql-api/making-calls/) documentation for help debugging your SQL errors.
+Since the Data Services API is used on top of the CARTO SQL API, you can refer to the [Making calls to the SQL API](https://carto.com/docs/carto-engine/sql-api/making-calls/) documentation for help debugging your SQL errors.
 
-If the requested information is not in the CartoDB geocoding database, or if CartoDB is unable to recognize your input and match it with a result, the geocoding function returns `null` as a result.
+If the requested information is not in the CARTO geocoding database, or if CARTO is unable to recognize your input and match it with a result, the geocoding function returns `null` as a result.
 
 ## Limits
 
-Usage of the Data Services API is subject to the CartoDB SQL API limits, stated in our [Terms of Service](https://cartodb.com/terms/#excessive).
+Usage of the Data Services API is subject to the CARTO SQL API limits, stated in our [Terms of Service](https://carto.com/terms/#excessive).

doc/quota_information.md

@@ -1,6 +1,6 @@
 # Quota Information
 
-**Based on your account plan, some of the Data Services API functions are subject to quota limitations and extra fees may apply.** View our [terms and conditions](https://cartodb.com/terms/), or [contact us](mailto:sales@cartodb.com) for details about which functions require service credits to your account.
+**Based on your account plan, some of the Data Services API functions are subject to quota limitations and extra fees may apply.** View our [terms and conditions](https://carto.com/terms/), or [contact us](mailto:sales@carto.com) for details about which functions require service credits to your account.
 
 ## Quota Consumption
 

doc/routing_functions.md

@@ -1,6 +1,6 @@
 # Routing Functions
 
-Routing is the navigation from a defined start location to a defined end location. The calculated results are displayed as turn-by-turn directions on your map, based on the transportation mode that you specified. Routing services through CartoDB are available by using the available functions in the Data Services API.
+Routing is the navigation from a defined start location to a defined end location. The calculated results are displayed as turn-by-turn directions on your map, based on the transportation mode that you specified. Routing services through CARTO are available by using the available functions in the Data Services API.
 
 ## cdb_route_point_to_point(_origin geometry(Point), destination geometry(Point), mode text, [options text[], units text]_)
 

doc/segmentation_functions.md

@@ -1,6 +1,6 @@
 # Segmentation Functions
 
-The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions](/cartodb-platform/dataservices-api/geocoding-functions/) documentation.
+The Segmentation Snapshot functions enable you to determine the pre-calculated population segment for a location. Segmentation is a method that divides a populations into subclassifications based on common traits. For example, you can take the a store location and determine what classification of population exists around that location. If you need help creating coordinates from addresses, see the [Geocoding Functions](https://carto.com/docs/carto-engine/dataservices-api/geocoding-functions/) documentation.
 
 _**Note:** The Segmentation Snapshot functions are only available for the United States. Our first release (May 18, 2016) is derived from Census 2010 variables. Our next release will be based on Census 2014 data. For the latest information, see the [Open Segments](https://github.com/CartoDB/open-segments) project repository._
 
@@ -158,7 +158,7 @@ The possible segments are:
 ### Examples
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
+https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
 OBS_GetSegmentSnapshot({{point geometry}})
 ```
 
@@ -168,14 +168,14 @@ __Get the Segmentation Snapshot around the MGM Grand__
 
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
+https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
 OBS_GetSegmentSnapshot(CDB_LatLng(36.10222, -115.169516))
 ```
 
-__Get the Segmentation Snapshot at CartoDB's NYC HQ__
+__Get the Segmentation Snapshot at CARTO's NYC HQ__
 
 
 ```bash
-https://{username}.cartodb.com/api/v2/sql?q=SELECT * FROM
+https://{username}.carto.com/api/v2/sql?q=SELECT * FROM
 OBS_GetSegmentSnapshot(CDB_LatLng(40.704512, -73.936669))
 ```

server/extension/README.md

@@ -1,15 +1,15 @@
-# CartoDB dataservices API server extension
-Postgres extension for the CartoDB dataservices API, server side.
+# CARTO Data Services API server extension
+Postgres extension for the CARTO Data Services API, server side.
 
 ## Dependencies
-This extension is thought to be used on top of CartoDB geocoder extension, for the internal geocoder. 
+This extension is thought to be used on top of CARTO geocoder extension, for the internal geocoder. 
 
 The following is a non-comprehensive list of dependencies:
 
 - Postgres 9.3+
 - Postgis extension
 - Schema triggers extension
-- CartoDB extension
+- cartodb-postgresql CARTO extension
 
 ## Installation into the db cluster
 This requires root privileges
@@ -28,7 +28,7 @@ One-liner:
 sudo PGUSER=postgres make all install installcheck
 ```
 
-## Install onto a cartodb user's database
+## Install onto a CARTO user's database
 
 Remember that **is mandatory to install it on top of cdb_geocoder**
 

server/extension/cdb_dataservices_server--0.13.0--0.13.1.sql

@@ -0,0 +1,21 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
+    return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
+        .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
+        )[0]
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_ConnectUserTable;
+$$ LANGUAGE plproxy;
+

server/extension/cdb_dataservices_server--0.13.1--0.13.0.sql

@@ -0,0 +1,21 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text, text);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
+        .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
+        )[0]
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_ConnectUserTable;
+$$ LANGUAGE plproxy;
+
+

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.11.0'
+default_version = '0.13.1'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/old_versions/cdb_dataservices_server--0.11.0--0.12.0.sql

@@ -0,0 +1,133 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
+   username TEXT,
+   orgname TEXT,
+   isotype TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient
+  from cartodb_services.mapzen import MapzenIsolines
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+
+  # -- Check the quota
+  quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+    client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
+    mapzen_isolines = MapzenIsolines(client)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    # -- TODO Support options properly
+    isolines = {}
+    if isotype == 'isodistance':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+          isolines[r] = (isoline)
+    elif isotype == 'isochrone':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
+          isolines[r] = (isoline)
+
+    result = []
+    for r in data_range:
+
+      # -- TODO encapsulate this block into a func/method
+      locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+      wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
+      sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+      multipolygon = plpy.execute(sql, 1)[0]['geom']
+
+      result.append([source, r, multipolygon])
+
+    quota_service.increment_success_service_use()
+    quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
+    plpy.debug(traceback.format_tb(traceback_))
+    raise e
+    #plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+
+
+-- mapzen isodistance
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isodistance'
+
+  mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  isolines = []
+  for element in result:
+    isoline = element['isoline']
+    isoline = isoline.translate(None, "()").split(',')
+    isolines.append(isoline)
+
+  return isolines
+$$ LANGUAGE plpythonu;
+
+
+
+-- mapzen isochrones
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isochrone'
+
+  mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  isolines = []
+  for element in result:
+    isoline = element['isoline']
+    isoline = isoline.translate(None, "()").split(',') #--TODO what is this for?
+    isolines.append(isoline)
+
+  return isolines
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import MapzenIsolinesRoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = mapzen_isolines_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/old_versions/cdb_dataservices_server--0.11.0.sql

@@ -2131,4 +2131,4 @@ BEGIN
     GRANT USAGE ON SCHEMA cdb_dataservices_server TO geocoder_api;
     GRANT USAGE ON SCHEMA public TO geocoder_api;
     GRANT SELECT ON ALL TABLES IN SCHEMA public TO geocoder_api;
-END$$;
+END$$;

server/extension/old_versions/cdb_dataservices_server--0.12.0--0.11.0.sql

@@ -0,0 +1,8 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.11.0'" to load this file. \quit
+
+DROP FUNCTION IF EXISTS cdb_dataservices_server._cdb_mapzen_isolines(text, text, text, geometry(Geometry, 4326), text, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_isodistance(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.cdb_mapzen_isochrone(TEXT, TEXT, geometry(Geometry, 4326), TEXT, integer[], text[]);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._get_mapzen_isolines_config(text, text);

server/extension/old_versions/cdb_dataservices_server--0.12.0--0.13.0.sql

@@ -0,0 +1,149 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.13.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text);
+
+CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {table_name}::text)"
+        .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name))
+        )[0]
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_ConnectUserTable;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
+RETURNS cdb_dataservices_server.ds_return_metadata AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_GetReturnMetadata;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS SETOF record AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
+$$ LANGUAGE plproxy;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
+RETURNS boolean AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_DisconnectUserTable;
+$$ LANGUAGE plproxy;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
+   username TEXT,
+   orgname TEXT,
+   isotype TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient
+  from cartodb_services.mapzen import MapzenIsolines
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+
+  # -- Check the quota
+  quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+    client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
+    mapzen_isolines = MapzenIsolines(client)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    # -- TODO Support options properly
+    isolines = {}
+    if isotype == 'isodistance':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+          isolines[r] = isoline
+    elif isotype == 'isochrone':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
+          isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    quota_service.increment_success_service_use()
+    quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
+    plpy.debug(traceback.format_tb(traceback_))
+    raise e
+    #plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isodistance'
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isochrone'
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text)
+RETURNS text AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.obs_dumpversion();
+$$ LANGUAGE plproxy;

server/extension/old_versions/cdb_dataservices_server--0.13.0--0.12.0.sql

@@ -0,0 +1,124 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.12.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_ConnectUserTable(text, text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.__OBS_ConnectUserTable(text, text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_GetReturnMetadata(text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_FetchJoinFdwTableData(text, text, text, text, text, json);
+DROP FUNCTION IF EXISTS cdb_dataservices_server._OBS_DisconnectUserTable(text, text, text, text, text);
+DROP FUNCTION IF EXISTS cdb_dataservices_server.OBS_DumpVersion(text, text);
+
+DROP TYPE IF EXISTS cdb_dataservices_server.ds_fdw_metadata;
+DROP TYPE IF EXISTS cdb_dataservices_server.ds_return_metadata;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isodistance'
+
+  mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  isolines = []
+  for element in result:
+    isoline = element['isoline']
+    isoline = isoline.translate(None, "()").split(',')
+    isolines.append(isoline)
+
+  return isolines
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isochrone'
+
+  mapzen_plan = plpy.prepare("SELECT cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+  isolines = []
+  for element in result:
+    isoline = element['isoline']
+    isoline = isoline.translate(None, "()").split(',') #--TODO what is this for?
+    isolines.append(isoline)
+
+  return isolines
+$$ LANGUAGE plpythonu;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
+   username TEXT,
+   orgname TEXT,
+   isotype TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient
+  from cartodb_services.mapzen import MapzenIsolines
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+
+  # -- Check the quota
+  quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+    client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
+    mapzen_isolines = MapzenIsolines(client)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    # -- TODO Support options properly
+    isolines = {}
+    if isotype == 'isodistance':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+          isolines[r] = (isoline)
+    elif isotype == 'isochrone':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
+          isolines[r] = (isoline)
+
+    result = []
+    for r in data_range:
+
+      # -- TODO encapsulate this block into a func/method
+      locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+      wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
+      sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+      multipolygon = plpy.execute(sql, 1)[0]['geom']
+
+      result.append([source, r, multipolygon])
+
+    quota_service.increment_success_service_use()
+    quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
+    plpy.debug(traceback.format_tb(traceback_))
+    raise e
+    #plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+

server/extension/sql/125_data_observatory_table_augment.sql

@@ -0,0 +1,36 @@
+CREATE TYPE cdb_dataservices_server.ds_fdw_metadata as (schemaname text, tabname text, servername text);
+
+CREATE TYPE cdb_dataservices_server.ds_return_metadata as (colnames text[], coltypes text[]);
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    host_addr = plpy.execute("SELECT split_part(inet_client_addr()::text, '/', 1) as user_host")[0]['user_host']
+    return plpy.execute("SELECT * FROM cdb_dataservices_server.__OBS_ConnectUserTable({username}::text, {orgname}::text, {user_db_role}::text, {schema}::text, {dbname}::text, {host_addr}::text, {table_name}::text)"
+        .format(username=plpy.quote_nullable(username), orgname=plpy.quote_nullable(orgname), user_db_role=plpy.quote_literal(user_db_role), schema=plpy.quote_literal(input_schema), dbname=plpy.quote_literal(dbname), table_name=plpy.quote_literal(table_name), host_addr=plpy.quote_literal(host_addr))
+        )[0]
+$$ LANGUAGE plpythonu;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.__OBS_ConnectUserTable(username text, orgname text, user_db_role text, input_schema text, dbname text, host_addr text, table_name text)
+RETURNS cdb_dataservices_server.ds_fdw_metadata AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_ConnectUserTable;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_GetReturnMetadata(username text, orgname text, function_name text, params json)
+RETURNS cdb_dataservices_server.ds_return_metadata AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_GetReturnMetadata;
+$$ LANGUAGE plproxy;
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_FetchJoinFdwTableData(username text, orgname text, table_schema text, table_name text, function_name text, params json)
+RETURNS SETOF record AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_FetchJoinFdwTableData;
+$$ LANGUAGE plproxy;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._OBS_DisconnectUserTable(username text, orgname text, table_schema text, table_name text, servername text)
+RETURNS boolean AS $$
+    CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+    TARGET cdb_observatory._OBS_DisconnectUserTable;
+$$ LANGUAGE plproxy;

server/extension/sql/130_data_observatory_helper.sql

@@ -0,0 +1,5 @@
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.obs_dumpversion(username text, orgname text)
+RETURNS text AS $$
+  CONNECT cdb_dataservices_server._obs_server_conn_str(username, orgname);
+  SELECT cdb_observatory.obs_dumpversion();
+$$ LANGUAGE plproxy;

server/extension/sql/15_config_helper.sql

@@ -26,6 +26,20 @@ RETURNS boolean AS $$
     return True
 $$ LANGUAGE plpythonu SECURITY DEFINER;
 
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_mapzen_isolines_config(username text, orgname text)
+RETURNS boolean AS $$
+  cache_key = "user_mapzen_isolines_routing_config_{0}".format(username)
+  if cache_key in GD:
+    return False
+  else:
+    from cartodb_services.metrics import MapzenIsolinesRoutingConfig
+    plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+    redis_conn = GD["redis_connection_{0}".format(username)]['redis_metadata_connection']
+    mapzen_isolines_config = MapzenIsolinesRoutingConfig(redis_conn, plpy, username, orgname)
+    GD[cache_key] = mapzen_isolines_config
+    return True
+$$ LANGUAGE plpythonu SECURITY DEFINER;
+
 CREATE OR REPLACE FUNCTION cdb_dataservices_server._get_internal_geocoder_config(username text, orgname text)
 RETURNS boolean AS $$
   cache_key = "user_internal_geocoder_config_{0}".format(username)

server/extension/sql/80_isolines_helper.sql

@@ -53,3 +53,77 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
   finally:
     quota_service.increment_total_service_use()
 $$ LANGUAGE plpythonu SECURITY DEFINER;
+
+
+CREATE OR REPLACE FUNCTION cdb_dataservices_server._cdb_mapzen_isolines(
+   username TEXT,
+   orgname TEXT,
+   isotype TEXT,
+   source geometry(Geometry, 4326),
+   mode TEXT,
+   data_range integer[],
+   options text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  import json
+  from cartodb_services.mapzen import MatrixClient
+  from cartodb_services.mapzen import MapzenIsolines
+  from cartodb_services.metrics import QuotaService
+
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  user_mapzen_isolines_routing_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+
+  # -- Check the quota
+  quota_service = QuotaService(user_mapzen_isolines_routing_config, redis_conn)
+  if not quota_service.check_user_quota():
+    plpy.error('You have reached the limit of your quota')
+
+  try:
+    client = MatrixClient(user_mapzen_isolines_routing_config.mapzen_matrix_api_key)
+    mapzen_isolines = MapzenIsolines(client)
+
+    if source:
+      lat = plpy.execute("SELECT ST_Y('%s') AS lat" % source)[0]['lat']
+      lon = plpy.execute("SELECT ST_X('%s') AS lon" % source)[0]['lon']
+      origin = {'lat': lat, 'lon': lon}
+    else:
+      raise Exception('source is NULL')
+
+    # -- TODO Support options properly
+    isolines = {}
+    if isotype == 'isodistance':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isodistance(origin, mode, r)
+          isolines[r] = isoline
+    elif isotype == 'isochrone':
+      for r in data_range:
+          isoline = mapzen_isolines.calculate_isochrone(origin, mode, r)
+          isolines[r] = isoline
+
+    result = []
+    for r in data_range:
+
+      if len(isolines[r]) >= 3:
+        # -- TODO encapsulate this block into a func/method
+        locations = isolines[r] + [ isolines[r][0] ] # close the polygon repeating the first point
+        wkt_coordinates = ','.join(["%f %f" % (l['lon'], l['lat']) for l in locations])
+        sql = "SELECT ST_MPolyFromText('MULTIPOLYGON((({0})))', 4326) as geom".format(wkt_coordinates)
+        multipolygon = plpy.execute(sql, 1)[0]['geom']
+      else:
+        multipolygon = None
+
+      result.append([source, r, multipolygon])
+
+    quota_service.increment_success_service_use()
+    quota_service.increment_isolines_service_use(len(isolines))
+    return result
+  except BaseException as e:
+    import sys, traceback
+    type_, value_, traceback_ = sys.exc_info()
+    quota_service.increment_failed_service_use()
+    error_msg = 'There was an error trying to obtain isolines using mapzen: {0}'.format(e)
+    plpy.debug(traceback.format_tb(traceback_))
+    raise e
+    #plpy.error(error_msg)
+  finally:
+    quota_service.increment_total_service_use()
+$$ LANGUAGE plpythonu SECURITY DEFINER;

server/extension/sql/85_isodistance.sql

@@ -19,3 +19,18 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
 
   return isolines
 $$ LANGUAGE plpythonu;
+
+-- mapzen isodistance
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isodistance(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isodistance'
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu;

server/extension/sql/90_isochrone.sql

@@ -19,3 +19,19 @@ RETURNS SETOF cdb_dataservices_server.isoline AS $$
 
   return isolines
 $$ LANGUAGE plpythonu;
+
+
+-- mapzen isochrones
+CREATE OR REPLACE FUNCTION cdb_dataservices_server.cdb_mapzen_isochrone(username TEXT, orgname TEXT, source geometry(Geometry, 4326), mode TEXT, range integer[], options text[] DEFAULT array[]::text[])
+RETURNS SETOF cdb_dataservices_server.isoline AS $$
+  plpy.execute("SELECT cdb_dataservices_server._connect_to_redis('{0}')".format(username))
+  redis_conn = GD["redis_connection_{0}".format(username)]['redis_metrics_connection']
+  plpy.execute("SELECT cdb_dataservices_server._get_mapzen_isolines_config({0}, {1})".format(plpy.quote_nullable(username), plpy.quote_nullable(orgname)))
+  user_isolines_config = GD["user_mapzen_isolines_routing_config_{0}".format(username)]
+  type = 'isochrone'
+
+  mapzen_plan = plpy.prepare("SELECT * FROM cdb_dataservices_server._cdb_mapzen_isolines($1, $2, $3, $4, $5, $6, $7) as isoline; ", ["text", "text", "text", "geometry(Geometry, 4326)", "text", "integer[]", "text[]"])
+  result = plpy.execute(mapzen_plan, [username, orgname, type, source, mode, range, options])
+
+  return result
+$$ LANGUAGE plpythonu;

server/extension/test/expected/00_install_test.out

@@ -1,6 +1,5 @@
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
@@ -27,7 +26,7 @@ SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy
  
 (1 row)
 
-SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
  cdb_conf_setconf 
 ------------------
  

server/extension/test/expected/100_data_observatory_tables_test.out

@@ -0,0 +1,44 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_connectusertable'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text, text');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_getreturnmetadata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_fetchjoinfdwtabledata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text, json');
+ exists 
+--------
+ t
+(1 row)
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_disconnectusertable'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text');
+ exists 
+--------
+ t
+(1 row)
+

server/extension/test/sql/00_install_test.sql

@@ -1,6 +1,5 @@
 -- Install dependencies
 CREATE EXTENSION postgis;
-CREATE EXTENSION schema_triggers;
 CREATE EXTENSION plpythonu;
 CREATE EXTENSION plproxy;
 CREATE EXTENSION cartodb;
@@ -14,7 +13,7 @@ CREATE EXTENSION cdb_dataservices_server;
 SELECT cartodb.cdb_conf_setconf('redis_metrics_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
 SELECT cartodb.cdb_conf_setconf('redis_metadata_config', '{"redis_host": "localhost", "redis_port": 6379, "timeout": 0.1, "redis_db": 5}');
 SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
-SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
 SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/dev/null"}');
 SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');
 

server/extension/test/sql/100_data_observatory_tables_test.sql

@@ -0,0 +1,28 @@
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_connectusertable'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text, text');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_getreturnmetadata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, json');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_fetchjoinfdwtabledata'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text, json');
+
+SELECT exists(SELECT *
+              FROM pg_proc p
+              INNER JOIN pg_namespace ns ON (p.pronamespace = ns.oid)
+              WHERE ns.nspname = 'cdb_dataservices_server'
+              AND proname = '_obs_disconnectusertable'
+              AND oidvectortypes(p.proargtypes)  = 'text, text, text, text, text');
+

server/lib/python/cartodb_services/README.md

@@ -1,6 +1,6 @@
 # CartoDB dataservices API python module
 
-This directory contains the python library used by the server side of CartoDB LDS (Location Data Services).
+This directory contains the python library used by the server side of CARTO LDS (Location Data Services).
 
 It is used from pl/python functions contained in the `cdb_dataservices_server` extension. It goes hand in hand with the extension so please consider running the integration tests.
 

server/lib/python/cartodb_services/cartodb_services/mapzen/__init__.py

@@ -1,2 +1,4 @@
 from routing import MapzenRouting, MapzenRoutingResponse
+from isolines import MapzenIsolines
 from geocoder import MapzenGeocoder
+from matrix_client import MatrixClient

server/lib/python/cartodb_services/cartodb_services/mapzen/isolines.py

@@ -0,0 +1,160 @@
+from math import cos, sin, tan, sqrt, pi, radians, degrees, asin, atan2
+import logging
+
+class MapzenIsolines:
+
+    NUMBER_OF_ANGLES = 24
+    MAX_ITERS = 5
+    TOLERANCE = 0.1
+
+    EARTH_RADIUS_METERS = 6367444
+
+    def __init__(self, matrix_client):
+        self._matrix_client = matrix_client
+
+    """Get an isochrone using mapzen API.
+
+    The implementation tries to sick close to the SQL API:
+    cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline
+
+    But this calculates just one isoline.
+
+    Args:
+        origin dict containing {lat: y, lon: x}
+        transport_mode string, for the moment just "car" or "walk"
+        isorange int range of the isoline in seconds
+
+    Returns:
+        Array of {lon: x, lat: y} as a representation of the isoline
+    """
+    def calculate_isochrone(self, origin, transport_mode, time_range):
+        if transport_mode == 'walk':
+            max_speed = 3.3333333 # In m/s, assuming 12km/h walking speed
+            costing_model = 'pedestrian'
+        elif transport_mode == 'car':
+            max_speed = 41.67 # In m/s, assuming 140km/h max speed
+            costing_model = 'auto'
+        else:
+            raise NotImplementedError('car and walk are the only supported modes for the moment')
+
+        upper_rmax = max_speed * time_range # an upper bound for the radius
+
+        return self.calculate_isoline(origin, costing_model, time_range, upper_rmax, 'time')
+
+    """Get an isodistance using mapzen API.
+
+    Args:
+        origin dict containing {lat: y, lon: x}
+        transport_mode string, for the moment just "car" or "walk"
+        isorange int range of the isoline in seconds
+
+    Returns:
+        Array of {lon: x, lat: y} as a representation of the isoline
+    """
+    def calculate_isodistance(self, origin, transport_mode, distance_range):
+        if transport_mode == 'walk':
+            costing_model = 'pedestrian'
+        elif transport_mode == 'car':
+            costing_model = 'auto'
+        else:
+            raise NotImplementedError('car and walk are the only supported modes for the moment')
+
+        upper_rmax = distance_range # an upper bound for the radius, going in a straight line
+
+        return self.calculate_isoline(origin, costing_model, distance_range, upper_rmax, 'distance', 1000.0)
+
+
+    """Get an isoline using mapzen API.
+
+    The implementation tries to sick close to the SQL API:
+    cdb_isochrone(source geometry, mode text, range integer[], [options text[]]) -> SETOF isoline
+
+    But this calculates just one isoline.
+
+    Args:
+        origin dict containing {lat: y, lon: x}
+        costing_model string "auto" or "pedestrian"
+        isorange int Range of the isoline in seconds
+        upper_rmax float An upper bound for the binary search
+        cost_variable string Variable to optimize "time" or "distance"
+        unit_factor float A factor to adapt units of isorange (meters) and units of distance (km)
+
+    Returns:
+        Array of {lon: x, lat: y} as a representation of the isoline
+    """
+    def calculate_isoline(self, origin, costing_model, isorange, upper_rmax, cost_variable, unit_factor=1.0):
+
+        # NOTE: not for production
+        #logging.basicConfig(level=logging.DEBUG, filename='/tmp/isolines.log')
+        #logging.basicConfig(level=logging.DEBUG)
+        logging.debug('origin = %s' % origin)
+        logging.debug('costing_model = %s' % costing_model)
+        logging.debug('isorange = %d' % isorange)
+
+        # Formally, a solution is an array of {angle, radius, lat, lon, cost} with cardinality NUMBER_OF_ANGLES
+        # we're looking for a solution in which abs(cost - isorange) / isorange <= TOLERANCE
+
+        # Initial setup
+        angles = self._get_angles(self.NUMBER_OF_ANGLES) # array of angles
+        rmax = [upper_rmax] * self.NUMBER_OF_ANGLES
+        rmin = [0.0] * self.NUMBER_OF_ANGLES
+        location_estimates = [self._calculate_dest_location(origin, a, upper_rmax / 2.0) for a in angles]
+
+        # Iterate to refine the first solution
+        for i in xrange(0, self.MAX_ITERS):
+            # Calculate the "actual" cost for each location estimate.
+            # NOTE: sometimes it cannot calculate the cost and returns None.
+            #   Just assume isorange and stop the calculations there
+
+            response = self._matrix_client.one_to_many([origin] + location_estimates,  costing_model)
+            costs = [None] * self.NUMBER_OF_ANGLES
+            for idx, c in enumerate(response['one_to_many'][0][1:]):
+                if c[cost_variable]:
+                    costs[idx] = c[cost_variable]*unit_factor
+                else:
+                    costs[idx] = isorange
+
+            logging.debug('i = %d, costs = %s' % (i, costs))
+
+            errors = [(cost - isorange) / float(isorange) for cost in costs]
+            max_abs_error = max([abs(e) for e in errors])
+            if max_abs_error <= self.TOLERANCE:
+                # good enough, stop there
+                break
+
+            # let's refine the solution, binary search
+            for j in xrange(0, self.NUMBER_OF_ANGLES):
+
+                if abs(errors[j]) > self.TOLERANCE:
+                    if errors[j] > 0:
+                        rmax[j] = (rmax[j] + rmin[j]) / 2.0
+                    else:
+                        rmin[j] = (rmax[j] + rmin[j]) / 2.0
+
+                    location_estimates[j] = self._calculate_dest_location(origin, angles[j], (rmax[j]+rmin[j])/2.0)
+
+        # delete points that got None
+        location_estimates_filtered = []
+        for i, c in enumerate(costs):
+            if c <> isorange:
+                location_estimates_filtered.append(location_estimates[i])
+
+        return location_estimates_filtered
+
+
+
+    # NOTE: all angles in calculations are in radians
+    def _get_angles(self, number_of_angles):
+        step = (2.0 * pi) / number_of_angles
+        return [(x * step) for x in xrange(0, number_of_angles)]
+
+    def _calculate_dest_location(self, origin, angle, radius):
+        origin_lat_radians = radians(origin['lat'])
+        origin_long_radians = radians(origin['lon'])
+        dest_lat_radians = asin(sin(origin_lat_radians) * cos(radius / self.EARTH_RADIUS_METERS) + cos(origin_lat_radians) * sin(radius / self.EARTH_RADIUS_METERS) * cos(angle))
+        dest_lng_radians = origin_long_radians + atan2(sin(angle) * sin(radius / self.EARTH_RADIUS_METERS) * cos(origin_lat_radians), cos(radius / self.EARTH_RADIUS_METERS) - sin(origin_lat_radians) * sin(dest_lat_radians))
+
+        return {
+            'lon': degrees(dest_lng_radians),
+            'lat': degrees(dest_lat_radians)
+        }

server/lib/python/cartodb_services/cartodb_services/mapzen/matrix_client.py

@@ -0,0 +1,43 @@
+import requests
+import json
+
+class MatrixClient:
+
+    """
+    A minimal client for Mapzen Time-Distance Matrix Service
+
+    Example:
+
+    client = MatrixClient('your_api_key')
+    locations = [{"lat":40.744014,"lon":-73.990508},{"lat":40.739735,"lon":-73.979713},{"lat":40.752522,"lon":-73.985015},{"lat":40.750117,"lon":-73.983704},{"lat":40.750552,"lon":-73.993519}]
+    costing = 'pedestrian'
+
+    print client.one_to_many(locations, costing)
+    """
+
+    ONE_TO_MANY_URL = 'https://matrix.mapzen.com/one_to_many'
+
+    def __init__(self, matrix_key):
+        self._matrix_key = matrix_key
+
+    """Get distances and times to a set of locations.
+    See https://mapzen.com/documentation/matrix/api-reference/
+
+    Args:
+        locations Array of {lat: y, lon: x}
+        costing Costing model to use
+
+    Returns:
+        A dict with one_to_many, units and locations
+    """
+    def one_to_many(self, locations, costing):
+        request_params = {
+            'json': json.dumps({'locations': locations}),
+            'costing': costing,
+            'api_key': self._matrix_key
+        }
+        response = requests.get(self.ONE_TO_MANY_URL, params=request_params)
+
+        response.raise_for_status() # raise exception if not 200 OK
+
+        return response.json()

server/lib/python/cartodb_services/cartodb_services/metrics/__init__.py

@@ -1,3 +1,3 @@
-from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
+from config import GeocoderConfig, MapzenGeocoderConfig, IsolinesRoutingConfig, MapzenIsolinesRoutingConfig, InternalGeocoderConfig, RoutingConfig, ConfigException, ObservatorySnapshotConfig, ObservatoryConfig
 from quota import QuotaService
 from user import UserMetricsService

server/lib/python/cartodb_services/cartodb_services/metrics/config.py

@@ -144,7 +144,7 @@ class MapzenGeocoderConfig(ServiceConfig):
             self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
             self._cost_per_hit = 0
         except Exception as e:
-            raise ConfigException("Malformed config for Mapzen geocoder: {1}".format(key, e))
+            raise ConfigException("Malformed config for Mapzen geocoder: {0}".format(e))
 
     @property
     def service_type(self):
@@ -241,6 +241,40 @@ class IsolinesRoutingConfig(ServiceConfig):
         return self._geocoder_type == self.GOOGLE_GEOCODER
 
 
+class MapzenIsolinesRoutingConfig(ServiceConfig):
+
+    PERIOD_END_DATE = 'period_end_date'
+
+    def __init__(self, redis_connection, db_conn, username, orgname=None):
+        super(MapzenIsolinesRoutingConfig, self).__init__(redis_connection, db_conn,
+                                                    username, orgname)
+        try:
+            self._mapzen_matrix_api_key = self._db_config.mapzen_matrix_api_key
+            self._isolines_quota = self._db_config.mapzen_matrix_monthly_quota
+            self._period_end_date = date_parse(self._redis_config[self.PERIOD_END_DATE])
+        except Exception as e:
+            raise ConfigException("Malformed config for Mapzen isolines: {0}".format(e))
+
+    @property
+    def service_type(self):
+        return 'mapzen_isolines'
+
+    @property
+    def isolines_quota(self):
+        return self._isolines_quota
+
+    @property
+    def soft_isolines_limit(self):
+        return False
+
+    @property
+    def period_end_date(self):
+        return self._period_end_date
+
+    @property
+    def mapzen_matrix_api_key(self):
+        return self._mapzen_matrix_api_key
+
 class InternalGeocoderConfig(ServiceConfig):
 
     def __init__(self, redis_connection, db_conn, username, orgname=None):
@@ -438,6 +472,8 @@ class ServicesDBConfig:
             raise ConfigException('Mapzen configuration missing')
         else:
             mapzen_conf = json.loads(mapzen_conf_json)
+            self._mapzen_matrix_api_key = mapzen_conf['matrix']['api_key']
+            self._mapzen_matrix_quota = mapzen_conf['matrix']['monthly_quota']
             self._mapzen_routing_api_key = mapzen_conf['routing']['api_key']
             self._mapzen_routing_quota = mapzen_conf['routing']['monthly_quota']
             self._mapzen_geocoder_api_key = mapzen_conf['geocoder']['api_key']
@@ -492,6 +528,14 @@ class ServicesDBConfig:
     def heremaps_geocoder_cost_per_hit(self):
         return self._heremaps_geocoder_cost_per_hit
 
+    @property
+    def mapzen_matrix_api_key(self):
+        return self._mapzen_matrix_api_key
+
+    @property
+    def mapzen_matrix_monthly_quota(self):
+        return self._mapzen_matrix_quota
+
     @property
     def mapzen_routing_api_key(self):
         return self._mapzen_routing_api_key

server/lib/python/cartodb_services/cartodb_services/metrics/quota.py

@@ -70,6 +70,9 @@ class QuotaChecker:
         elif re.match('here_isolines',
                       self._user_service_config.service_type) is not None:
             return self.__check_isolines_quota()
+        elif re.match('mapzen_isolines',
+                      self._user_service_config.service_type) is not None:
+            return self.__check_isolines_quota()
         elif re.match('routing_mapzen',
                       self._user_service_config.service_type) is not None:
             return self.__check_routing_quota()

server/lib/python/cartodb_services/setup.py

@@ -10,7 +10,7 @@ from setuptools import setup, find_packages
 setup(
     name='cartodb_services',
 
-    version='0.6.5',
+    version='0.7.0',
 
     description='CartoDB Services API Python Library',
 

server/lib/python/cartodb_services/test/test_helper.py

@@ -65,7 +65,7 @@ def _plpy_execute_side_effect(*args, **kwargs):
     if args[0] == "SELECT cartodb.CDB_Conf_GetConf('heremaps_conf') as conf":
         return [{'conf': '{"geocoder": {"app_id": "app_id", "app_code": "code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "app_id", "app_code": "code"}}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('mapzen_conf') as conf":
-        return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}}'}]
+        return [{'conf': '{"routing": {"api_key": "api_key_rou", "monthly_quota": 1500000}, "geocoder": {"api_key": "api_key_geo", "monthly_quota": 1500000}, "matrix": {"api_key": "api_key_mat", "monthly_quota": 1500000}}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('logger_conf') as conf":
         return [{'conf': '{"geocoder_log_path": "/dev/null"}'}]
     elif args[0] == "SELECT cartodb.CDB_Conf_GetConf('data_observatory_conf') as conf":

server/lib/python/cartodb_services/test/test_mapzengeocoder.py

@@ -11,7 +11,7 @@ requests_mock.Mocker.TEST_PREFIX = 'test_'
 
 
 @requests_mock.Mocker()
-class GoogleGeocoderTestCase(unittest.TestCase):
+class MapzenGeocoderTestCase(unittest.TestCase):
     MAPZEN_GEOCODER_URL = 'https://search.mapzen.com/v1/search'
 
     EMPTY_RESPONSE = """{

server/lib/python/cartodb_services/test/test_mapzenisolines.py

@@ -0,0 +1,76 @@
+import unittest
+from cartodb_services.mapzen import MapzenIsolines
+from math import radians, cos, sin, asin, sqrt
+
+"""
+This file is basically a sanity test on the algorithm.
+
+
+It uses a mocked client, which returns the cost based on a very simple model:
+just proportional to the distance from origin to the target point.
+"""
+
+class MatrixClientMock():
+
+    def __init__(self, speed):
+        """
+        Sets up the mock with a speed in km/h
+        """
+        self._speed = speed
+
+    def one_to_many(self, locations, costing):
+        origin = locations[0]
+        distances = [self._distance(origin, l) for l in locations]
+        response = {
+            'one_to_many': [
+                [
+                    {
+                        'distance': distances[i] * self._speed,
+                        'time': distances[i] / self._speed * 3600,
+                        'to_index': i,
+                        'from_index': 0
+                    }
+                    for i in xrange(0, len(distances))
+                ]
+            ],
+            'units': 'km',
+            'locations': [
+                locations
+            ]
+        }
+        return response
+
+    def _distance(self, a, b):
+        """
+        Calculate the great circle distance between two points
+        on the earth (specified in decimal degrees)
+        http://stackoverflow.com/questions/4913349/haversine-formula-in-python-bearing-and-distance-between-two-gps-points
+
+        Returns:
+            distance in meters
+        """
+
+        # convert decimal degrees to radians
+        lon1, lat1, lon2, lat2 = map(radians, [a['lon'], a['lat'], b['lon'], b['lat']])
+
+        # haversine formula
+        dlon = lon2 - lon1
+        dlat = lat2 - lat1
+        a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2
+        c = 2 * asin(sqrt(a))
+        r = 6371 # Radius of earth in kilometers. Use 3956 for miles
+        return c * r
+
+
+class MapzenIsolinesTestCase(unittest.TestCase):
+
+    def setUp(self):
+        speed = 4 # in km/h
+        matrix_client = MatrixClientMock(speed)
+        self.mapzen_isolines = MapzenIsolines(matrix_client)
+
+    def test_calculate_isochrone(self):
+        origin = {"lat":40.744014,"lon":-73.990508}
+        transport_mode = 'walk'
+        isorange = 10 * 60 # 10 minutes
+        solution = self.mapzen_isolines.calculate_isochrone(origin, transport_mode, isorange)