Add new params at the end #361

Changes in TYPEs obs_meta_geometry and obs_meta_timespan

NEWS.md

@@ -1,3 +1,10 @@
+May 9th, 2017
+=============
+* Version `0.17.0` of the client and version `0.24.0` of the server
+    * Fixed some missing return values documented but not present.
+        *  `OBS_GetAvailableGeometries` now returns `geom_type`, `geom_extra` and `geom_tags` in addition to existing values.
+        *  `OBS_GetAvailableTimespans` now returns `timespan_type`, `timespan_extra`, `timespan_tags` in addition to existing values.
+
 March 28th, 2017
 ================
 

README.md

@@ -23,26 +23,12 @@ Steps to deploy a new Data Services API version :
 
 ### Local install instructions
 
-- install data services geocoder extension
-
-    ```
-    git clone https://github.com/CartoDB/data-services.git
-    cd data-services/geocoder/extension
-    sudo make install
-    ```
-
-- install observatory extension
-
-    ```
-    git clone https://github.com/CartoDB/observatory-extension.git
-    cd observatory
-    sudo make install
-    ```
-
 - install server and client extensions
 
     ```
-    # in dataservices-api repo root path:
+    # in your workspace root path
+    git clone https://github.com/CartoDB/dataservices-api.git
+    cd dataservices-api
     cd client && sudo make install
     cd -
     cd server/extension && sudo make install
@@ -55,16 +41,64 @@ Steps to deploy a new Data Services API version :
     cd server/lib/python/cartodb_services && pip install -r requirements.txt && sudo pip install . --upgrade
     ```
 
-- install extensions in user database
+- Create a database to hold all the server part and a user for it
+
+  ```sql
+  CREATE DATABASE dataservices_db ENCODING = 'UTF8' LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8';
+  CREATE USER dataservices_user;
+  ```
+
+- Install needed extensions in `dataservices_db` database
 
     ```
-    create extension cdb_geocoder;
+    psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS plproxy; COMMIT" -e
-    create extension plproxy;
+    psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_dataservices_server; COMMIT" -e
-    create extension observatory;
-    create extension cdb_dataservices_server;
-    create extension cdb_dataservices_client;
     ```
 
+- [optional] install internal geocoder
+   - Make the extension available in postgres
+     ```
+     git clone https://github.com/CartoDB/data-services.git
+     cd data-services/geocoder/extension
+     sudo make install
+     ```
+
+   - Make sure you have `wget` installed because is needed for the next step.
+
+   - Go to `geocoder` folder and execute the `geocoder_dowload_dumps` script to download the internal geocoder data.
+
+   - Once the data is downloaded, execute this command:
+     ```bash
+     geocoder_restore_dump postgres dataservices_db {DOWNLOADED_DUMPS_FOLDER}/*.sql
+     ```
+
+   - Now we have to make available the extension to be installed by postgres. Follow [this](https://github.com/CartoDB/data-services/tree/master/geocoder/extension) instructions.
+
+   - Now install the extension with:
+     ```
+     psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS cdb_geocoder; COMMIT" -e
+     ```
+
+- [optional] install data observatory extension
+   - Make the extension available in postgresql to be installed
+     ```
+     git clone https://github.com/CartoDB/observatory-extension.git
+     cd observatory
+     sudo make install
+     ```
+   - This extension needs data, dumps are not available so we're going to use the test fixtures to make it work. Execute:
+     ```
+     psql -U postgres -d dataservices_db -f src/pg/test/fixtures/load_fixtures.sql
+     ```
+   - Give permission to execute and select to the `dataservices_user` user:
+     ```
+     psql -U postgres -d dataservices_db -c "BEGIN;CREATE EXTENSION IF NOT EXISTS observatoru; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA cdb_observatory TO dataservices_user; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT SELECT ON ALL TABLES IN SCHEMA observatory TO dataservices_user; COMMIT" -e
+     psql -U postgres -d dataservices_db -c "BEGIN;GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA observatory TO dataservices_user; COMMIT" -e
+     ```
+
 ### Server configuration
 
 Configuration for the different services must be stored in the server database using `CDB_Conf_SetConf()`.

client/cdb_dataservices_client--0.16.0--0.17.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.17.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry ADD ATTRIBUTE geom_type text;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry ADD ATTRIBUTE geom_extra jsonb;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry ADD ATTRIBUTE geom_tags jsonb;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan ADD ATTRIBUTE timespan_type text;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan ADD ATTRIBUTE timespan_extra jsonb;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan ADD ATTRIBUTE timespan_tags jsonb;

client/cdb_dataservices_client--0.17.0--0.16.0.sql

@@ -0,0 +1,14 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_client UPDATE TO '0.16.0'" to load this file. \quit
+
+-- Make sure we have a sane search path to create/update the extension
+SET search_path = "$user",cartodb,public,cdb_dataservices_client;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry DROP ATTRIBUTE geom_type;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry DROP ATTRIBUTE geom_extra;
+ALTER TYPE cdb_dataservices_client.obs_meta_geometry DROP ATTRIBUTE geom_tags;
+
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan DROP ATTRIBUTE timespan_type;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan DROP ATTRIBUTE timespan_extra;
+ALTER TYPE cdb_dataservices_client.obs_meta_timespan DROP ATTRIBUTE timespan_tags;

client/cdb_dataservices_client.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices client API extension'
-default_version = '0.16.0'
+default_version = '0.17.0'
 requires = 'plproxy, cartodb'
 superuser = true
 schema = cdb_dataservices_client

client/sql/16_custom_types.sql

@@ -15,9 +15,9 @@ CREATE TYPE cdb_dataservices_client.obs_meta_numerator AS (numer_id text, numer_
 
 CREATE TYPE cdb_dataservices_client.obs_meta_denominator AS (denom_id text, denom_name text, denom_description text, denom_weight text, denom_license text, denom_source text, denom_type text, denom_aggregate text, denom_extra jsonb, denom_tags jsonb, valid_numer boolean, valid_geom boolean, valid_timespan boolean);
 
-CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric);
+CREATE TYPE cdb_dataservices_client.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
 
-CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean);
+CREATE TYPE cdb_dataservices_client.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
 
 
 -- For quotas and services configuration

doc/rate_limits.md

@@ -83,7 +83,7 @@ The result is a JSON object with the configuration (`period` and `limit` attribu
 #### Example:
 
 ```
-SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoding');
+SELECT cdb_dataservices_client.cdb_service_get_rate_limit('geocoder');
 
    cdb_service_get_rate_limit
 ---------------------------------
@@ -108,16 +108,18 @@ This functions doesn't return any value.
 
 #### Example
 
-This will configure the geocoding service rate limit for  user  `myusername`, a non-organization user.
+This will configure the geocoder service rate limit for  user  `myusername`, a non-organization user.
 The limit will be set at 1000 requests per day. Since the user doesn't belong to any organization,
 `NULL` will be passed to the organization argument; otherwise the name of the user's organization should
 be provided.
 
+Note that the name of the geocoding services is `geocoder` and not `geocoding`.
+
 ```
 SELECT cdb_dataservices_client.cdb_service_set_user_rate_limit(
     'myusername',
     NULL,
-    'geocoding',
+    'geocoder',
     '{"limit":1000,"period":86400}'
 );
 
@@ -140,7 +142,7 @@ This functions doesn't return any value.
 
 #### Example
 
-This will configure the geocoding service rate limit for the `myorg` organization.
+This will configure the geocoder service rate limit for the `myorg` organization.
 The limit will be set at 100 requests per hour.
 Note that even we're setting the default configuration for the whole organization,
 the name of a user of the organization must be provided for technical reasons.
@@ -149,7 +151,7 @@ the name of a user of the organization must be provided for technical reasons.
 SELECT cdb_dataservices_client.cdb_service_set_org_rate_limit(
     'myorgadmin',
     'myorg',
-    'geocoding',
+    'geocoder',
     '{"limit":100,"period":3600}'
 );
 
@@ -172,7 +174,7 @@ This functions doesn't return any value.
 
 #### Example
 
-This will configure the default geocoding service rate limit for all users
+This will configure the default geocoder service rate limit for all users
 accesing the data-services server.
 The limit will be set at 10000 requests per month.
 Note that even we're setting the default configuration for the server,
@@ -183,7 +185,7 @@ must be provided for technical reasons.
 SELECT cdb_dataservices_client.cdb_service_set_server_rate_limit(
     'myorgadmin',
     'myorg',
-    'geocoding',
+    'geocoder',
     '{"limit":10000,"period":108000}'
 );
 

server/extension/cdb_dataservices_server--0.23.0--0.24.0.sql

@@ -0,0 +1,11 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.24.0'" to load this file. \quit
+
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry ADD ATTRIBUTE geom_type text;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry ADD ATTRIBUTE geom_extra jsonb;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry ADD ATTRIBUTE geom_tags jsonb;
+
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan ADD ATTRIBUTE timespan_type text;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan ADD ATTRIBUTE timespan_extra jsonb;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan ADD ATTRIBUTE timespan_tags jsonb;

server/extension/cdb_dataservices_server--0.24.0--0.23.0.sql

@@ -0,0 +1,12 @@
+--DO NOT MODIFY THIS FILE, IT IS GENERATED AUTOMATICALLY FROM SOURCES
+-- Complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "ALTER EXTENSION cdb_dataservices_server UPDATE TO '0.23.0'" to load this file. \quit
+
+-- HERE goes your code to upgrade/downgrade
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry DROP ATTRIBUTE geom_type;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry DROP ATTRIBUTE geom_extra;
+ALTER TYPE cdb_dataservices_server.obs_meta_geometry DROP ATTRIBUTE geom_tags;
+
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan DROP ATTRIBUTE timespan_type;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan DROP ATTRIBUTE timespan_extra;
+ALTER TYPE cdb_dataservices_server.obs_meta_timespan DROP ATTRIBUTE timespan_tags;

server/extension/cdb_dataservices_server.control

@@ -1,5 +1,5 @@
 comment = 'CartoDB dataservices server extension'
-default_version = '0.23.0'
+default_version = '0.24.0'
 requires = 'plpythonu, plproxy, postgis, cdb_geocoder'
 superuser = true
 schema = cdb_dataservices_server

server/extension/sql/130_data_observatory_helper.sql

@@ -35,7 +35,7 @@ RETURNS SETOF cdb_dataservices_server.obs_meta_denominator AS $$
   SELECT * FROM cdb_observatory.OBS_GetAvailableDenominators(bounds, filter_tags, numer_id, geom_id, timespan);
 $$ LANGUAGE plproxy;
 
-CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric);
+CREATE TYPE cdb_dataservices_server.obs_meta_geometry AS (geom_id text, geom_name text, geom_description text, geom_weight text, geom_aggregate text, geom_license text, geom_source text, valid_numer boolean, valid_denom boolean, valid_timespan boolean, score numeric, numtiles bigint, notnull_percent numeric, numgeoms numeric, percentfill numeric, estnumgeoms numeric, meanmediansize numeric, geom_type text, geom_extra jsonb, geom_tags jsonb);
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableGeometries(
   username TEXT,
@@ -50,7 +50,7 @@ RETURNS SETOF cdb_dataservices_server.obs_meta_geometry AS $$
   SELECT * FROM cdb_observatory.OBS_GetAvailableGeometries(bounds, filter_tags, numer_id, denom_id, timespan);
 $$ LANGUAGE plproxy;
 
-CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean);
+CREATE TYPE cdb_dataservices_server.obs_meta_timespan AS (timespan_id text, timespan_name text, timespan_description text, timespan_weight text, timespan_aggregate text, timespan_license text, timespan_source text, valid_numer boolean, valid_denom boolean, valid_geom boolean, timespan_type text, timespan_extra jsonb, timespan_tags jsonb);
 
 CREATE OR REPLACE FUNCTION cdb_dataservices_server.OBS_GetAvailableTimespans(
   username TEXT,