Release version 0.19.2

305 fix unique identifiers

.travis.yml

@@ -1,14 +1,41 @@
 language: c
 
 addons:
-  postgresql: 9.3
+  postgresql: 9.5
 
 before_install:
+  # Add custom PPAs from cartodb
+  - sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
+  - sudo add-apt-repository -y ppa:cartodb/gis
+  - sudo add-apt-repository -y ppa:cartodb/gis-testing
   - sudo apt-get update
-  #- sudo apt-get install -q postgresql-9.3-postgis-2.1
-  - sudo apt-get update
-  - sudo apt-get install -q postgresql-server-dev-9.3
-  - sudo apt-get install -q postgresql-plpython-9.3
+
+  # Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis)
+  - sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2
+
+  # Install postgres db and build deps
+  - sudo /etc/init.d/postgresql stop # stop travis default instance
+  - sudo apt-get -y remove --purge postgresql-9.1
+  - sudo apt-get -y remove --purge postgresql-9.2
+  - sudo apt-get -y remove --purge postgresql-9.3
+  - sudo apt-get -y remove --purge postgresql-9.4
+  - sudo apt-get -y remove --purge postgresql-9.5
+  - sudo rm -rf /var/lib/postgresql/
+  - sudo rm -rf /var/log/postgresql/
+  - sudo rm -rf /etc/postgresql/
+  - sudo apt-get -y remove --purge postgis-2.2
+  - sudo apt-get -y autoremove
+
+  - sudo apt-get -y install postgresql-9.5=9.5.2-3cdb3
+  - sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb3
+  - sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb3
+  - sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
+  - sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
+
+  # configure it to accept local connections from postgres
+  - echo -e "# TYPE  DATABASE        USER            ADDRESS                 METHOD \nlocal   all             postgres                                trust\nlocal   all             all                                     trust\nhost    all             all             127.0.0.1/32            trust" \
+    | sudo tee /etc/postgresql/9.5/main/pg_hba.conf
+  - sudo /etc/init.d/postgresql restart 9.5
 
 script:
   - make

CONTRIBUTING.md

@@ -62,3 +62,7 @@ A useful query:
 ```sql
 SELECT * FROM pg_extension_update_paths('cartodb') WHERE path IS NOT NULL AND source = cdb_version();
 ```
+
+## Submitting Contributions
+
+* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.14.4
+EXTVERSION = 0.19.2
 
 SED = sed
 
@@ -63,7 +63,25 @@ UPGRADABLE = \
   0.14.1 \
   0.14.2 \
   0.14.3 \
-	0.14.4 \
+  0.14.4 \
+  0.15.0 \
+  0.15.1 \
+  0.16.0 \
+  0.16.1 \
+  0.16.2 \
+  0.16.3 \
+  0.16.4 \
+  0.17.0 \
+  0.17.1 \
+  0.18.0 \
+  0.18.1 \
+  0.18.2 \
+  0.18.3 \
+  0.18.4 \
+  0.18.5 \
+  0.19.0 \
+  0.19.1 \
+  0.19.2 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -93,7 +111,7 @@ PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
 include $(PGXS)
 
-$(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile 
+$(EXTENSION)--$(EXTVERSION).sql: $(CDBSCRIPTS) cartodb_version.sql Makefile
 	echo '\echo Use "CREATE EXTENSION $(EXTENSION)" to load this file. \quit' > $@
 	cat $(CDBSCRIPTS) | \
     $(SED) -e 's/public\./cartodb./g' \
@@ -144,7 +162,7 @@ test_organization:
 test_extension_new:
 	bash test/extension/test.sh
 
-legacy_tests: legacy_regress 
+legacy_tests: legacy_regress
 
 installcheck: legacy_tests test_extension_new test_organization
 

NEWS.md

@@ -1,4 +1,132 @@
+0.19.2 (2017-06-30)
+* Improved functions to generate unique identifiers #305
+
+0.19.1 (2017-06-05)
+
+* Fixed a deadlock problem when trying to regenarate overviews #302
+
+0.19.0 (2017-04-11)
+
+* Add new function `CDB_EstimateRowCount` #295
+
+0.18.5 (2016-11-30)
+
+* Add to new overview creation strategies #290
+* Fix tests: race condition with publicuser #157
+* Fix: CDB_Stats divisions by zero #181
+* Better implementation of `CDB_EqualIntervalBins` #244
+* New tests for binning functions #249
+
+0.18.4 (2016-11-04)
+
+* No functional changes; fixes the migration from previous versions #288
+
+0.18.3 (2016-11-03)
+
+* Exclude analysis cache tables from the quota #281
+
+0.18.2 (2016-10-20)
+-------------------
+
+* Fix: cleanup inconsistent position of `username` column in analysis catalog after upgrades
+  [#285](https://github.com/cartodb/cartodb-postgresql/pull/285)
+
+0.18.1 (2016-10-19)
+-------------------
+
+* Increase analysis limit factor to 2 [#284](https://github.com/CartoDB/cartodb-postgresql/pull/284)
+
+0.18.0 (2016-10-17)
+-------------------
+
+* Fix: exclude NULL geometries when creating Overviews #269
+* Function to check analysis tables limits #279
+
+0.17.1 (2016-08-16)
+-------------------
+
+* Add cache_tables column to cdb_analysis_catalog table #274.
+
+
+0.17.0 (2016-07-04)
+-------------------
+
+* Add export config for cdb_analysis_catalog table #268.
+* Add some extra fields to cdb_analysis_catalog table. Track user, error_message for failures, and last entity modifying the node #267.
+* Exclude overviews from user data size #262.
+
+
+0.16.4 (2016-05-27)
+-------------------
+
+* Change CDB_ZoomFromScale() to use a formula and raise
+  maximum overview level from 23 to 29.
+  [#259](https://github.com/CartoDB/cartodb-postgresql/pull/259)
+
+* Fix bug in overview creating causing it to fail when `x` or
+  `y` columns exist with non-integer type. Prevent also
+  potential integer overflows limiting maximum overview level
+  to 23.
+  [#258](https://github.com/CartoDB/cartodb-postgresql/pull/258)
+
+
+0.16.3 (2016-05-09)
+-------------------
+
+* Fix overview creation problem for organization users
+  with names that require quoting:
+  [#253](https://github.com/CartoDB/cartodb-postgresql/pull/253)
+
+0.16.2 (2016-04-27)
+-------------------
+
+* Use the mode to aggregate category columns in overviews
+  [#246](https://github.com/CartoDB/cartodb-postgresql/pull/246)
+
+0.16.1 (2016-04-25)
+-------------------
+
+* Optimize column information functions performance
+  [#238](https://github.com/CartoDB/cartodb-postgresql/pull/238)
+
+* Adjust overview points to pixel CDB_EqualIntervalBins
+  [#242](https://github.com/CartoDB/cartodb-postgresql/pull/242)
+
+* Compute webmercator resolution using full numeric precision
+  [#243](https://github.com/CartoDB/cartodb-postgresql/pull/243)
+
+
+0.16.0 (2016-04-15)
+-------------------
+* Adds table for storing camshaft analysis nodes
+  [#237](https://github.com/CartoDB/cartodb-postgresql/pull/237)
+
+0.15.1 (2016-04-15)
+-------------------
+* Fix problems with org users in overviews functions
+  [#224](https://github.com/CartoDB/cartodb-postgresql/pull/224)
+* Add `_feature_count` to overviews
+  [#227](https://github.com/CartoDB/cartodb-postgresql/pull/227)
+* Change point clustering behaviour of overviews
+  [#228](https://github.com/CartoDB/cartodb-postgresql/pull/228)
+* Change default tolerance of overviews
+  [#230](https://github.com/CartoDB/cartodb-postgresql/pull/230)
+* Fix problem with aggregated numerical fields in overviews
+  [#233](https://github.com/CartoDB/cartodb-postgresql/pull/233)
+* Enhance aggregation of text fields in overviews
+  [#234]https://github.com/CartoDB/cartodb-postgresql/pull/234
+
+0.15.0 (2016-04-05)
+-------------------
+* New function CDB_CreateOverviewsWithToleranceInPixels that adds tolerance parameter for overview creation
+  [#221](https://github.com/CartoDB/cartodb-postgresql/pull/221)
+* New default value for the overviews tolerance in pixels is 2 (used to be 7.5) (also in #221)
+* The feature density limit used to choose the reference Z level now depends on the tolerance in pixels (also in #221)
+* Tables that require an explicit schema can now be passed to overview functions
+  [#220](https://github.com/CartoDB/cartodb-postgresql/pull/220)
+
 0.14.4 (2016-03-29)
+-------------------
 * Fix creating overviews for tables with boolean columns
   [#214](https://github.com/CartoDB/cartodb-postgresql/pull/214)
 * Fix tests for some systems [#215](https://github.com/CartoDB/cartodb-postgresql/pull/215)

README.md

@@ -1,8 +1,7 @@
 cartodb-postgresql
 ==================
 
-[![Build Status](http://api.travis-ci.org/CartoDB/cartodb-postgresql.svg?branch=master)]
-(http://travis-ci.org/CartoDB/cartodb-postgresql)
+[![Build Status](http://api.travis-ci.org/CartoDB/cartodb-postgresql.svg?branch=master)](http://travis-ci.org/CartoDB/cartodb-postgresql)
 
 PostgreSQL extension for CartoDB
 

doc/CDB_EstimateRowCount.md

@@ -0,0 +1,25 @@
+Estimate the number of rows of a query.
+
+
+#### Using the function
+
+```sql
+SELECT CDB_EstimateRowCount($$
+  UPDATE addresses SET the_geom = cdb_geocode_street_point(addr, city, state, 'US');
+$$) AS row_count;
+```
+
+Result:
+
+```
+ row_count
+-----------
+         5
+(1 row)
+```
+
+#### Arguments
+
+CDB_EstimateRowCount(query)
+
+* **query** text: the SQL query to estimate the row count for.

doc/CDB_Overviews.md

@@ -2,18 +2,25 @@ Overviews are tables that represent a *reduced* version of a dataset intended
 for efficient rendering at certain zoom levels while preserving the
 general visual appearance of the complete dataset.
 
-The *reduction* consists in a fewer number of records
+The *reduction* consists in havig a fewer number of records
 (while each overview record may represent an aggregation of multiple records)
 and/or simplified record geometries.
 
-Overviews are created through the `CDB_CreateOverviews`.
+Overviews are created through the `CDB_CreateOverviews` function.
 The statement timeout may need to be adjusted before using this function,
 as overview creation for large tables is a time-consuming operation.
 
 The `CDB_Overviews` function can be used determine what overview tables
 exist for a given dataset table and which zoom levels correspond to it.
 
-The `CDB_DropOverviews` remove a dataset's existing overviews.
+The `CDB_DropOverviews` function removes a dataset's existing overviews.
+
+To know if overview tables exist for some base table, and to obtain
+a list of which overview tables are approrpiate for which zoom levels,
+the `CDB_Overviews` functions can be used.
+
+The zoom level we're referring here to are those used
+by the tiler: http://wiki.openstreetmap.org/wiki/Zoom_levels
 
 ### CDB_CreateOverviews
 
@@ -48,6 +55,17 @@ CDB_CreateOverviews(table_name, ref_z_strategy, reduction_strategy)
   - **base_z** integer, base Z level assigned to the base table.
   - **overview_z** integer, Z level for which to generate the overview.
 
+#### Tolerance / level of detail
+
+The level of detail to be representable by each overview layer can
+be specified as a tolerance in pixels (if different from the default of 1 pixel)
+with the function `CDB_CreateOverviewsWithToleranceInPixels`
+which has as a second additional argument the desired tolerance.
+
+This tolerance defines the maximum deviation in pixels of the overviews
+geometries with respect to the original geometries when overview tables
+are used for their intendend zoom level.
+
 ### CDB_Overviews
 
 Obtain overview metadata for a given table (existing overviews).
@@ -72,7 +90,7 @@ SELECT CDB_Overviews(CDB_QueryTablesText('SELECT * FROM table1, table2'));
 The result of `CDB_Overviews` has three columns:
 
 | base_table | z | overview_table |
-|------------+---+----------------|
+| ---------- | - | -------------- |
 | table1     | 1 | table1_ov1     |
 | table1     | 2 | table1_ov2     |
 | table1     | 4 | table1_ov4     |

doc/CDB_UserTables.md

@@ -3,7 +3,11 @@ List the name of available tables (only the usable ones)
 #### Using the function
 
 ```sql
---- Returns a row for each table having given permission with the table name
+--- Returns a row for each table having given permission with the table name.
+--- It also returns tables from others users if you've permission to see them. For example, consider the following scenario:
+--- User X and User Y at account C.
+--- User X has a public table T.
+--- User Y will see table T. 
 --- Currently accepted permissions are: 'public', 'private' or 'all'
 SELECT CDB_UserTables(perms)
 ```

doc/cartodbfy-requirements.rst

@@ -33,7 +33,7 @@ Additionally, a CartoDB table can contain other columns.
 
 See the `CartoDB User Table documentation`_
 
-.. _CartoDB User Table documentation: https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/CartoDB-user-table.md 
+.. _CartoDB User Table documentation: https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/CartoDB-user-table.rst 
 for further information.
 
 High level requirements

scripts-available/CDB_AnalysisCatalog.sql

@@ -0,0 +1,95 @@
+-- Table to register analysis nodes from https://github.com/cartodb/camshaft
+CREATE TABLE IF NOT EXISTS
+cartodb.cdb_analysis_catalog (
+    -- md5 hex hash
+    node_id char(40) CONSTRAINT cdb_analysis_catalog_pkey PRIMARY KEY,
+    -- being json allows to do queries like analysis_def->>'type' = 'buffer'
+    analysis_def json NOT NULL,
+    -- can reference other nodes in this very same table, allowing recursive queries
+    input_nodes char(40) ARRAY NOT NULL DEFAULT '{}',
+    status TEXT NOT NULL DEFAULT 'pending',
+    CONSTRAINT valid_status CHECK (
+        status IN ( 'pending', 'waiting', 'running', 'canceled', 'failed', 'ready' )
+    ),
+    created_at timestamp with time zone NOT NULL DEFAULT now(),
+    -- should be updated when some operation was performed in the node
+    -- and anything associated to it might have changed
+    updated_at timestamp with time zone DEFAULT NULL,
+    -- should register last time the node was used
+    used_at timestamp with time zone NOT NULL DEFAULT now(),
+    -- should register the number of times the node was used
+    hits NUMERIC DEFAULT 0,
+    -- should register what was the last node using current node
+    last_used_from char(40),
+    -- last job modifying the node
+    last_modified_by uuid,
+    -- store error message for failures
+    last_error_message text,
+    -- cached tables involved in the analysis
+    cache_tables regclass[] NOT NULL DEFAULT '{}',
+    -- useful for multi account deployments
+    username text
+);
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.cdb_analysis_catalog', '');
+END
+$$;
+
+-- Migrations to add new columns from old versions.
+-- IMPORTANT: Those columns will be added in order of creation. To be consistent
+-- in column order, ensure that new columns are added at the end and in the same order.
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN last_modified_by uuid;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN last_error_message text;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN cache_tables regclass[] NOT NULL DEFAULT '{}';
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username text;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+-- We want the "username" column to be moved to the last position if it was on a position from other versions
+-- see https://github.com/CartoDB/cartodb-postgresql/issues/276
+DO LANGUAGE 'plpgsql' $$
+    DECLARE
+        column_index int;
+    BEGIN
+        SELECT ordinal_position FROM information_schema.columns WHERE table_name='cdb_analysis_catalog' AND table_schema='cartodb' AND column_name='username' INTO column_index;
+        IF column_index = 1 OR column_index = 10 THEN
+           ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username_final text;
+           UPDATE cartodb.cdb_analysis_catalog SET username_final = username;
+           ALTER TABLE cartodb.cdb_analysis_catalog DROP COLUMN username;
+           ALTER TABLE cartodb.cdb_analysis_catalog RENAME COLUMN username_final TO username;
+        END IF;
+    END;
+$$;

scripts-available/CDB_AnalysisCheck.sql

@@ -0,0 +1,62 @@
+-- Read configuration parameter analysis_quota_factor, making it
+-- accessible to regular users (which don't have access to cdb_conf)
+CREATE OR REPLACE FUNCTION _CDB_GetConfAnalysisQuotaFactor()
+RETURNS float8 AS
+$$
+BEGIN
+  RETURN CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE SECURITY DEFINER;
+
+
+-- Get the factor (fraction of the quota) for Camshaft cached analysis tables
+CREATE OR REPLACE FUNCTION _CDB_AnalysisQuotaFactor()
+RETURNS float8 AS
+$$
+DECLARE
+  factor float8;
+BEGIN
+  -- We use a floating point cdb_conf parameter
+  factor := _CDB_GetConfAnalysisQuotaFactor();
+  -- With a default value
+  IF factor IS NULL THEN
+    factor := 2;
+  END IF;
+  RETURN factor;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE;
+
+-- This checks the space used up by Camshaft cached analysis tables.
+-- An exception will be raised if the limits are exceeded.
+-- The name of an analysis table is passed; this, in addition to the
+-- db role that executes this function is used to determined which
+-- analysis tables will be considered.
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+DECLARE
+  schema_name TEXT;
+  user_name TEXT;
+  nominal_quota int8;
+  cache_size float8;
+BEGIN
+  -- We rely on the search_path to determine the user's schema and
+  -- check for all analysis tables in that schema.
+  -- An alternative would be to use cdb_analysis_catalog to
+  -- select analysis tables (cache_tables) from the same user, analysis or node.
+  -- For example:
+  --   SELECT unnest(cache_tables) FROM cdb_analysis_catalog
+  --     WHERE username IN (SELECT username FROM cdb_analysis_catalog
+  --       WHERE table_name::regclass = ANY (cache_tables));
+  -- At the moment we're not using the provided table_name.
+
+  SELECT current_schema() INTO schema_name;
+  EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO nominal_quota;
+  IF nominal_quota*_CDB_AnalysisQuotaFactor() < _CDB_AnalysisDataSize(schema_name) THEN
+    -- The limit is defined by a factor applied to the total space quota for the user
+    RAISE EXCEPTION 'Analysis cache space limits exceeded';
+  END IF;
+END;
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_AnalysisSupport.sql

@@ -0,0 +1,55 @@
+-- Internal auxiliar functions to deal with [Camshaft](https://github.com/cartodb/camshaft) cached analysis tables.
+
+-- This function returns TRUE if a given table name corresponds to a Camshaft cached analysis table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsAnalysisTableName(table_name TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN table_name SIMILAR TO '\Aanalysis_[0-9a-f]{10}_[0-9a-f]{40}\Z';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- This function returns a relation of Camshaft cached analysis tables in the given schema.
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_AnalysisTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE _CDB_IsAnalysisTableName(table_name);
+$$ LANGUAGE 'sql';
+
+-- This function returns a relation user tables excluding analysis tables
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_NonAnalysisTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE Not _CDB_IsAnalysisTableName(table_name);
+$$ LANGUAGE 'sql';
+
+-- Total spaced used up by Camshaft cached analysis tables in the given schema.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_AnalysisDataSize(schema_name TEXT DEFAULT NULL)
+RETURNS bigint AS
+$$
+DECLARE
+  total_size bigint;
+BEGIN
+  WITH analysis_tables AS (
+    SELECT t.schema_name, t.table_name FROM _CDB_AnalysisTablesInSchema(schema_name) t
+  )
+  SELECT COALESCE(INT8(SUM(_CDB_total_relation_size(analysis_tables.schema_name, analysis_tables.table_name))))::int8
+    INTO total_size FROM analysis_tables;
+  IF total_size IS NOT NULL THEN
+    RETURN total_size;
+  ELSE
+    RETURN 0;
+  END IF;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;

scripts-available/CDB_ColumnNames.sql

@@ -2,15 +2,13 @@
 CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
 RETURNS SETOF information_schema.sql_identifier
 AS $$
-
-    SELECT c.column_name
-      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
-      WHERE table_name = _tn.relname
-        AND table_schema = _sn.nspname
-        AND _tn.oid = $1::oid
-        AND _sn.oid = _tn.relnamespace
-      ORDER BY ordinal_position;
-
+  SELECT
+    a.attname::information_schema.sql_identifier column_name
+    FROM pg_class c
+         LEFT JOIN pg_attribute a ON a.attrelid = c.oid
+    WHERE c.oid = $1::oid
+    AND a.attstattarget < 0 -- exclude system columns
+   ORDER BY a.attnum;
 $$ LANGUAGE SQL;
 
 -- This is to migrate from pre-0.2.0 version

scripts-available/CDB_ColumnType.sql

@@ -2,15 +2,13 @@
 CREATE OR REPLACE FUNCTION CDB_ColumnType(REGCLASS, TEXT)
 RETURNS information_schema.character_data
 AS $$
-
-    SELECT c.data_type
-      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
-      WHERE table_name = _tn.relname
-        AND table_schema = _sn.nspname
-        AND column_name = $2
-        AND _tn.oid = $1::oid
-        AND _sn.oid = _tn.relnamespace;
-         
+  SELECT
+    format_type(a.atttypid, NULL)::information_schema.character_data data_type
+  FROM pg_class c
+       LEFT JOIN pg_attribute a ON a.attrelid = c.oid
+  WHERE c.oid = $1::oid
+  AND a.attname = $2
+  AND a.attstattarget < 0; -- exclude system columns
 $$ LANGUAGE SQL;
 
 -- This is to migrate from pre-0.2.0 version

scripts-available/CDB_EqualIntervalBins.sql

@@ -1,8 +1,8 @@
 --
 -- Calculate the equal interval bins for a given column
 --
--- @param in_array A numeric array of numbers to determine the best
---                   to determine the bin boundary
+-- @param in_array An array of numbers to determine the best
+--                   bin boundary
 --
 -- @param breaks The number of bins you want to find.
 --  
@@ -11,27 +11,14 @@
 -- 
 --
 
-CREATE OR REPLACE FUNCTION CDB_EqualIntervalBins ( in_array NUMERIC[], breaks INT ) RETURNS NUMERIC[] as $$
-DECLARE 
-    diff numeric;
-    min_val numeric;
-    max_val numeric;
-    tmp_val numeric;
-    i INT := 1;
-    reply numeric[];
-BEGIN
-    SELECT min(e), max(e) INTO min_val, max_val FROM ( SELECT unnest(in_array) e ) x WHERE e IS NOT NULL;
-    diff = (max_val - min_val) / breaks::numeric;
-    LOOP
-        IF i < breaks THEN
-            tmp_val = min_val + i::numeric * diff;
-            reply = array_append(reply, tmp_val);
-            i := i+1;
-        ELSE
-            reply = array_append(reply, max_val);
-            EXIT;
-        END IF;
-    END LOOP;
-    RETURN reply;
-END;
-$$ language plpgsql IMMUTABLE;
+CREATE OR REPLACE FUNCTION CDB_EqualIntervalBins ( in_array anyarray, breaks INT ) RETURNS anyarray as $$
+WITH stats AS (
+  SELECT min(e), (max(e)-min(e))/breaks AS del
+    FROM (SELECT unnest(in_array) e) AS p)
+SELECT array_agg(bins)
+  FROM (
+    SELECT min + generate_series(1,breaks)*del AS bins
+      FROM stats) q;
+$$ LANGUAGE SQL IMMUTABLE;
+
+DROP FUNCTION IF EXISTS CDB_EqualIntervalBins( numeric[], integer);

scripts-available/CDB_EstimateRowCount.sql

@@ -0,0 +1,31 @@
+-- Internal function to generate stats for a table if they don't exist
+CREATE OR REPLACE FUNCTION _CDB_GenerateStats(reloid REGCLASS)
+RETURNS VOID
+AS $$
+DECLARE
+  has_stats BOOLEAN;
+BEGIN
+  SELECT EXISTS (
+    SELECT * FROM pg_catalog.pg_statistic WHERE starelid = reloid
+  ) INTO has_stats;
+  IF NOT has_stats THEN
+    EXECUTE Format('ANALYZE %s;', reloid);
+  END IF;
+END
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT SECURITY DEFINER;
+
+-- Return a row count estimate of the result of a query using statistics
+CREATE OR REPLACE FUNCTION CDB_EstimateRowCount(query text)
+RETURNS Numeric
+AS $$
+DECLARE
+  plan JSON;
+BEGIN
+  -- Make sure statistics exist for all the tables of the query
+  PERFORM _CDB_GenerateStats(tabname) FROM  unnest(CDB_QueryTablesText(query)) AS tabname;
+
+  -- Use the query planner to obtain an estimate of the number of result rows
+  EXECUTE 'EXPLAIN (FORMAT JSON) ' || query INTO STRICT plan;
+  RETURN plan->0->'Plan'->'Plan Rows';
+END
+$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

scripts-available/CDB_Helper.sql

@@ -1,3 +1,7 @@
+-- Create a sequence that belongs to the schema of the extension.
+-- It will be used to generate unique identifiers within the
+
+
 -- UTF8 safe and length aware. Find a unique identifier with a given prefix
 -- and/or suffix and withing a schema. If a schema is not specified, the identifier
 -- is guaranteed to be unique for all schemas.
@@ -15,8 +19,8 @@ DECLARE
 
   i INTEGER;
 BEGIN
-  -- Accounts for the _XX incremental suffix in case the identifier is taken
-  usedspace := 3;
+  -- Accounts for the XXXX incremental suffix in case the identifier is taken
+  usedspace := 4;
   usedspace := usedspace + coalesce(octet_length(prefix), 0);
   usedspace := usedspace + coalesce(octet_length(suffix), 0);
 
@@ -31,7 +35,7 @@ BEGIN
   i := 0;
   origident := ident;
 
-  WHILE i < 100 LOOP
+  WHILE i < 10000 LOOP
     IF schema IS NOT NULL THEN
       SELECT c.relname, n.nspname
       INTO rec
@@ -51,7 +55,7 @@ BEGIN
       RETURN ident;
     END IF;
 
-    ident := origident || '_' || i;
+    ident := origident || i;
     i := i + 1;
   END LOOP;
 
@@ -76,8 +80,8 @@ DECLARE
 
   i INTEGER;
 BEGIN
-  -- Accounts for the _XX incremental suffix in case the identifier is taken
-  usedspace := 3;
+  -- Accounts for the XXXX incremental suffix in case the identifier is taken
+  usedspace := 4;
   usedspace := usedspace + coalesce(octet_length(prefix), 0);
   usedspace := usedspace + coalesce(octet_length(suffix), 0);
 
@@ -92,7 +96,7 @@ BEGIN
   i := 0;
   origident := ident;
 
-  WHILE i < 100 LOOP
+  WHILE i < 10000 LOOP
     SELECT a.attname
     INTO rec
     FROM pg_class c
@@ -106,7 +110,7 @@ BEGIN
       RETURN ident;
     END IF;
 
-    ident := origident || '_' || i;
+    ident := origident || i;
     i := i + 1;
   END LOOP;
 

scripts-available/CDB_Overviews.sql

@@ -1,74 +1,3 @@
--- security definer
-
--- Pattern that can be used to detect overview tables and Extract
--- the intended zoom level from the table name.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN '\A_vovw_(\d+)_';
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
--- substring(tablename from _CDB_OverviewTableDiscriminator())
-
-
--- Pattern matched by the overview tables of a given base table name.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN _CDB_OverviewTableDiscriminator() || base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
--- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
-
--- Name of an overview table, given the base table name and the Z level
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN '_vovw_' || z::text || '_' || base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Condition to check if a tabla is an overview table of some base table
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
-RETURNS BOOLEAN
-AS $$
-  BEGIN
-    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Extract the Z level from an overview table name
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
-RETURNS INTEGER
-AS $$
-  BEGIN
-    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Name of the base table corresponding to an overview table
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
-RETURNS TEXT
-AS $$
-  BEGIN
-    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
-      RETURN overview_table;
-    ELSE
-      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
-    END IF;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
-
 -- Remove a dataset's existing  overview tables.
 -- Scope: public
 -- Parameters:
@@ -77,8 +6,11 @@ CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS)
 RETURNS void
 AS $$
 DECLARE
-    row     record;
+    row record;
+    schema_name TEXT;
+    table_name TEXT;
 BEGIN
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
     FOR row IN
         SELECT * FROM CDB_Overviews(reloid)
     LOOP
@@ -100,18 +32,20 @@ $$ LANGUAGE PLPGSQL VOLATILE;
 CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS)
 RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
 AS $$
-  -- FIXME: this will fail if the overview tables
-  -- require a explicit schema name
-  -- possible solutions: return table names as text instead of regclass
-  -- or add schema of reloid before casting to regclass
-  SELECT
-    reloid AS base_table,
-    _CDB_OverviewTableZ(cdb_usertables) AS z,
-    cdb_usertables::regclass AS overview_table
-    FROM CDB_UserTables()
-    WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
-    ORDER BY z;
-$$ LANGUAGE SQL;
+  DECLARE
+    schema_name TEXT;
+    base_table_name TEXT;
+  BEGIN
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, base_table_name;
+    RETURN QUERY SELECT
+      reloid AS base_table,
+      _CDB_OverviewTableZ(table_name) AS z,
+      table_regclass AS overview_table
+      FROM _CDB_UserTablesInSchema(schema_name)
+      WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), table_name)
+      ORDER BY z;
+  END
+$$ LANGUAGE PLPGSQL;
 
 -- Return existing overviews (if any) for multiple dataset tables.
 -- Scope: public
@@ -127,30 +61,16 @@ RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
 AS $$
   SELECT
     base_table::regclass AS base_table,
-    _CDB_OverviewTableZ(cdb_usertables) AS z,
-    cdb_usertables::regclass AS overview_table
+    _CDB_OverviewTableZ(table_name) AS z,
+    table_regclass AS overview_table
     FROM
-      CDB_UserTables(), unnest(tables) base_table
-    WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables)
+      _CDB_UserTablesInSchema(), unnest(tables) base_table
+    WHERE
+      schema_name = _cdb_schema_name(base_table)
+      AND _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), table_name)
     ORDER BY base_table, z;
 $$ LANGUAGE SQL;
 
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
-AS $$
-  BEGIN
-    SELECT n.nspname, c.relname
-    INTO STRICT schema_name, table_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
 -- Calculate the estimated extent of a cartodbfy'ed table.
 -- Scope: private.
 -- Parameters
@@ -169,17 +89,23 @@ AS $$
       FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid;
 
     ext_query = format(
-      'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');',
+      'SELECT ST_EstimatedExtent(''%1$s'', ''%2$s'', ''%3$s'');',
       table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
     );
 
     BEGIN
       EXECUTE ext_query INTO ext;
-      EXCEPTION
+    EXCEPTION
         -- This is the typical ERROR: stats for "mytable" do not exist
         WHEN internal_error THEN
           -- Get stats and execute again
-          EXECUTE format('ANALYZE %1$I', reloid);
+          EXECUTE format('ANALYZE %1$s', reloid);
+
+          -- We check the geometry type in case the error is due to empty geometries
+          IF _CDB_GeometryTypes(reloid) IS NULL THEN
+            RETURN NULL;
+          END IF;
+
           EXECUTE ext_query INTO ext;
     END;
 
@@ -225,7 +151,11 @@ AS $$
     WITH RECURSIVE t(x, y, z, e) AS (
       WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g),
       base AS (
-        SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z
+        SELECT
+          least(
+           -floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)),
+           _CDB_MaxOverviewLevel()+1
+          )::integer z
         FROM ext
       ),
       lim AS (
@@ -247,10 +177,10 @@ AS $$
       UNION ALL
       SELECT x*2 + xx, y*2 + yy, t.z+1, (
         SELECT count(*) FROM %1$s
-          WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1)
+          WHERE the_geom_webmercator && CDB_XYZ_Extent(t.x*2 + c.xx, t.y*2 + c.yy, t.z+1)
       )
       FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy)
-      WHERE t.e > %2$s AND t.z < (base.z + %3$s)
+      WHERE t.e > %2$s AND t.z < least(base.z + %3$s, _CDB_MaxZoomLevel())
     )
     SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0;
   ', reloid::text, min_features, nz, n, c, reloid::oid)
@@ -266,26 +196,32 @@ $$ LANGUAGE PLPGSQL STABLE;
 -- Parameters:
 --   reloid: oid of the input table. It must be a cartodbfy'ed table.
 -- Return value: Z level as an integer
-CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
+CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS INTEGER
 AS $$
   DECLARE
-    lim FLOAT8 := 500; -- TODO: determine/parameterize this
+    lim FLOAT8;
     nz integer := 4;
     fd FLOAT8;
     c FLOAT8;
   BEGIN
+    IF (tolerance_px IS NULL) OR tolerance_px = 0 THEN
+      lim := 500;
+    ELSE
+      lim := floor(power(256/tolerance_px, 2))/2;
+    END IF;
+
     -- Compute fd as an estimation of the (maximum) number
     -- of features per unit of tile area (in webmercator squared meters)
     SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
     -- lim maximum number of (desiderable) features per tile
     -- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference)
-    -- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z)
-    -- => fd*ta(z) if the average number of features per tile at level z
+    -- ta(z): tile area = power(c*power(2,-z), 2) = c*c*power(2,-2*z)
+    -- => fd*ta(z) is the average number of features per tile at level z
     -- find minimum z so that fd*ta(z) <= lim
     -- compute a rough 'feature density' value
     SELECT CDB_XYZ_Resolution(-8) INTO c;
-    RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2);
+    RETURN least(_CDB_MaxOverviewLevel()+1, ceil(log(2.0, (c*c*fd/lim)::numeric)/2));
   END;
 $$ LANGUAGE PLPGSQL STABLE;
 
@@ -321,7 +257,7 @@ $$ LANGUAGE PLPGSQL IMMUTABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, tolerance_px FLOAT8 DEFAULT NULL, has_overview_created BOOLEAN DEFAULT FALSE)
 RETURNS REGCLASS
 AS $$
   DECLARE
@@ -330,12 +266,26 @@ AS $$
     base_name TEXT;
     class_info RECORD;
     num_samples INTEGER;
+    schema_name TEXT;
+    table_name TEXT;
+    overview_table_name TEXT;
+    creation_clause TEXT;
   BEGIN
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+    -- TODO: compute fraction from tolerance_px if not NULL
     fraction := power(2, 2*(overview_z - ref_z));
 
-    -- FIXME: handle schema name for overview_rel if reloid requires it
-    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Sampling reduce stategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Sampling reduce stategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
 
     -- Estimate number of rows
     SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
@@ -344,21 +294,21 @@ AS $$
     IF class_info.relpages < 2 OR fraction > 0.5 THEN
       -- We'll avoid possible CDB_RandomTids problems
       EXECUTE Format('
-        CREATE TABLE %I AS SELECT * FROM %s WHERE random() < %s;
-      ', overview_rel, reloid, fraction);
+        %s SELECT * FROM %s WHERE random() < %s;
+      ', creation_clause, reloid, fraction);
     ELSE
       num_samples := ceil(class_info.reltuples*fraction);
       EXECUTE Format('
-        CREATE TABLE %1$I AS SELECT * FROM %2$s
+        %1$s SELECT * FROM %2$s
           WHERE ctid = ANY (
             ARRAY[
               (SELECT CDB_RandomTids(''%2$s'', %3$s))
             ]
           );
-      ', overview_rel, reloid, num_samples);
+      ', creation_clause, reloid, num_samples);
     END IF;
 
-    RETURN overview_rel;
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL;
 
@@ -394,9 +344,12 @@ AS $$
 
       -- preserve the owner of the base table
       SELECT u.usename
-        FROM pg_catalog.pg_class c JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
-        WHERE c.relname = dataset::text
+        FROM pg_catalog.pg_class c
+          JOIN pg_catalog.pg_user u ON (c.relowner=u.usesysid)
+          JOIN pg_namespace n ON n.oid = c.relnamespace
+        WHERE c.relname = dataset_name::text AND n.nspname = dataset_scheme
         INTO table_owner;
+
       EXECUTE Format('ALTER TABLE IF EXISTS %s OWNER TO %I;', overview_table::text, table_owner);
 
       -- preserve the table privileges
@@ -450,6 +403,70 @@ BEGIN
 END
 $$ LANGUAGE PLPGSQL STABLE;
 
+-- Check if a column of a table is of an unlimited-length text type
+CREATE OR REPLACE FUNCTION _cdb_unlimited_text_column(reloid REGCLASS, col_name TEXT)
+RETURNS BOOLEAN
+AS $$
+  SELECT EXISTS (
+    SELECT a.attname
+    FROM pg_class c
+         LEFT JOIN pg_attribute a ON a.attrelid = c.oid
+         LEFT JOIN pg_type t ON t.oid = a.atttypid
+    WHERE c.oid = reloid
+      AND a.attname = col_name
+      AND format_type(a.atttypid, NULL) IN ('text', 'character varying', 'character')
+      AND format_type(a.atttypid, NULL) = format_type(a.atttypid, a.atttypmod)
+  );
+$$ LANGUAGE SQL STABLE;
+
+CREATE OR REPLACE FUNCTION _cdb_categorical_column(reloid REGCLASS, col_name TEXT)
+RETURNS BOOLEAN
+AS $$
+DECLARE
+    schema_name TEXT;
+    table_name TEXT;
+    available BOOLEAN;
+    categorical BOOLEAN;
+BEGIN
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+    SELECT n_distinct IS NOT NULL
+    FROM pg_stats
+    WHERE pg_stats.schemaname = schema_name
+      AND pg_stats.tablename = table_name
+      AND pg_stats.attname = col_name
+    INTO available;
+    IF available IS NULL OR NOT available THEN
+      EXECUTE Format('ANALYZE %s;', reloid);
+    END IF;
+    SELECT n_distinct > 0 AND n_distinct <= 20
+    FROM pg_stats
+    WHERE pg_stats.schemaname = schema_name
+      AND pg_stats.tablename = table_name
+      AND pg_stats.attname = col_name
+    INTO categorical;
+    RETURN categorical;
+END;
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE FUNCTION _cdb_mode_of_array(anyarray)
+  RETURNS anyelement AS
+$$
+    SELECT a
+    FROM unnest($1) a
+    GROUP BY 1
+    ORDER BY COUNT(1) DESC, 1
+    LIMIT 1;
+$$
+LANGUAGE SQL IMMUTABLE;
+
+DROP AGGREGATE IF EXISTS _cdb_mode(anyelement);
+CREATE AGGREGATE _cdb_mode(anyelement) (
+  SFUNC=array_append,
+  STYPE=anyarray,
+  FINALFUNC=_cdb_mode_of_array,
+  INITCOND='{}'
+);
+
 -- SQL Aggregation expression for a datase attribute
 -- Scope: private.
 -- Parameters
@@ -464,6 +481,10 @@ AS $$
 DECLARE
   column_type TEXT;
   qualified_column TEXT;
+  has_counter_column BOOLEAN;
+  feature_count TEXT;
+  total_feature_count TEXT;
+  base_table REGCLASS;
 BEGIN
   IF table_alias <> '' THEN
     qualified_column := Format('%I.%I', table_alias, column_name);
@@ -473,19 +494,42 @@ BEGIN
 
   column_type := CDB_ColumnType(reloid, column_name);
 
+  SELECT EXISTS (
+    SELECT * FROM CDB_ColumnNames(reloid)  as colname WHERE colname = '_feature_count'
+  ) INTO has_counter_column;
+  IF has_counter_column THEN
+    feature_count := '_feature_count';
+    total_feature_count := 'SUM(_feature_count)';
+  ELSE
+    feature_count := '1';
+    total_feature_count := 'count(*)';
+  END IF;
+
+  base_table := _CDB_OverviewBaseTable(reloid);
+
   CASE column_type
-  WHEN 'double precision', 'real', 'integer', 'bigint' THEN
-    RETURN Format('AVG(%s)::' || column_type, qualified_column);
-  WHEN 'text' THEN
-    -- TODO: we could define a new aggregate function that returns distinct
-    -- separated values with a limit, adding ellipsis if more values existed
-    -- e.g. with '/' as separator and a limit of three:
-    --     'A', 'B', 'A', 'C', 'D' => 'A/B/C/...'
-    -- Other ideas: if value is unique then use it, otherwise use something
-    -- like '*' or '(varies)' or '(multiple values)', or NULL
-    -- Using 'string_agg(' || qualified_column || ',''/'')'
-    -- here causes
-    RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+  WHEN 'double precision', 'real', 'integer', 'bigint', 'numeric' THEN
+    IF column_name = '_feature_count' THEN
+      RETURN 'SUM(_feature_count)';
+    ELSE
+      IF column_type = 'integer' AND _cdb_categorical_column(base_table, column_name) THEN
+        RETURN Format('CDB_Math_Mode(%s)::', qualified_column) || column_type;
+      ELSE
+        RETURN Format('SUM(%s*%s)/%s::' || column_type, qualified_column, feature_count, total_feature_count);
+      END IF;
+    END IF;
+  WHEN 'text', 'character varying', 'character' THEN
+    IF _cdb_categorical_column(base_table, column_name) THEN
+      RETURN Format('_cdb_mode(%s)::', qualified_column) || column_type;
+    ELSE
+      IF _cdb_unlimited_text_column(base_table, column_name) THEN
+        -- TODO: this should not be applied to columns containing largish text;
+        -- it is intended only to short names/identifiers
+        RETURN  'CASE WHEN count(distinct ' || qualified_column || ') = 1 THEN MIN(' || qualified_column || ') WHEN ' || total_feature_count || ' < 5 THEN string_agg(distinct ' || qualified_column || ','' / '') ELSE ''*'' END::' || column_type;
+      ELSE
+        RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+      END IF;
+    END IF;
   WHEN 'boolean' THEN
     RETURN 'CASE count(*) WHEN 1 THEN BOOL_AND(' || qualified_column || ') ELSE NULL END::' || column_type;
   ELSE
@@ -547,33 +591,51 @@ $$ LANGUAGE PLPGSQL STABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, grid_px FLOAT8 DEFAULT NULL, has_overview_created BOOLEAN DEFAULT FALSE)
 RETURNS REGCLASS
 AS $$
   DECLARE
     overview_rel TEXT;
     reduction FLOAT8;
     base_name TEXT;
-    grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
+    pixel_m FLOAT8;
     grid_m FLOAT8;
+    offset_m FLOAT8;
+    offset_x TEXT;
+    offset_y TEXT;
+    cell_x TEXT;
+    cell_y TEXT;
     aggr_attributes TEXT;
     attributes TEXT;
     columns TEXT;
     gtypes TEXT[];
+    schema_name TEXT;
+    table_name TEXT;
+    point_geom TEXT;
+    overview_table_name TEXT;
+    creation_clause TEXT;
   BEGIN
     SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
-    IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
       -- This strategy only supports datasets with point geomety
       RETURN NULL;
-      RETURN 'x';
     END IF;
 
     --TODO: check applicability: geometry type, minimum number of points...
 
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
 
-    -- compute grid cell size using the overview_z dimension...
-    SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
 
     attributes := _CDB_Aggregable_Attributes_Expression(reloid);
     aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
@@ -584,19 +646,31 @@ AS $$
       aggr_attributes := aggr_attributes || ', ';
     END IF;
 
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
     -- compute the resulting columns in the same order as in the base table
-    -- cartodb_id,
-    -- ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom,
-    -- ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator
-    -- %4$s
     WITH cols AS (
       SELECT
         CASE c
         WHEN 'cartodb_id' THEN 'cartodb_id'
         WHEN 'the_geom' THEN
-          'ST_Transform(ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857), 4326) AS the_geom'
+          Format('ST_Transform(%s, 4326) AS the_geom', point_geom)
         WHEN 'the_geom_webmercator' THEN
-           'ST_SetSRID(ST_MakePoint(sx/n, sy/n), 3857) AS the_geom_webmercator'
+           Format('%s AS the_geom_webmercator', point_geom)
         ELSE c
         END AS column
         FROM CDB_ColumnNames(reloid) c
@@ -605,31 +679,295 @@ AS $$
       SELECT * FROM cols
     ) AS s INTO columns;
 
-    -- FIXME: handle schema name for overview_rel if reloid requires it
-    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
 
     -- Now we cluster the data using a grid of size grid_m
     -- and selecte the centroid (average coordinates) of each cluster.
     -- If we had a selected numeric attribute of interest we could use it
     -- as a weight for the average coordinates.
     EXECUTE Format('
-      CREATE TABLE %3$I AS
+      %3$s
          WITH clusters AS (
            SELECT
              %5$s
              count(*) AS n,
-             SUM(ST_X(f.the_geom_webmercator)) AS sx,
-             SUM(ST_Y(f.the_geom_webmercator)) AS sy,
              Floor(ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
              Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
              MIN(cartodb_id) AS cartodb_id
           FROM %1$s f
+          WHERE f.the_geom_webmercator IS NOT NULL
+          GROUP BY gx, gy
+         )
+         SELECT %6$s FROM clusters
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
+
+    RETURN Format('%s', overview_table_name)::regclass;
+  END;
+$$ LANGUAGE PLPGSQL;
+
+-- This strategy places the aggregation of each cluster at the centroid of the cluster members.
+CREATE OR REPLACE FUNCTION _CDB_GridClusterCentroid_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, grid_px FLOAT8 DEFAULT NULL, has_overview_created BOOLEAN DEFAULT FALSE)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    overview_rel TEXT;
+    reduction FLOAT8;
+    base_name TEXT;
+    pixel_m FLOAT8;
+    grid_m FLOAT8;
+    offset_m FLOAT8;
+    offset_x TEXT;
+    offset_y TEXT;
+    cell_x TEXT;
+    cell_y TEXT;
+    aggr_attributes TEXT;
+    attributes TEXT;
+    columns TEXT;
+    gtypes TEXT[];
+    schema_name TEXT;
+    table_name TEXT;
+    point_geom TEXT;
+    overview_table_name TEXT;
+    creation_clause TEXT;
+  BEGIN
+    SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
+
+    attributes := _CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
+    -- compute the resulting columns in the same order as in the base table
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        WHEN 'the_geom' THEN
+          'ST_Transform(ST_SetSRID(ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857), 4326) AS the_geom'
+        WHEN 'the_geom_webmercator' THEN
+          'ST_SetSRID(ST_MakePoint(_sum_of_x/n, _sum_of_y/n), 3857) AS the_geom_webmercator'
+        ELSE c
+        END AS column
+        FROM CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster centroid strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster centroid strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and selecte the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+      %3$s
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             SUM(ST_X(f.the_geom_webmercator)) AS _sum_of_x,
+             SUM(ST_Y(f.the_geom_webmercator)) AS _sum_of_y,
+             Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
+             Floor(ST_X(f.the_geom_webmercator)/%2$s)::int AS gx,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s f
           GROUP BY gx, gy
          )
          SELECT %6$s FROM clusters
-    ', reloid::text, grid_m, overview_rel, attributes, aggr_attributes, columns);
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
 
-    RETURN overview_rel;
+    RETURN Format('%s', overview_table_name)::regclass;
+  END;
+$$ LANGUAGE PLPGSQL;
+
+-- This strategy places the aggregation of each cluster at the position of one of the cluster members.
+CREATE OR REPLACE FUNCTION _CDB_GridClusterSample_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, grid_px FLOAT8 DEFAULT NULL, has_overview_created BOOLEAN DEFAULT FALSE)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    overview_rel TEXT;
+    reduction FLOAT8;
+    base_name TEXT;
+    pixel_m FLOAT8;
+    grid_m FLOAT8;
+    offset_m FLOAT8;
+    offset_x TEXT;
+    offset_y TEXT;
+    cell_x TEXT;
+    cell_y TEXT;
+    aggr_attributes TEXT;
+    attributes TEXT;
+    columns TEXT;
+    gtypes TEXT[];
+    schema_name TEXT;
+    table_name TEXT;
+    point_geom TEXT;
+    overview_table_name TEXT;
+    creation_clause TEXT;
+  BEGIN
+    SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
+    IF gtypes IS NULL OR array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
+      -- This strategy only supports datasets with point geomety
+      RETURN NULL;
+    END IF;
+
+    --TODO: check applicability: geometry type, minimum number of points...
+
+    overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 1.0;
+    END IF;
+
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
+
+    attributes := _CDB_Aggregable_Attributes_Expression(reloid);
+    aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
+    IF attributes <> '' THEN
+      attributes := ', ' || attributes;
+    END IF;
+    IF aggr_attributes <> '' THEN
+      aggr_attributes := aggr_attributes || ', ';
+    END IF;
+
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
+
+    -- compute the resulting columns in the same order as in the base table
+    WITH cols AS (
+      SELECT
+        CASE c
+        WHEN 'cartodb_id' THEN 'cartodb_id'
+        ELSE c
+        END AS column
+        FROM CDB_ColumnNames(reloid) c
+    )
+    SELECT string_agg(s.column, ',') FROM (
+      SELECT * FROM cols
+    ) AS s INTO columns;
+
+    IF NOT columns LIKE '%_feature_count%' THEN
+      columns := columns || ', n AS _feature_count';
+    END IF;
+
+    overview_table_name := Format('%I.%I', schema_name, overview_rel);
+    IF has_overview_created THEN
+      RAISE NOTICE 'Grid cluster sampling strategy deleting and inserting because % has overviews', overview_table_name;
+      EXECUTE Format('DELETE FROM %s;', overview_table_name);
+      creation_clause := Format('INSERT INTO %s', overview_table_name);
+    ELSE
+      RAISE NOTICE 'Grid cluster sampling strategy creating a new table overview %', overview_table_name;
+      creation_clause := Format('CREATE TABLE %s AS', overview_table_name);
+    END IF;
+
+    -- Now we cluster the data using a grid of size grid_m
+    -- and select the centroid (average coordinates) of each cluster.
+    -- If we had a selected numeric attribute of interest we could use it
+    -- as a weight for the average coordinates.
+    EXECUTE Format('
+       %3$s
+         WITH clusters AS (
+           SELECT
+             %5$s
+             count(*) AS n,
+             Floor(ST_X(_f.the_geom_webmercator)/%2$s)::int AS gx,
+             Floor(ST_Y(_f.the_geom_webmercator)/%2$s)::int AS gy,
+             MIN(cartodb_id) AS cartodb_id
+          FROM %1$s _f
+          GROUP BY gx, gy
+         ),
+         cluster_geom AS (
+           SELECT the_geom, the_geom_webmercator, clusters.*
+             FROM clusters INNER JOIN %1$s _g ON (clusters.cartodb_id = _g.cartodb_id)
+         )
+         SELECT %6$s FROM cluster_geom
+    ', reloid::text, grid_m, creation_clause, attributes, aggr_attributes, columns);
+
+    RETURN Format('%s', overview_table_name)::regclass;
   END;
 $$ LANGUAGE PLPGSQL;
 
@@ -644,7 +982,20 @@ $$ LANGUAGE PLPGSQL;
 --                    created by the strategy must have the same columns
 --                    as the base table and in the same order.
 -- Return value: Array with the names of the generated overview tables
-CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
+CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8,BOOLEAN)'::regprocedure)
+RETURNS text[]
+AS $$
+DECLARE
+  tolerance_px FLOAT8;
+BEGIN
+  -- Use the default tolerance
+  tolerance_px := 1.0;
+  RETURN CDB_CreateOverviewsWithToleranceInPixels(reloid, tolerance_px, refscale_strategy, reduce_strategy);
+END;
+$$ LANGUAGE PLPGSQL;
+
+-- Create overviews with additional parameter to define the desired detail/tolerance in pixels
+CREATE OR REPLACE FUNCTION CDB_CreateOverviewsWithToleranceInPixels(reloid REGCLASS, tolerance_px FLOAT8, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8,BOOLEAN)'::regprocedure)
 RETURNS text[]
 AS $$
 DECLARE
@@ -655,9 +1006,15 @@ DECLARE
   overview_z integer;
   overview_tables REGCLASS[];
   overviews_step integer := 1;
+  has_counter_column boolean;
+  has_overviews_for_z boolean;
 BEGIN
   -- Determine the referece zoom level
-  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'', %s);', reloid, tolerance_px) INTO ref_z;
+
+  IF ref_z < 0 OR ref_z IS NULL THEN
+    RETURN NULL;
+  END IF;
 
   -- Determine overlay zoom levels
   -- TODO: should be handled by the refscale_strategy?
@@ -667,19 +1024,49 @@ BEGIN
     overview_z := overview_z - overviews_step;
   END LOOP;
 
-  -- Create overlay tables
+  --  TODO Check for non-used overview to delete them but we have to be aware that the
+  --       current queries, for example from a tiler, are been used with the old overviews
+  --       so if we remove the overviews in the process this could lead to errors
+
+  -- Create or reganerate overlay tables
   base_z := ref_z;
   base_rel := reloid;
   FOREACH overview_z IN ARRAY overviews_z LOOP
-    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
+    SELECT CASE WHEN count(*) > 0 THEN TRUE ELSE FALSE END from CDB_Overviews(reloid) WHERE z = overview_z INTO has_overviews_for_z;
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s, %s, ''%s'');', base_rel, base_z, overview_z, tolerance_px, has_overviews_for_z) INTO base_rel;
     IF base_rel IS NULL THEN
       EXIT;
     END IF;
     base_z := overview_z;
-    PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
+    IF NOT has_overviews_for_z THEN
+      RAISE NOTICE 'Registering overview: %', base_rel;
+      PERFORM _CDB_Register_Overview(reloid, base_rel, base_z);
+    END IF;
     SELECT array_append(overview_tables, base_rel) INTO overview_tables;
   END LOOP;
 
+  IF overview_tables IS NOT NULL AND array_length(overview_tables, 1) > 0 THEN
+    SELECT EXISTS (
+      SELECT * FROM CDB_ColumnNames(reloid)  as colname WHERE colname = '_feature_count'
+    ) INTO has_counter_column;
+    IF NOT has_counter_column THEN
+      EXECUTE Format('
+        ALTER TABLE %s ADD COLUMN _feature_count integer DEFAULT 1;
+      ', reloid);
+    END IF;
+  END IF;
+
   RETURN overview_tables;
 END;
 $$ LANGUAGE PLPGSQL;
+
+-- Here are some older signatures of these functions, no longer in use.
+-- They must be droped here, after the (new) definition of the function `CDB_CreateOverviews`
+-- because that function used to contain references to them in the default argument values.
+DROP FUNCTION IF EXISTS _CDB_Feature_Density_Ref_Z_Strategy(REGCLASS);
+DROP FUNCTION IF EXISTS _CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);
+DROP FUNCTION IF EXISTS _CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8);
+DROP FUNCTION IF EXISTS _CDB_GridClusterCentroid_Reduce_Strategy(REGCLASS, INTEGER, INTEGER, FLOAT8);
+DROP FUNCTION IF EXISTS _CDB_GridClusterSample_Reduce_Strategy(REGCLASS, INTEGER, INTEGER, FLOAT8);
+DROP FUNCTION IF EXISTS _CDB_Sampling_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);
+DROP FUNCTION IF EXISTS _CDB_Sampling_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8);

scripts-available/CDB_OverviewsSupport.sql

@@ -0,0 +1,173 @@
+-- Auxiliary overviews FUNCTIONS
+
+-- Maximum zoom level for which overviews may be created
+CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel()
+RETURNS INTEGER
+AS $$
+  BEGIN
+    -- Zoom level will be limited so that both tile coordinates
+    -- and gridding coordinates within a tile up to 1px
+    -- (i.e. tile coordinates / 256)
+    -- can be stored in a 32-bit signed integer.
+    -- We have 31 bits por positive numbers
+    -- For zoom level Z coordinates range from 0 to 2^Z-1, so they
+    -- need Z bits, and need 8 bits more to address pixels within a tile
+    -- (gridding), so we'll limit Z to a maximum of 31 - 8
+    RETURN 23;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Maximum zoom level usable with integer coordinates
+CREATE OR REPLACE FUNCTION _CDB_MaxZoomLevel()
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN 31;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Information about tables in a schema.
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT
+    c.oid::regclass AS table_regclass,
+    n.nspname::text AS schema_name,
+    c.relname::text AS table_relname
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  AND c.relname NOT IN ('cdb_tablemetadata', 'cdb_analysis_catalog', 'cdb_conf', 'spatial_ref_sys')
+  AND CASE WHEN schema_name IS NULL
+             THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
+           ELSE n.nspname = schema_name
+           END;
+$$ LANGUAGE 'sql';
+
+-- Pattern that can be used to detect overview tables and Extract
+-- the intended zoom level from the table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '\A_vovw_(\d+)_';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- substring(tablename from _CDB_OverviewTableDiscriminator())
+
+
+-- Pattern matched by the overview tables of a given base table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN _CDB_OverviewTableDiscriminator() || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
+
+-- Name of an overview table, given the base table name and the Z level
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '_vovw_' || z::text || '_' || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Condition to check if a tabla is an overview table of some base table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Extract the Z level from an overview table name
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Name of the base table corresponding to an overview table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
+      RETURN overview_table;
+    ELSE
+      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
+    END IF;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTable(overview_table REGCLASS)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    table_name TEXT;
+    schema_name TEXT;
+    base_name TEXT;
+    base_table REGCLASS;
+  BEGIN
+    SELECT * FROM _cdb_split_table_name(overview_table) INTO schema_name, table_name;
+    base_name := _CDB_OverviewBaseTableName(table_name);
+    IF base_name != table_name THEN
+      base_table := Format('%I.%I', schema_name, base_name)::regclass;
+    ELSE
+      base_table := overview_table;
+    END IF;
+    RETURN base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
+AS $$
+  BEGIN
+    SELECT n.nspname, c.relname
+    INTO STRICT schema_name, table_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
+RETURNS TEXT
+AS $$
+  DECLARE
+    schema_name TEXT;
+  BEGIN
+    SELECT n.nspname
+    INTO STRICT schema_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+    RETURN schema_name;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;

scripts-available/CDB_Quota.sql

@@ -26,16 +26,15 @@ BEGIN
       WHERE o_table_schema = schema_name AND o_table_catalog = current_database()
   ),
   user_tables AS (
-    SELECT table_name FROM information_schema.tables
-      WHERE table_catalog = current_database() AND table_schema = schema_name
-        AND table_name != 'spatial_ref_sys'
-        AND table_name != 'cdb_tablemetadata'
-        AND table_type = 'BASE TABLE'
+    SELECT table_name FROM _CDB_NonAnalysisTablesInSchema(schema_name)
   ),
   table_cat AS (
     SELECT
       table_name,
-      EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
+      (
+        EXISTS(select * from raster_tables where o_table_name = table_name)
+        OR table_name SIMILAR TO _CDB_OverviewTableDiscriminator() || '[\w\d]*'
+      ) AS is_overview,
       EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
     FROM user_tables
   ),

scripts-available/CDB_Stats.sql

@@ -4,7 +4,7 @@
 -- @param in_array A numeric array of numbers
 --
 -- Returns: statistical quantity chosen
--- 
+--
 -- References: http://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm
 --
 
@@ -13,17 +13,21 @@ CREATE OR REPLACE FUNCTION CDB_Kurtosis ( in_array NUMERIC[] ) RETURNS NUMERIC a
 DECLARE
     a numeric;
     c numeric;
-    s numeric;
     k numeric;
 BEGIN
-    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+    SELECT AVG(e), COUNT(e)::numeric * power(stddev(e),4) INTO a, c FROM ( SELECT unnest(in_array) e ) x;
 
-    EXECUTE 'SELECT sum(power($1 - e, 4)) / ( $2 * power($3, 4)) - 3
-             FROM (SELECT unnest($4) e ) x'
-    INTO k
-    USING a, c, s, in_array;
+    IF c=0 THEN
+      RETURN 0;
+    ELSE
 
-    RETURN k;
+      EXECUTE 'SELECT sum(power($1 - e, 4)) / ($2 ) - 3
+             FROM (SELECT unnest($3) e ) x'
+      INTO k
+      USING a, c, in_array;
+
+      RETURN k;
+    END IF;
 END;
 $$ language plpgsql IMMUTABLE;
 
@@ -32,16 +36,18 @@ CREATE OR REPLACE FUNCTION CDB_Skewness ( in_array NUMERIC[] ) RETURNS NUMERIC a
 DECLARE
     a numeric;
     c numeric;
-    s numeric;
     sk numeric;
 BEGIN
-    SELECT AVG(e), COUNT(e)::numeric, stddev(e) INTO a, c, s FROM ( SELECT unnest(in_array) e ) x;
+    SELECT AVG(e), COUNT(e)::numeric * power(stddev(e),3) INTO a, c FROM ( SELECT unnest(in_array) e ) x;
+    IF c=0 THEN
+      RETURN 0;
+    ELSE
+      EXECUTE 'SELECT sum(power($1 - e, 3)) / ( $2 )
+             FROM (SELECT unnest($3) e ) x'
+      INTO sk
+      USING a, c, in_array;
 
-    EXECUTE 'SELECT sum(power($1 - e, 3)) / ( $2 * power($3, 3))
-             FROM (SELECT unnest($4) e ) x'
-    INTO sk
-    USING a, c, s, in_array;
-
-    RETURN sk;
+      RETURN sk;
+    END IF;
 END;
 $$ language plpgsql IMMUTABLE;

scripts-available/CDB_UserTables.sql

@@ -14,7 +14,7 @@ SELECT c.relname
 FROM pg_class c 
 JOIN pg_namespace n ON n.oid = c.relnamespace
 WHERE c.relkind = 'r' 
-AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
+AND c.relname NOT IN ('cdb_tablemetadata', 'cdb_analysis_catalog', 'cdb_conf', 'spatial_ref_sys')
 AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
 AND CASE WHEN perm = 'public' THEN has_table_privilege('publicuser', c.oid, 'SELECT')
          WHEN perm = 'private' THEN has_table_privilege(current_user, c.oid, 'SELECT') AND NOT has_table_privilege('publicuser', c.oid, 'SELECT')

scripts-available/CDB_XYZ.sql

@@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION CDB_XYZ_Resolution(z INTEGER)
 RETURNS FLOAT8
 AS $$
   -- circumference divided by 256 is z0 resolution, then divide by 2^z
-  SELECT 40075017.0 / 256 / power(2, z);
+  SELECT 6378137.0*2.0*pi() / 256.0 / power(2.0, z);
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 -- }
 

scripts-available/CDB_ZoomFromScale.sql

@@ -1,30 +1,36 @@
-CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
-BEGIN
-  CASE
-    WHEN scaleDenominator > 500000000 THEN RETURN 0;
-    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
-    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
-    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
-    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
-    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
-    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
-    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
-    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
-    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
-    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
-    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
-    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
-    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
-    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
-    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
-    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
-    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
-    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
-    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
-    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
-    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
-    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
-    WHEN scaleDenominator <= 100 THEN RETURN 23;
-  END CASE;
-END
-$$ LANGUAGE plpgsql IMMUTABLE;
+-- Maximum supported zoom level
+CREATE OR REPLACE FUNCTION _CDB_MaxSupportedZoom()
+RETURNS int
+LANGUAGE SQL
+IMMUTABLE
+AS $$
+  -- The maximum zoom level has to be limited for various reasons,
+  -- e.g. zoom levels greater than 31 would require tile coordinates
+  -- that would not fit in an INTEGER (which is signed, 32 bits long).
+  -- We'll choose 20 as a limit which is safe also when the JavaScript shift
+  -- operator (<<) is used for computing powers of two.
+  SELECT 29;
+$$;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric)
+RETURNS int
+LANGUAGE SQL
+IMMUTABLE
+AS $$
+  SELECT
+    CASE
+      WHEN scaleDenominator > 600000000 THEN
+        -- Scale is smaller than zoom level 0
+        NULL
+      WHEN scaleDenominator = 0 THEN
+        -- Actual zoom level would be infinite
+        _CDB_MaxSupportedZoom()
+      ELSE
+        CAST (
+          LEAST(
+            ROUND(LOG(2, 559082264.028/scaleDenominator)),
+            _CDB_MaxSupportedZoom()
+          )
+        AS INTEGER)
+    END;
+$$;

scripts-enabled/085-CDB_OverviewsSupport.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_OverviewsSupport.sql

scripts-enabled/260-CDB_AnalysisCatalog.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisCatalog.sql

scripts-enabled/270-CDB_AnalysisSupport.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisSupport.sql

scripts-enabled/275-CDB_AnalysisCheck.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisCheck.sql

scripts-enabled/280-CDB_EstimateRowCount.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_EstimateRowCount.sql

test/CDB_AnalysisCheckTest.sql

@@ -0,0 +1,20 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
+SELECT CDB_SetUserQuotaInBytes(1000000);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9(id int);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+SELECT CDB_SetUserQuotaInBytes(1);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id) VALUES (1),(2),(3),(4),(5);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9;
+DROP FUNCTION "public"._CDB_UserQuotaInBytes();

test/CDB_AnalysisCheckTest_expect

@@ -0,0 +1,18 @@
+SET
+1000000
+0
+CREATE TABLE
+CREATE TABLE
+CREATE TABLE
+(analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5,public,analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5)
+(analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94,public,analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94)
+0
+
+1
+
+INSERT 0 5
+ERROR:  Analysis cache space limits exceeded
+DROP TABLE
+DROP TABLE
+DROP TABLE
+DROP FUNCTION

test/CDB_CartodbfyTableTest.sql

@@ -372,6 +372,19 @@ SELECT column_name FROM information_schema.columns WHERE table_name = 'test' AND
 DROP TABLE test;
 SET client_min_messages TO error;
 
+-- Unique identifier generation can break CDB_CartodbfyTable #305
+BEGIN;
+    DO $$
+    BEGIN
+       FOR i IN 1..150 LOOP
+            EXECUTE 'CREATE TABLE untitled_table();';
+            EXECUTE $query$SELECT CDB_CartodbfyTable('untitled_table');$query$;
+            EXECUTE 'ALTER TABLE untitled_table RENAME TO my_renamed_table_' || i;
+       END LOOP;
+    END;
+    $$;
+ROLLBACK;
+
 -- TODO: table with existing custom-triggered the_geom
 
 DROP FUNCTION CDB_CartodbfyTableCheck(regclass, text);

test/CDB_CartodbfyTableTest_expect

@@ -147,5 +147,8 @@ NOTICE:  Trying to recover data from _cartodb_id0 column
 
 DROP TABLE
 SET
+BEGIN
+DO
+ROLLBACK
 DROP FUNCTION
 DROP FUNCTION

test/CDB_DigitSeparatorTest_expect

@@ -1,5 +1,6 @@
 BEGIN
 CREATE TABLE
+COPY 3
 none||
 only_com_dec|.|,
 only_dot_dec|,|.

test/CDB_EqualIntervalBinsTest.sql

@@ -2,4 +2,10 @@ WITH data AS (
     SELECT array_agg(x::numeric) s FROM generate_series(1,300) x 
         WHERE x % 5 != 0 AND x % 7 != 0
     ) 
-SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data
+SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT round(unnest(CDB_EqualIntervalBins(s, 7)),7) FROM data_nulls;

test/CDB_EqualIntervalBinsTest_expect

@@ -5,3 +5,10 @@
 213.8571429
 256.4285714
 299.0000000
+15.0000000
+29.0000000
+43.0000000
+57.0000000
+71.0000000
+85.0000000
+99.0000000

test/CDB_EstimateRowCountTest.sql

@@ -0,0 +1,10 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+CREATE TABLE tmptab1(id INT);
+INSERT INTO tmptab1(id) VALUES (1), (2), (3);
+CREATE TABLE tmptab2(id INT, value NUMERIC);
+INSERT INTO tmptab2(id, value) VALUES (1, 10.0), (2, 20.0);
+SELECT CDB_EstimateRowCount('SELECT SUM(value) FROM tmptab1 INNER JOIN tmptab2 ON (tmptab1.id = tmptab2.id);') AS row_count;
+SELECT CDB_EstimateRowCount('UPDATE tmptab2 SET value = 30 WHERE id=2;') AS row_count;
+DROP TABLE tmptab2;
+DROP TABLE tmptab1;

test/CDB_EstimateRowCountTest_expect

@@ -0,0 +1,9 @@
+SET
+CREATE TABLE
+INSERT 0 3
+CREATE TABLE
+INSERT 0 2
+1
+1
+DROP TABLE
+DROP TABLE

test/CDB_HeadsTailsBinsTest.sql

@@ -1,5 +1,11 @@
 WITH data AS (
-    SELECT array_agg(x) x FROM generate_series(1,100) x 
+    SELECT array_agg(x::numeric) s FROM generate_series(1,100) x
         WHERE x % 5 != 0 AND x % 7 != 0
-    ) 
-SELECT round(unnest(CDB_HeadsTailsBins(x, 7)),2) FROM data
+    )
+SELECT round(unnest(CDB_HeadsTailsBins(s, 7)),2) FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT round(unnest(CDB_HeadsTailsBins(s, 7)),2) FROM data_nulls;

test/CDB_HeadsTailsBinsTest_expect

@@ -5,3 +5,9 @@
 96.50
 98.00
 99.00
+49.76
+74.65
+88.50
+94.50
+98.00
+99.00

test/CDB_HelperTest.sql

@@ -14,17 +14,17 @@ SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolar
 SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
 
 -- Test new identifier is found when name is taken from previous case
-CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargolar (name text);
+CREATE TABLE prefix_largolargolargolargolargolargolargolargolargolargola (name text);
 SELECT * FROM cartodb._CDB_Unique_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL);
-DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargolar;
+DROP TABLE prefix_largolargolargolargolargolargolargolargolargolargola;
 
 -- Test unique identifier creation with suffix with long length normal relname
 SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
 
 -- Test new identifier is found when name is taken from previous case
-CREATE TABLE largolargolargolargolargolargolargolargolargolargolar_suffix (name text);
+CREATE TABLE largolargolargolargolargolargolargolargolargolargola_suffix (name text);
 SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix');
-DROP TABLE largolargolargolargolargolargolargolargolargolargolar_suffix;
+DROP TABLE largolargolargolargolargolargolargolargolargolargola_suffix;
 
 -- Test unique identifier creation with normal length UTF8 relname
 SELECT * FROM cartodb._CDB_Unique_Identifier(NULL, 'piraña', NULL);
@@ -72,7 +72,7 @@ SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargol
 DROP TABLE test;
 
 -- Test new identifier is found when name is taken from previous case
-CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargolar text);
+CREATE TABLE test (prefix_largolargolargolargolargolargolargolargolargolargola text);
 SELECT * FROM cartodb._CDB_Unique_Column_Identifier('prefix_', 'largolargolargolargolargolargolargolargolargolargolargolargolar', NULL, 'test'::regclass);
 DROP TABLE test;
 
@@ -82,7 +82,7 @@ SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargol
 DROP TABLE test;
 
 -- Test new identifier is found when name is taken from previous case
-CREATE TABLE test (largolargolargolargolargolargolargolargolargolargolar_suffix text);
+CREATE TABLE test (largolargolargolargolargolargolargolargolargolargola_suffix text);
 SELECT * FROM cartodb._CDB_Unique_Column_Identifier(NULL, 'largolargolargolargolargolargolargolargolargolargolargolargolar', '_suffix', 'test'::regclass);
 DROP TABLE test;
 

test/CDB_HelperTest_expect

@@ -1,58 +1,58 @@
 relname
 prefix_relname
 relname_suffix
-largolargolargolargolargolargolargolargolargolargolargolargo
-prefix_largolargolargolargolargolargolargolargolargolargolar
+largolargolargolargolargolargolargolargolargolargolargolarg
+prefix_largolargolargolargolargolargolargolargolargolargola
 CREATE TABLE
-prefix_largolargolargolargolargolargolargolargolargolargolar_0
+prefix_largolargolargolargolargolargolargolargolargolargola0
 DROP TABLE
-largolargolargolargolargolargolargolargolargolargolar_suffix
+largolargolargolargolargolargolargolargolargolargola_suffix
 CREATE TABLE
-largolargolargolargolargolargolargolargolargolargolar_suffix_0
+largolargolargolargolargolargolargolargolargolargola_suffix0
 DROP TABLE
 piraña
 prefix_piraña
 piraña_suffix
-piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaáci
 prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
 CREATE TABLE
-prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi0
 DROP TABLE
 piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
 CREATE TABLE
-piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix0
 DROP TABLE
 CREATE TABLE
 colname
 prefix_colname
 colname_suffix
-largolargolargolargolargolargolargolargolargolargolargolargo
-prefix_largolargolargolargolargolargolargolargolargolargolar
+largolargolargolargolargolargolargolargolargolargolargolarg
+prefix_largolargolargolargolargolargolargolargolargolargola
 DROP TABLE
 CREATE TABLE
-prefix_largolargolargolargolargolargolargolargolargolargolar_0
+prefix_largolargolargolargolargolargolargolargolargolargola0
 DROP TABLE
 CREATE TABLE
-largolargolargolargolargolargolargolargolargolargolar_suffix
+largolargolargolargolargolargolargolargolargolargola_suffix
 DROP TABLE
 CREATE TABLE
-largolargolargolargolargolargolargolargolargolargolar_suffix_0
+largolargolargolargolargolargolargolargolargolargola_suffix0
 DROP TABLE
 CREATE TABLE
 piraña
 prefix_piraña
 piraña_suffix
-piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaácid
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpiñaáci
 prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi
 DROP TABLE
 CREATE TABLE
-prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_0
+prefix_piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi0
 DROP TABLE
 CREATE TABLE
 piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix
 DROP TABLE
 CREATE TABLE
-piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix_0
+piñaácidpiñaácidpiñaácidpiñaácidpiñaácidpi_suffix0
 DROP TABLE
 pira
 pirañ

test/CDB_JenksBinsTest.sql

@@ -1,5 +1,11 @@
 WITH data AS (
-    SELECT array_agg(x) x FROM generate_series(1,100) x 
+    SELECT array_agg(x::numeric) s FROM generate_series(1,300) x
         WHERE x % 5 != 0 AND x % 7 != 0
     ) 
-SELECT unnest(CDB_JenksBins(x, 7)) FROM data
+SELECT unnest(CDB_JenksBins(s, 7)) FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,300) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT unnest(CDB_JenksBins(s, 7)) FROM data_nulls;

test/CDB_JenksBinsTest_expect

@@ -1,7 +1,14 @@
-13
-29
 43
-57
-71
-83
-99
+86
+129
+172
+213
+257
+299
+37
+51
+97
+157
+213
+241
+

test/CDB_OverviewsTest.sql

@@ -8,7 +8,7 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_bare_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_bare_t;
+SELECT count(*) FROM _vovw_2_base_bare_t;
 
 
 SELECT _CDB_Aggregable_Attributes_Expression('base_t'::regclass);
@@ -16,13 +16,14 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
 
 SELECT CDB_CreateOverviews('polyg_t'::regclass);
 
 SELECT CDB_CreateOverviews('column_types_t'::regclass);
 
 SELECT CDB_Overviews('base_t'::regclass);
+SELECT CDB_Overviews('"public"."base_t"'::regclass);
 SELECT CDB_Overviews(ARRAY['base_t'::regclass, 'base_bare_t'::regclass]);
 SELECT CDB_Overviews('polyg_t'::regclass);
 SELECT CDB_Overviews('column_types_t'::regclass);
@@ -30,7 +31,11 @@ SELECT CDB_Overviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('base_bare_t'::regclass);
 SELECT CDB_DropOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
+
+SELECT CDB_CreateOverviewsWithToleranceInPixels('base_t'::regclass, 7.5);
+SELECT count(*) FROM _vovw_2_base_t;
+SELECT CDB_DropOverviews('base_t'::regclass);
 
 DROP TABLE column_types_t;
 DROP TABLE base_bare_t;

test/CDB_OverviewsTest_expect

@@ -9,45 +9,39 @@ SELECT 1114
 
 
 
-{_vovw_5_base_bare_t,_vovw_4_base_bare_t,_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
-125
+{_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
+126
 number,int_number,name,start
-AVG(number)::double precision AS number,AVG(int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
-AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(tab.name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
-{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
-125
+SUM(number*1)/count(*)::double precision AS number,SUM(int_number*1)/count(*)::integer AS int_number,CASE WHEN count(distinct name) = 1 THEN MIN(name) WHEN count(*) < 5 THEN string_agg(distinct name,' / ') ELSE '*' END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
+SUM(tab.number*1)/count(*)::double precision AS number,SUM(tab.int_number*1)/count(*)::integer AS int_number,CASE WHEN count(distinct tab.name) = 1 THEN MIN(tab.name) WHEN count(*) < 5 THEN string_agg(distinct tab.name,' / ') ELSE '*' END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
+{_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+126
 
-{_vovw_5_column_types_t,_vovw_4_column_types_t,_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+{_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
-(base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (base_bare_t,0,_vovw_0_base_bare_t)
 (base_bare_t,1,_vovw_1_base_bare_t)
 (base_bare_t,2,_vovw_2_base_bare_t)
-(base_bare_t,3,_vovw_3_base_bare_t)
-(base_bare_t,4,_vovw_4_base_bare_t)
-(base_bare_t,5,_vovw_5_base_bare_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
-(base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (column_types_t,0,_vovw_0_column_types_t)
 (column_types_t,1,_vovw_1_column_types_t)
 (column_types_t,2,_vovw_2_column_types_t)
-(column_types_t,3,_vovw_3_column_types_t)
-(column_types_t,4,_vovw_4_column_types_t)
-(column_types_t,5,_vovw_5_column_types_t)
 
 
 
-ERROR:  relation "_vovw_5_base_t" does not exist
-LINE 1: SELECT count(*) FROM _vovw_5_base_t;
+ERROR:  relation "_vovw_2_base_t" does not exist
+LINE 1: SELECT count(*) FROM _vovw_2_base_t;
                              ^
+{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+38
+
 DROP TABLE
 DROP TABLE
 DROP TABLE

test/CDB_QuantileBinsTest.sql

@@ -1,5 +1,11 @@
 WITH data AS (
-    SELECT array_agg(x) x FROM generate_series(1,100) x 
+    SELECT array_agg(x::numeric) s FROM generate_series(1,100) x
         WHERE x % 5 != 0 AND x % 7 != 0
     ) 
-SELECT unnest(CDB_QuantileBins(x, 7)) FROM data
+SELECT unnest(CDB_QuantileBins(s, 7)) FROM data;
+
+WITH data_nulls AS (
+    SELECT array_agg(CASE WHEN x % 2 != 0 THEN x ELSE NULL END::numeric) s FROM generate_series(1,100) x
+        WHERE x % 5 != 0 AND x % 7 != 0
+    )
+SELECT unnest(CDB_QuantileBins(s, 7)) FROM data_nulls;

test/CDB_QuantileBinsTest_expect

@@ -4,4 +4,11 @@
 57
 71
 86
+99
+29
+57
+87
+
+
+
 99

test/CDB_QueryStatementsTest.sql

@@ -1,3 +1,6 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
 WITH q AS ( SELECT CDB_QueryStatements('
 SELECT * FROM geometry_columns;
 ') as statement )

test/CDB_QueryStatementsTest_expect

@@ -1,3 +1,4 @@
+SET
 1|1|SELECT * FROM geometry_columns
 2|1|SELECT * FROM geometry_columns
 3|1|SELECT * FROM geometry_columns

test/CDB_QueryTablesTest.sql

@@ -1,3 +1,5 @@
+SET client_min_messages TO warning;
+\set VERBOSITY terse
 
 WITH inp AS ( select 'SELECT * FROM geometry_columns'::text as q )
  SELECT q, CDB_QueryTables(q) from inp;

test/CDB_QueryTablesTest_expect

@@ -1,17 +1,14 @@
+SET
 SELECT * FROM geometry_columns|{pg_catalog.pg_attribute,pg_catalog.pg_class,pg_catalog.pg_constraint,pg_catalog.pg_namespace,pg_catalog.pg_type}
 SELECT a.attname FROM pg_class c JOIN pg_attribute a on (a.attrelid = c.oid)|{pg_catalog.pg_attribute,pg_catalog.pg_class}
 CREATE table "my'tab;le" as select 1|{}
 SELECT a.oid, b.oid FROM pg_class a, pg_class b|{pg_catalog.pg_class}
 SELECT 1 as col1; select 2 as col2|{}
 WARNING:  CDB_QueryTables cannot explain query: select 1 from nonexistant (42P01: relation "nonexistant" does not exist)
-CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 ERROR:  relation "nonexistant" does not exist
-CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 begin; select * from pg_class; commit;|{pg_catalog.pg_class}
 WARNING:  CDB_QueryTables cannot explain query: select * from test (42P01: relation "test" does not exist)
-CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 ERROR:  relation "test" does not exist
-CONTEXT:  PL/pgSQL function cdb_querytables(text) line 3 at RETURN
 WITH a AS (select * from pg_class) select * from a|{pg_catalog.pg_class}
 CREATE SCHEMA
 CREATE TABLE

test/CDB_QuotaTest.sql

@@ -27,5 +27,21 @@ INSERT INTO big VALUES (8193);
 SELECT CDB_SetUserQuotaInBytes(0);
 INSERT INTO big VALUES (8194);
 DROP TABLE big;
+
+
+--analysis tables should be excluded from quota:
+CREATE TABLE big(a int);
+CREATE TRIGGER test_quota BEFORE UPDATE OR INSERT ON big
+      EXECUTE PROCEDURE CDB_CheckQuota(1, 1, 'public');
+SELECT CDB_SetUserQuotaInBytes(1);
+CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id int);
+INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4(id) VALUES (1),(2),(3),(4),(5);
+INSERT INTO big VALUES (1); -- allowed, check runs before
+DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d4;
+INSERT INTO big VALUES (2); -- disallowed, quota exceeds before
+DROP TABLE big;
+SELECT CDB_SetUserQuotaInBytes(0);
+
+
 set client_min_messages to NOTICE;
 DROP FUNCTION _CDB_UserQuotaInBytes();

test/CDB_QuotaTest_expect

@@ -18,5 +18,15 @@ ERROR:  Quota exceeded by 443.998046875KB
 0
 INSERT 0 1
 DROP TABLE
+CREATE TABLE
+CREATE TRIGGER
+1
+CREATE TABLE
+INSERT 0 5
+INSERT 0 1
+DROP TABLE
+ERROR:  Quota exceeded by 3.9990234375KB
+DROP TABLE
+0
 SET
 DROP FUNCTION

test/extension/run_at_cartodb_schema.sql

@@ -3,4 +3,5 @@ SET SCHEMA 'cartodb';
 \i scripts-available/CDB_TableMetadata.sql
 \i scripts-available/CDB_ColumnNames.sql
 \i scripts-available/CDB_ColumnType.sql
-SET SCHEMA 'public';
+\i scripts-available/CDB_AnalysisCatalog.sql
+SET SCHEMA 'public';

test/extension/test.sh

@@ -181,6 +181,8 @@ function setup_database() {
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_OverviewsSupport.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_AnalysisSupport.sql
     # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
     ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
 }
@@ -226,6 +228,7 @@ function tear_down() {
     sql 'DROP ROLE cdb_testmember_2;'
 
     tear_down_database
+    DATABASE=postgres sql postgres 'DROP ROLE IF EXISTS publicuser';
 }
 
 
@@ -484,6 +487,18 @@ function test_foreign_tables() {
     ${CMD} -d fdw_target -f scripts-available/CDB_QueryTables.sql
     ${CMD} -d fdw_target -f scripts-available/CDB_TableMetadata.sql
 
+    DATABASE=fdw_target sql postgres "DO
+\$\$
+BEGIN
+   IF NOT EXISTS (
+      SELECT *
+      FROM   pg_catalog.pg_user
+      WHERE  usename = 'publicuser') THEN
+
+      CREATE ROLE publicuser LOGIN;
+   END IF;
+END
+\$\$;"
     DATABASE=fdw_target sql postgres 'CREATE SCHEMA test_fdw;'
     DATABASE=fdw_target sql postgres 'CREATE TABLE test_fdw.foo (a int);'
     DATABASE=fdw_target sql postgres 'INSERT INTO test_fdw.foo (a) values (42);'
@@ -563,6 +578,13 @@ test_extension|public|"local-table-with-dashes"'
     DATABASE=fdw_target tear_down_database
 }
 
+function test_cdb_catalog_basic_node() {
+    DEF="'{\"type\":\"buffer\",\"source\":\"b2db66bc7ac02e135fd20bbfef0fdd81b2d15fad\",\"radio\":10000}'"
+    sql postgres "INSERT INTO cartodb.cdb_analysis_catalog (node_id, analysis_def) VALUES ('1bbc4c41ea7c9d3a7dc1509727f698b7', ${DEF}::json)"
+    sql postgres "SELECT status from cartodb.cdb_analysis_catalog where node_id = '1bbc4c41ea7c9d3a7dc1509727f698b7'" should 'pending'
+    sql postgres "DELETE FROM cartodb.cdb_analysis_catalog"
+}
+
 #################################################### TESTS END HERE ####################################################
 
 run_tests $@