Release 0.15.0

This release includes some overviews enhancements

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.14.2
+EXTVERSION = 0.15.0
 
 SED = sed
 
@@ -62,6 +62,9 @@ UPGRADABLE = \
   0.14.0 \
   0.14.1 \
   0.14.2 \
+  0.14.3 \
+  0.14.4 \
+  0.15.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)
@@ -115,6 +118,9 @@ $(EXTENSION).control: $(EXTENSION).control.in Makefile
 cartodb_version.sql: cartodb_version.sql.in Makefile $(GITDIR)/index
 	$(SED) -e 's/@@VERSION@@/$(EXTVERSION)/' $< > $@
 
+# Needed for consistent `echo` results with backslashes
+SHELL = bash
+
 legacy_regress: $(REGRESS_OLD) Makefile
 	mkdir -p sql/test/
 	mkdir -p expected/test/
@@ -122,14 +128,14 @@ legacy_regress: $(REGRESS_OLD) Makefile
 	for f in $(REGRESS_OLD); do \
     tn=`basename $${f} .sql`; \
     of=sql/test/$${tn}.sql; \
-    echo '\\set ECHO none' > $${of}; \
-    echo '\\a' >> $${of}; \
-    echo '\\t' >> $${of}; \
-    echo '\\set QUIET off' >> $${of}; \
+    echo '\set ECHO none' > $${of}; \
+    echo '\a' >> $${of}; \
+    echo '\t' >> $${of}; \
+    echo '\set QUIET off' >> $${of}; \
     cat $${f} | \
       $(SED) -e 's/public\./cartodb./g' >> $${of}; \
     exp=expected/test/$${tn}.out; \
-    echo '\\set ECHO none' > $${exp}; \
+    echo '\set ECHO none' > $${exp}; \
     cat test/$${tn}_expect >> $${exp}; \
   done
 

NEWS.md

@@ -1,3 +1,22 @@
+0.15.0 (2016-04-05)
+
+* New function CDB_CreateOverviewsWithToleranceInPixels that adds tolerance parameter for overview creation
+  [#221](https://github.com/CartoDB/cartodb-postgresql/pull/221)
+* New default value for the overviews tolerance in pixels is 2 (used to be 7.5) (also in #221)
+* The feature density limit used to choose the reference Z level now depends on the tolerance in pixels (also in #221)
+* Tables that require an explicit schema can now be passed to overview functions
+  [#220](https://github.com/CartoDB/cartodb-postgresql/pull/220)
+
+0.14.4 (2016-03-29)
+
+* Fix creating overviews for tables with boolean columns
+  [#214](https://github.com/CartoDB/cartodb-postgresql/pull/214)
+* Fix tests for some systems [#215](https://github.com/CartoDB/cartodb-postgresql/pull/215)
+
+0.14.3 (2016-03-17)
+-------------------
+* Fix for `cartodb_id` bigint casting hardcoded in 0.14.2 to support `cartodb_id` text columns [#210](https://github.com/CartoDB/cartodb-postgresql/pull/210)
+
 0.14.2 (2016-03-15)
 -------------------
 * Support text `cartodb_id` columns in `_CDB_Has_Usable_Primary_ID` [#202](https://github.com/CartoDB/cartodb-postgresql/pull/202)

README.md

@@ -1,7 +1,7 @@
 cartodb-postgresql
 ==================
 
-[![Build Status](http://travis-ci.org/CartoDB/cartodb-postgresql.png)]
+[![Build Status](http://api.travis-ci.org/CartoDB/cartodb-postgresql.svg?branch=master)]
 (http://travis-ci.org/CartoDB/cartodb-postgresql)
 
 PostgreSQL extension for CartoDB

doc/CDB_Overviews.md

@@ -48,6 +48,13 @@ CDB_CreateOverviews(table_name, ref_z_strategy, reduction_strategy)
   - **base_z** integer, base Z level assigned to the base table.
   - **overview_z** integer, Z level for which to generate the overview.
 
+#### Tolerance / level of detail
+
+The level of detail to be representable by each overview layer can
+be specified as a tolerance in pixels (if different from the default of 2 pixels)
+with the function `CDB_CreateOverviewsWithToleranceInPixels`
+which has as a second additional argument the desired tolerance.
+
 ### CDB_Overviews
 
 Obtain overview metadata for a given table (existing overviews).

doc/CartoDB-user-table.md

@@ -1,29 +0,0 @@
-A "cartodb" user table is a table with a well-known set of fields and a well-known set of triggers attached on. 
-
-The fields are:
-
- - `cartodb_id`, a numerical primary key of serial type
- - `created_at`, timestamp with timezone not null default now()
- - `updated_at`, timestamp with timezone not null default now()
- - `the_geom`, geometry, GiST indexed, constrained (see below)
- - `the_geom_webmercator`, geometry, GiST indexed, constrained (see below)
-
-The values of "the_geom" and "the_geom_webmercator" must match these constraints:
-
- - Only POINT, MULTILINE, MULTIPOLYGON types ? Maybe UNCONSTRAINED
- - Only 2 dimensions ? Maybe UNCONSTRAINED
- - SRID=4326 for the_geom and SRID=3857 for the_geom_webmercator
-
-The triggers are:
-
- - `track_updates` after modifying statement updates cdb_tablemetadata
- - `test_quota` before changing statement to forbid if overquota
- - `test_quota_per_row` before changing row to forbod if overquota (checked on a probabilistic basis)
- - `update_the_geom_webmercator` before insert or update row to maintain the_geom_webmercator
- - `update_updated_at_trigger` before update row to maintain updated_at
-
-Some conversions will be attempted to perform upon cartodbfication when certain fields appear:
-
- - `cartodb_id`: If found type TEXT will be attempted to cast
- - `created_at`: If found type TEXT will be attempted to cast
- - `updated_at`: If found type TEXT will be attempted to cast

doc/CartoDB-user-table.rst

@@ -0,0 +1,68 @@
+CartoDB User Table
+==================
+
+Introduction
+----------
+A CartoDB user table is a table with a well-known set of columns and a well-known set of triggers attached on.
+
+Columns
+----------
+The required columns of a CartoDB table are:
+
+-  ``cartodb_id``
+  - This column will be used as the primary key of the table and it has a sequence as default value
+  - Its values must be integer, non-zero, non-null and unique
+  -  B-Tree indexed
+-  ``the_geom``
+  - This column stores the main geometric features of a table
+  - The type of the column in the Postgres database is ``geometry(Geometry,4326)```
+  - GiST indexed
+  -  geometry, GiST indexed, constrained (see below)
+-  ``the_geom_webmercator``
+  - This column stores the geometries used for rendering purposes
+  - The type of the column in the Postgres database is ``geometry(Geometry,3857)``
+  - GiST indexed
+  - This column is automatically updated by the system when the ``the_geom`` column is updated or when there is an insertion of a new row into the table (See triggers below)
+
+The values of ``the_geom`` and ``the_geom_webmercator`` must be two-dimensional Points, MultiLineStrings or MultiPolygons. Different geometric types in a CartoDB table are not supported.
+
+Described table example
+^^^^^^^^^^
+::
+
+        Column        |          Type           |                       Modifiers                        
+  ----------------------+-------------------------+--------------------------------------------------------
+   cartodb_id           | bigint                  | not null default nextval('t_cartodb_id_seq'::regclass)
+   the_geom             | geometry(Geometry,4326) | 
+   the_geom_webmercator | geometry(Geometry,3857) | 
+  Indexes:
+      "table_name_pkey" PRIMARY KEY, btree (cartodb_id)
+      "table_name_the_geom_idx" gist (the_geom)
+      "table_name_the_geom_webmercator_idx" gist (the_geom_webmercator)
+
+Triggers
+----------
+The triggers generated in each CartoDB table are:
+
+-  ``track_updates`` after modifying statement updates ``cdb_tablemetadata``
+-  ``test_quota`` before changing statement to forbid if overquota
+-  ``test_quota_per_row`` before insert ot update row to forbid if overquota (checked on a probabilistic basis)
+-  ``update_the_geom_webmercator`` before insert or update row to maintain the ``the_geom_webmercator`` updated with the contents in ``the_geom``
+
+Described triggers example
+^^^^^^^^^^
+::
+
+  test_quota BEFORE INSERT OR UPDATE ON t FOR EACH STATEMENT EXECUTE PROCEDURE cdb_checkquota('0.1', '-1', 'public')
+  test_quota_per_row BEFORE INSERT OR UPDATE ON t FOR EACH ROW EXECUTE PROCEDURE cdb_checkquota('0.001', '-1', 'public')
+  track_updates AFTER INSERT OR DELETE OR UPDATE OR TRUNCATE ON t FOR EACH STATEMENT EXECUTE PROCEDURE cdb_tablemetadata_trigger()
+  update_the_geom_webmercator_trigger BEFORE INSERT OR UPDATE OF the_geom ON t FOR EACH ROW EXECUTE PROCEDURE _cdb_update_the_geom_webmercator()
+
+
+Further details
+----------
+
+Some conversions will be attempted to perform upon cartodbfication when certain fields appear:
+
+-  ``cartodb_id``: If found type TEXT will be attempted to cast to integer. If not casteable, an eror will be raised.
+-  ``the_geom``: If found type TEXT will be attempted to cast to geometry(Geometry,4326).

doc/cartodbfy-requirements.rst

@@ -1,59 +1,63 @@
+CartoDBfy Requirements
+======================
+
 Introduction
-============
+------------
 
-This document aims at describing what cartodbfy is and what its formal requirements are, with the following goals in mind:
+This document aims at describing what the CartoDBfication is and what its formal requirements are, with the following goals in mind:
 
-- clarify what are the expectations of the "cartodbfycation process".
-- define an important part of what should be a stable, public API
-- allow for better testing, which should in turn...
+- Clarify what are the expectations of the "cartodbfycation process".
+- Define an important part of what should be a stable, public API
+- Allow for better testing, which should in turn...
 - ...ease modifications and increase quality of the code
 
 
+What is the CartoDBfycation
+---------------------------
 
-What is the cartodbfycation
-===========================
-
-The cartodbfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis.
-
+The CartoDBfycation is the process of converting an arbitrary postgres table into a valid CartoDB table, and register it in the system so that it can be used in the CartoDB editor and platform to generate maps and analysis.
 
+It is performed by running the function ``CDB_CartodbfyTable(reloid REGCLASS)`` over a target table.
 
 Valid CartoDB tables
-====================
+--------------------
 
 A valid CartoDB table shall meet the following conditions:
 
-- Have a ``cartodb_id`` integer column as primary key with a sequence as default value
+- Have a ``cartodb_id`` column with integer, unique, non-zero and non-null values as primary key with a sequence as default value
 - Have a ``the_geom`` column of type ``Geometry`` with SRID 4326
 - Have a ``the_geom_webmercator`` column of type ``Geometry`` with SRID 3857
-- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync
+- The columns ``the_geom`` and ``the_geom_webmercator`` shall be in sync (task of the ``update_the_geom_webmercator`` trigger)
 
 Additionally, a CartoDB table can contain other columns.
 
+See the `CartoDB User Table documentation`_
 
+.. _CartoDB User Table documentation: https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/CartoDB-user-table.md 
+for further information.
 
 High level requirements
-=======================
+-----------------------
 
 Here is a list of high level requirments for the public function ``CDB_CartodbfyTable()``:
 
-- A call to ``CDB_CartodbfyTable()`` shall modify/rewrite the table and produce a valid CartoDB table with the same name.
-- A call to ``CDB_CartodbfyTable()`` shall cause the registration of the table into the platform
-- It shall be idempotent, meaning that successive calls to ``CDB_CartodbfyTable()`` shall not produce any visible effect in the system.
+- A call to the function shall modify/rewrite the table and produce a valid CartoDB table with the same name.
+- A call to the function shall cause the registration of the table into the platform.
+- It shall be idempotent, meaning that successive calls to the function shall not produce any visible effect in the system.
 - If there's a column containing a geometry, it shall be used to generate ``the_geom`` and the ``the_geom_webmercator`` columns.
 - Exporting and re-importing the same table in CartoDB shall produce equivalent tables, with the same features associated to the same ``cartodb_id``'s.
 
-Note that there should be only one feature per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined.
 
+Note that there should be only one geometry per row in the source table. If there's more than one, then which one is used for ``the_geom`` and ``the_geom_webmercator`` fields is not determined.
 
 
 Low-level requirements
-======================
+----------------------
 
-- If the original table contains a valid (unique and not null) ``cartodb_id`` column, it shall be used
-- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used.
+- If the original table contains a valid (integer, unique, non-zero and not null) ``cartodb_id`` column, it shall be used
+- If the original table contains a ``the_geom`` column or a ``the_geom_webmercator`` geometric column in the expected projection (EPSG 4326 and EPSG 3857, respectively) they shall be used.
 - A modification of a cartodbfy'ed table shall insert or update a row in ``CDB_TableMetadata``
 - A cartodbfy'ed table shall have a ``btree`` index on ``cartodb_id``
 - A cartodbfy'ed table shall have ``gist`` indices on ``the_geom`` and ``the_geom_webmercator``
-- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns
-
+- Cartodbfy shall deal with text columns for imports, regarding CartoDB columns (``cartodb_id``, ``the_geom``, ``the_geom_webmercator``)
 

scripts-available/CDB_CartodbfyTable.sql

@@ -919,7 +919,7 @@ BEGIN
 
   -- Add cartodb ID!
   IF has_usable_primary_key THEN
-    sql := sql || const.pkey || '::bigint ';
+    sql := sql || const.pkey || '::integer ';
   ELSE
     sql := sql || 'nextval(''' || destseq || ''') AS ' || const.pkey;
   END IF;

scripts-available/CDB_Overviews.sql

@@ -68,6 +68,41 @@ AS $$
   END;
 $$ LANGUAGE PLPGSQL IMMUTABLE;
 
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
+AS $$
+  BEGIN
+    SELECT n.nspname, c.relname
+    INTO STRICT schema_name, table_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
+RETURNS TEXT
+AS $$
+  DECLARE
+    schema_name TEXT;
+  BEGIN
+    SELECT n.nspname
+    INTO STRICT schema_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+    RETURN schema_name;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
 
 -- Remove a dataset's existing  overview tables.
 -- Scope: public
@@ -77,12 +112,15 @@ CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS)
 RETURNS void
 AS $$
 DECLARE
-    row     record;
+    row record;
+    schema_name TEXT;
+    table_name TEXT;
 BEGIN
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
     FOR row IN
         SELECT * FROM CDB_Overviews(reloid)
     LOOP
-        EXECUTE Format('DROP TABLE %s;', row.overview_table);
+        EXECUTE Format('DROP TABLE %I.%I;', schema_name, row.overview_table);
         RAISE NOTICE 'Dropped overview for level %: %', row.z, row.overview_table;
     END LOOP;
 END;
@@ -100,18 +138,21 @@ $$ LANGUAGE PLPGSQL VOLATILE;
 CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS)
 RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
 AS $$
-  -- FIXME: this will fail if the overview tables
-  -- require a explicit schema name
-  -- possible solutions: return table names as text instead of regclass
-  -- or add schema of reloid before casting to regclass
-  SELECT
-    reloid AS base_table,
-    _CDB_OverviewTableZ(cdb_usertables) AS z,
-    cdb_usertables::regclass AS overview_table
-    FROM CDB_UserTables()
-    WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
-    ORDER BY z;
-$$ LANGUAGE SQL;
+  DECLARE
+    schema_name TEXT;
+    table_name TEXT;
+  BEGIN
+    -- TODO: review implementation of CDB_UserTables an suitability for this
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+    RETURN QUERY SELECT
+      reloid AS base_table,
+      _CDB_OverviewTableZ(cdb_usertables) AS z,
+      ('"' || schema_name|| '"."' ||cdb_usertables || '"')::regclass AS overview_table
+      FROM CDB_UserTables()
+      WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
+      ORDER BY z;
+  END
+$$ LANGUAGE PLPGSQL;
 
 -- Return existing overviews (if any) for multiple dataset tables.
 -- Scope: public
@@ -128,29 +169,13 @@ AS $$
   SELECT
     base_table::regclass AS base_table,
     _CDB_OverviewTableZ(cdb_usertables) AS z,
-    cdb_usertables::regclass AS overview_table
+    ('"' || _cdb_schema_name(base_table::regclass) || '"."' || cdb_usertables || '"')::regclass AS overview_table
     FROM
       CDB_UserTables(), unnest(tables) base_table
     WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables)
     ORDER BY base_table, z;
 $$ LANGUAGE SQL;
 
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
-AS $$
-  BEGIN
-    SELECT n.nspname, c.relname
-    INTO STRICT schema_name, table_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
 -- Calculate the estimated extent of a cartodbfy'ed table.
 -- Scope: private.
 -- Parameters
@@ -266,22 +291,28 @@ $$ LANGUAGE PLPGSQL STABLE;
 -- Parameters:
 --   reloid: oid of the input table. It must be a cartodbfy'ed table.
 -- Return value: Z level as an integer
-CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
+CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS INTEGER
 AS $$
   DECLARE
-    lim FLOAT8 := 500; -- TODO: determine/parameterize this
+    lim FLOAT8;
     nz integer := 4;
     fd FLOAT8;
     c FLOAT8;
   BEGIN
+    IF (tolerance_px IS NULL) OR tolerance_px = 0 THEN
+      lim := 500;
+    ELSE
+      lim := floor(power(256/tolerance_px, 2))/2;
+    END IF;
+
     -- Compute fd as an estimation of the (maximum) number
     -- of features per unit of tile area (in webmercator squared meters)
     SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
     -- lim maximum number of (desiderable) features per tile
     -- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference)
-    -- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z)
-    -- => fd*ta(z) if the average number of features per tile at level z
+    -- ta(z): tile area = power(c*power(2,-z), 2) = c*c*power(2,-2*z)
+    -- => fd*ta(z) is the average number of features per tile at level z
     -- find minimum z so that fd*ta(z) <= lim
     -- compute a rough 'feature density' value
     SELECT CDB_XYZ_Resolution(-8) INTO c;
@@ -321,7 +352,7 @@ $$ LANGUAGE PLPGSQL IMMUTABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS REGCLASS
 AS $$
   DECLARE
@@ -330,12 +361,16 @@ AS $$
     base_name TEXT;
     class_info RECORD;
     num_samples INTEGER;
+    schema_name TEXT;
+    table_name TEXT;
   BEGIN
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+    -- TODO: compute fraction from tolerance_px if not NULL
     fraction := power(2, 2*(overview_z - ref_z));
 
-    -- FIXME: handle schema name for overview_rel if reloid requires it
-    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    EXECUTE Format('DROP TABLE IF EXISTS %I.%I CASCADE;', schema_name.overview_rel);
 
     -- Estimate number of rows
     SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
@@ -486,6 +521,8 @@ BEGIN
     -- Using 'string_agg(' || qualified_column || ',''/'')'
     -- here causes
     RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+  WHEN 'boolean' THEN
+    RETURN 'CASE count(*) WHEN 1 THEN BOOL_AND(' || qualified_column || ') ELSE NULL END::' || column_type;
   ELSE
     RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
   END CASE;
@@ -545,19 +582,20 @@ $$ LANGUAGE PLPGSQL STABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, grid_px FLOAT8 DEFAULT NULL)
 RETURNS REGCLASS
 AS $$
   DECLARE
     overview_rel TEXT;
     reduction FLOAT8;
     base_name TEXT;
-    grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
     grid_m FLOAT8;
     aggr_attributes TEXT;
     attributes TEXT;
     columns TEXT;
     gtypes TEXT[];
+    schema_name TEXT;
+    table_name TEXT;
   BEGIN
     SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
     IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
@@ -570,6 +608,13 @@ AS $$
 
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
 
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 7.5;
+    END IF;
+
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
     -- compute grid cell size using the overview_z dimension...
     SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
 
@@ -603,8 +648,7 @@ AS $$
       SELECT * FROM cols
     ) AS s INTO columns;
 
-    -- FIXME: handle schema name for overview_rel if reloid requires it
-    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+    EXECUTE Format('DROP TABLE IF EXISTS %I.%I CASCADE;', schema_name, overview_rel);
 
     -- Now we cluster the data using a grid of size grid_m
     -- and selecte the centroid (average coordinates) of each cluster.
@@ -642,7 +686,20 @@ $$ LANGUAGE PLPGSQL;
 --                    created by the strategy must have the same columns
 --                    as the base table and in the same order.
 -- Return value: Array with the names of the generated overview tables
-CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
+CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8)'::regprocedure)
+RETURNS text[]
+AS $$
+DECLARE
+  tolerance_px FLOAT8;
+BEGIN
+  -- Use the default tolerance
+  tolerance_px := 2.0;
+  RETURN CDB_CreateOverviewsWithToleranceInPixels(reloid, tolerance_px, refscale_strategy, reduce_strategy);
+END;
+$$ LANGUAGE PLPGSQL;
+
+-- Create overviews with additional parameter to define the desired detail/tolerance in pixels
+CREATE OR REPLACE FUNCTION CDB_CreateOverviewsWithToleranceInPixels(reloid REGCLASS, tolerance_px FLOAT8, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT  '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8)'::regprocedure)
 RETURNS text[]
 AS $$
 DECLARE
@@ -655,7 +712,7 @@ DECLARE
   overviews_step integer := 1;
 BEGIN
   -- Determine the referece zoom level
-  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'', %s);', reloid, tolerance_px) INTO ref_z;
 
   -- Determine overlay zoom levels
   -- TODO: should be handled by the refscale_strategy?
@@ -669,7 +726,7 @@ BEGIN
   base_z := ref_z;
   base_rel := reloid;
   FOREACH overview_z IN ARRAY overviews_z LOOP
-    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s, %s);', base_rel, base_z, overview_z, tolerance_px) INTO base_rel;
     IF base_rel IS NULL THEN
       EXIT;
     END IF;
@@ -681,3 +738,10 @@ BEGIN
   RETURN overview_tables;
 END;
 $$ LANGUAGE PLPGSQL;
+
+-- Here are some older signatures of these functions, no longar in use.
+-- They must be droped here, after the (new) definition of the function `CDB_CreateOverviews`
+-- because that function used to contain references to them in the default argument values.
+DROP FUNCTION IF EXISTS _CDB_Feature_Density_Ref_Z_Strategy(REGCLASS);
+DROP FUNCTION IF EXISTS _CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);
+DROP FUNCTION IF EXISTS _CDB_Sampling_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);

test/CDB_OverviewsTest.sql

@@ -8,7 +8,7 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_bare_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_bare_t;
+SELECT count(*) FROM _vovw_2_base_bare_t;
 
 
 SELECT _CDB_Aggregable_Attributes_Expression('base_t'::regclass);
@@ -16,18 +16,28 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
 
 SELECT CDB_CreateOverviews('polyg_t'::regclass);
 
+SELECT CDB_CreateOverviews('column_types_t'::regclass);
+
 SELECT CDB_Overviews('base_t'::regclass);
+SELECT CDB_Overviews('"public"."base_t"'::regclass);
 SELECT CDB_Overviews(ARRAY['base_t'::regclass, 'base_bare_t'::regclass]);
 SELECT CDB_Overviews('polyg_t'::regclass);
+SELECT CDB_Overviews('column_types_t'::regclass);
 
+SELECT CDB_DropOverviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('base_bare_t'::regclass);
 SELECT CDB_DropOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
 
+SELECT CDB_CreateOverviewsWithToleranceInPixels('base_t'::regclass, 7.5);
+SELECT count(*) FROM _vovw_2_base_t;
+SELECT CDB_DropOverviews('base_t'::regclass);
+
+DROP TABLE column_types_t;
 DROP TABLE base_bare_t;
 DROP TABLE base_t;
 DROP TABLE polyg_t;

test/CDB_OverviewsTest_expect

@@ -5,40 +5,49 @@ CREATE TABLE
 INSERT 0 1114
 CREATE TABLE
 INSERT 0 5
+SELECT 1114
 
 
 
-{_vovw_5_base_bare_t,_vovw_4_base_bare_t,_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
-125
+{_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
+113
 number,int_number,name,start
 AVG(number)::double precision AS number,AVG(int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
 AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(tab.name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
-{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
-125
+{_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+113
 
+{_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(base_t,3,_vovw_3_base_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (base_bare_t,0,_vovw_0_base_bare_t)
 (base_bare_t,1,_vovw_1_base_bare_t)
 (base_bare_t,2,_vovw_2_base_bare_t)
 (base_bare_t,3,_vovw_3_base_bare_t)
-(base_bare_t,4,_vovw_4_base_bare_t)
-(base_bare_t,5,_vovw_5_base_bare_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
+(column_types_t,0,_vovw_0_column_types_t)
+(column_types_t,1,_vovw_1_column_types_t)
+(column_types_t,2,_vovw_2_column_types_t)
+(column_types_t,3,_vovw_3_column_types_t)
 
 
-ERROR:  relation "_vovw_5_base_t" does not exist
-LINE 1: SELECT count(*) FROM _vovw_5_base_t;
+
+ERROR:  relation "_vovw_2_base_t" does not exist
+LINE 1: SELECT count(*) FROM _vovw_2_base_t;
                              ^
+{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+38
+
+DROP TABLE
 DROP TABLE
 DROP TABLE
 DROP TABLE

test/overviews/fixtures.sql

@@ -2240,3 +2240,10 @@ INSERT INTO polyg_t VALUES
 (3, 'C', 'SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, ST_Transform('SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, 3857)),
 (4, 'D', 'SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, ST_Transform('SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, 3857)),
 (5, 'E', 'SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, ST_Transform('SRID=4326;POLYGON((9 40,8 39,8.5 40,9 41,9 40))'::geometry, 3857));
+
+CREATE TABLE column_types_t
+AS SELECT cartodb_id,
+          the_geom,
+          the_geom_webmercator,
+          (CASE cartodb_id % 2 WHEN 0 THEN 'f' ELSE 't' END)::boolean AS is_odd
+FROM base_bare_t;