Relase 0.17.1

Adding a new column to have the name of the cached analysis in case there is one

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.16.0
+EXTVERSION = 0.17.1
 
 SED = sed
 
@@ -67,6 +67,12 @@ UPGRADABLE = \
   0.15.0 \
   0.15.1 \
   0.16.0 \
+  0.16.1 \
+  0.16.2 \
+  0.16.3 \
+  0.16.4 \
+  0.17.0 \
+  0.17.1 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,57 @@
+0.17.1 (2016-08-16)
+-------------------
+
+* Add cache_tables column to cdb_analysis_catalog table #274.
+
+
+0.17.0 (2016-07-04)
+-------------------
+
+* Add export config for cdb_analysis_catalog table #268.
+* Add some extra fields to cdb_analysis_catalog table. Track user, error_message for failures, and last entity modifying the node #267.
+* Exclude overviews from user data size #262.
+
+
+0.16.4 (2016-05-27)
+-------------------
+
+* Change CDB_ZoomFromScale() to use a formula and raise
+  maximum overview level from 23 to 29.
+  [#259](https://github.com/CartoDB/cartodb-postgresql/pull/259)
+
+* Fix bug in overview creating causing it to fail when `x` or
+  `y` columns exist with non-integer type. Prevent also
+  potential integer overflows limiting maximum overview level
+  to 23.
+  [#258](https://github.com/CartoDB/cartodb-postgresql/pull/258)
+
+
+0.16.3 (2016-05-09)
+-------------------
+
+* Fix overview creation problem for organization users
+  with names that require quoting:
+  [#253](https://github.com/CartoDB/cartodb-postgresql/pull/253)
+
+0.16.2 (2016-04-27)
+-------------------
+
+* Use the mode to aggregate category columns in overviews
+  [#246](https://github.com/CartoDB/cartodb-postgresql/pull/246)
+
+0.16.1 (2016-04-25)
+-------------------
+
+* Optimize column information functions performance
+  [#238](https://github.com/CartoDB/cartodb-postgresql/pull/238)
+
+* Adjust overview points to pixel CDB_EqualIntervalBins
+  [#242](https://github.com/CartoDB/cartodb-postgresql/pull/242)
+
+* Compute webmercator resolution using full numeric precision
+  [#243](https://github.com/CartoDB/cartodb-postgresql/pull/243)
+
+
 0.16.0 (2016-04-15)
 -------------------
 * Adds table for storing camshaft analysis nodes

doc/CDB_Overviews.md

@@ -2,18 +2,25 @@ Overviews are tables that represent a *reduced* version of a dataset intended
 for efficient rendering at certain zoom levels while preserving the
 general visual appearance of the complete dataset.
 
-The *reduction* consists in a fewer number of records
+The *reduction* consists in havig a fewer number of records
 (while each overview record may represent an aggregation of multiple records)
 and/or simplified record geometries.
 
-Overviews are created through the `CDB_CreateOverviews`.
+Overviews are created through the `CDB_CreateOverviews` function.
 The statement timeout may need to be adjusted before using this function,
 as overview creation for large tables is a time-consuming operation.
 
 The `CDB_Overviews` function can be used determine what overview tables
 exist for a given dataset table and which zoom levels correspond to it.
 
-The `CDB_DropOverviews` remove a dataset's existing overviews.
+The `CDB_DropOverviews` function removes a dataset's existing overviews.
+
+To know if overview tables exist for some base table, and to obtain
+a list of which overview tables are approrpiate for which zoom levels,
+the `CDB_Overviews` functions can be used.
+
+The zoom level we're referring here to are those used
+by the tiler: http://wiki.openstreetmap.org/wiki/Zoom_levels
 
 ### CDB_CreateOverviews
 
@@ -51,10 +58,14 @@ CDB_CreateOverviews(table_name, ref_z_strategy, reduction_strategy)
 #### Tolerance / level of detail
 
 The level of detail to be representable by each overview layer can
-be specified as a tolerance in pixels (if different from the default of 2 pixels)
+be specified as a tolerance in pixels (if different from the default of 1 pixel)
 with the function `CDB_CreateOverviewsWithToleranceInPixels`
 which has as a second additional argument the desired tolerance.
 
+This tolerance defines the maximum deviation in pixels of the overviews
+geometries with respect to the original geometries when overview tables
+are used for their intendend zoom level.
+
 ### CDB_Overviews
 
 Obtain overview metadata for a given table (existing overviews).
@@ -79,7 +90,7 @@ SELECT CDB_Overviews(CDB_QueryTablesText('SELECT * FROM table1, table2'));
 The result of `CDB_Overviews` has three columns:
 
 | base_table | z | overview_table |
-|------------+---+----------------|
+| ---------- | - | -------------- |
 | table1     | 1 | table1_ov1     |
 | table1     | 2 | table1_ov2     |
 | table1     | 4 | table1_ov4     |

doc/cartodbfy-requirements.rst

@@ -33,7 +33,7 @@ Additionally, a CartoDB table can contain other columns.
 
 See the `CartoDB User Table documentation`_
 
-.. _CartoDB User Table documentation: https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/CartoDB-user-table.md 
+.. _CartoDB User Table documentation: https://github.com/CartoDB/cartodb-postgresql/blob/master/doc/CartoDB-user-table.rst 
 for further information.
 
 High level requirements

scripts-available/CDB_AnalysisCatalog.sql

@@ -1,6 +1,8 @@
 -- Table to register analysis nodes from https://github.com/cartodb/camshaft
 CREATE TABLE IF NOT EXISTS
 cartodb.cdb_analysis_catalog (
+    -- useful for multi account deployments
+    username text,
     -- md5 hex hash
     node_id char(40) CONSTRAINT cdb_analysis_catalog_pkey PRIMARY KEY,
     -- being json allows to do queries like analysis_def->>'type' = 'buffer'
@@ -20,5 +22,54 @@ cartodb.cdb_analysis_catalog (
     -- should register the number of times the node was used
     hits NUMERIC DEFAULT 0,
     -- should register what was the last node using current node
-    last_used_from char(40)
+    last_used_from char(40),
+    -- last job modifying the node
+    last_modified_by uuid,
+    -- store error message for failures
+    last_error_message text,
+    -- cached tables involved in the analysis
+    cache_tables regclass[] NOT NULL DEFAULT '{}'
 );
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.cdb_analysis_catalog', '');
+END
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username text;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN last_modified_by uuid;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN last_error_message text;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN cache_tables regclass[] NOT NULL DEFAULT '{}';
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;

scripts-available/CDB_ColumnNames.sql

@@ -2,15 +2,13 @@
 CREATE OR REPLACE FUNCTION CDB_ColumnNames(REGCLASS)
 RETURNS SETOF information_schema.sql_identifier
 AS $$
-
-    SELECT c.column_name
-      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
-      WHERE table_name = _tn.relname
-        AND table_schema = _sn.nspname
-        AND _tn.oid = $1::oid
-        AND _sn.oid = _tn.relnamespace
-      ORDER BY ordinal_position;
-
+  SELECT
+    a.attname::information_schema.sql_identifier column_name
+    FROM pg_class c
+         LEFT JOIN pg_attribute a ON a.attrelid = c.oid
+    WHERE c.oid = $1::oid
+    AND a.attstattarget < 0 -- exclude system columns
+   ORDER BY a.attnum;
 $$ LANGUAGE SQL;
 
 -- This is to migrate from pre-0.2.0 version

scripts-available/CDB_ColumnType.sql

@@ -2,15 +2,13 @@
 CREATE OR REPLACE FUNCTION CDB_ColumnType(REGCLASS, TEXT)
 RETURNS information_schema.character_data
 AS $$
-
-    SELECT c.data_type
-      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
-      WHERE table_name = _tn.relname
-        AND table_schema = _sn.nspname
-        AND column_name = $2
-        AND _tn.oid = $1::oid
-        AND _sn.oid = _tn.relnamespace;
-         
+  SELECT
+    format_type(a.atttypid, NULL)::information_schema.character_data data_type
+  FROM pg_class c
+       LEFT JOIN pg_attribute a ON a.attrelid = c.oid
+  WHERE c.oid = $1::oid
+  AND a.attname = $2
+  AND a.attstattarget < 0; -- exclude system columns
 $$ LANGUAGE SQL;
 
 -- This is to migrate from pre-0.2.0 version

scripts-available/CDB_Overviews.sql

@@ -1,129 +1,3 @@
--- Information about tables in a schema.
--- If the schema name parameter is NULL, then tables from all schemas
--- that may contain user tables are returned.
--- For each table, the regclass, schema name and table name are returned.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
-RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
-AS $$
-  SELECT
-    c.oid::regclass AS table_regclass,
-    n.nspname::text AS schema_name,
-    c.relname::text AS table_relname
-  FROM pg_class c
-  JOIN pg_namespace n ON n.oid = c.relnamespace
-  WHERE c.relkind = 'r'
-  AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
-  AND CASE WHEN schema_name IS NULL
-             THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
-           ELSE n.nspname = schema_name
-           END;
-$$ LANGUAGE 'sql';
-
--- Pattern that can be used to detect overview tables and Extract
--- the intended zoom level from the table name.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN '\A_vovw_(\d+)_';
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
--- substring(tablename from _CDB_OverviewTableDiscriminator())
-
-
--- Pattern matched by the overview tables of a given base table name.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN _CDB_OverviewTableDiscriminator() || base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
--- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
-
--- Name of an overview table, given the base table name and the Z level
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN '_vovw_' || z::text || '_' || base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Condition to check if a tabla is an overview table of some base table
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
-RETURNS BOOLEAN
-AS $$
-  BEGIN
-    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Extract the Z level from an overview table name
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
-RETURNS INTEGER
-AS $$
-  BEGIN
-    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Name of the base table corresponding to an overview table
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
-RETURNS TEXT
-AS $$
-  BEGIN
-    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
-      RETURN overview_table;
-    ELSE
-      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
-    END IF;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
-AS $$
-  BEGIN
-    SELECT n.nspname, c.relname
-    INTO STRICT schema_name, table_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
-RETURNS TEXT
-AS $$
-  DECLARE
-    schema_name TEXT;
-  BEGIN
-    SELECT n.nspname
-    INTO STRICT schema_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-    RETURN schema_name;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
 -- Remove a dataset's existing  overview tables.
 -- Scope: public
 -- Parameters:
@@ -215,7 +89,7 @@ AS $$
       FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid;
 
     ext_query = format(
-      'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');',
+      'SELECT ST_EstimatedExtent(''%1$s'', ''%2$s'', ''%3$s'');',
       table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
     );
 
@@ -277,7 +151,11 @@ AS $$
     WITH RECURSIVE t(x, y, z, e) AS (
       WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g),
       base AS (
-        SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z
+        SELECT
+          least(
+           -floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)),
+           _CDB_MaxOverviewLevel()+1
+          )::integer z
         FROM ext
       ),
       lim AS (
@@ -299,10 +177,10 @@ AS $$
       UNION ALL
       SELECT x*2 + xx, y*2 + yy, t.z+1, (
         SELECT count(*) FROM %1$s
-          WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1)
+          WHERE the_geom_webmercator && CDB_XYZ_Extent(t.x*2 + c.xx, t.y*2 + c.yy, t.z+1)
       )
       FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy)
-      WHERE t.e > %2$s AND t.z < (base.z + %3$s)
+      WHERE t.e > %2$s AND t.z < least(base.z + %3$s, _CDB_MaxZoomLevel())
     )
     SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0;
   ', reloid::text, min_features, nz, n, c, reloid::oid)
@@ -343,7 +221,7 @@ AS $$
     -- find minimum z so that fd*ta(z) <= lim
     -- compute a rough 'feature density' value
     SELECT CDB_XYZ_Resolution(-8) INTO c;
-    RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2);
+    RETURN least(_CDB_MaxOverviewLevel()+1, ceil(log(2.0, (c*c*fd/lim)::numeric)/2));
   END;
 $$ LANGUAGE PLPGSQL STABLE;
 
@@ -520,18 +398,65 @@ CREATE OR REPLACE FUNCTION _cdb_unlimited_text_column(reloid REGCLASS, col_name
 RETURNS BOOLEAN
 AS $$
   SELECT EXISTS (
-    SELECT  *
-      FROM information_schema.columns c, pg_class _tn, pg_namespace _sn
-      WHERE table_name = _tn.relname
-        AND table_schema = _sn.nspname
-        AND c.column_name = col_name
-        AND _tn.oid = reloid
-        AND _sn.oid = _tn.relnamespace
-        AND character_maximum_length IS NULL
-        AND c.data_type IN ('text', 'character varying', 'character')
+    SELECT a.attname
+    FROM pg_class c
+         LEFT JOIN pg_attribute a ON a.attrelid = c.oid
+         LEFT JOIN pg_type t ON t.oid = a.atttypid
+    WHERE c.oid = reloid
+      AND a.attname = col_name
+      AND format_type(a.atttypid, NULL) IN ('text', 'character varying', 'character')
+      AND format_type(a.atttypid, NULL) = format_type(a.atttypid, a.atttypmod)
   );
 $$ LANGUAGE SQL STABLE;
 
+CREATE OR REPLACE FUNCTION _cdb_categorical_column(reloid REGCLASS, col_name TEXT)
+RETURNS BOOLEAN
+AS $$
+DECLARE
+    schema_name TEXT;
+    table_name TEXT;
+    available BOOLEAN;
+    categorical BOOLEAN;
+BEGIN
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+    SELECT n_distinct IS NOT NULL
+    FROM pg_stats
+    WHERE pg_stats.schemaname = schema_name
+      AND pg_stats.tablename = table_name
+      AND pg_stats.attname = col_name
+    INTO available;
+    IF available IS NULL OR NOT available THEN
+      EXECUTE Format('ANALYZE %s;', reloid);
+    END IF;
+    SELECT n_distinct > 0 AND n_distinct <= 20
+    FROM pg_stats
+    WHERE pg_stats.schemaname = schema_name
+      AND pg_stats.tablename = table_name
+      AND pg_stats.attname = col_name
+    INTO categorical;
+    RETURN categorical;
+END;
+$$ LANGUAGE PLPGSQL VOLATILE;
+
+CREATE OR REPLACE FUNCTION _cdb_mode_of_array(anyarray)
+  RETURNS anyelement AS
+$$
+    SELECT a
+    FROM unnest($1) a
+    GROUP BY 1
+    ORDER BY COUNT(1) DESC, 1
+    LIMIT 1;
+$$
+LANGUAGE SQL IMMUTABLE;
+
+DROP AGGREGATE IF EXISTS _cdb_mode(anyelement);
+CREATE AGGREGATE _cdb_mode(anyelement) (
+  SFUNC=array_append,
+  STYPE=anyarray,
+  FINALFUNC=_cdb_mode_of_array,
+  INITCOND='{}'
+);
+
 -- SQL Aggregation expression for a datase attribute
 -- Scope: private.
 -- Parameters
@@ -549,6 +474,7 @@ DECLARE
   has_counter_column BOOLEAN;
   feature_count TEXT;
   total_feature_count TEXT;
+  base_table REGCLASS;
 BEGIN
   IF table_alias <> '' THEN
     qualified_column := Format('%I.%I', table_alias, column_name);
@@ -569,20 +495,30 @@ BEGIN
     total_feature_count := 'count(*)';
   END IF;
 
+  base_table := _CDB_OverviewBaseTable(reloid);
+
   CASE column_type
   WHEN 'double precision', 'real', 'integer', 'bigint', 'numeric' THEN
     IF column_name = '_feature_count' THEN
       RETURN 'SUM(_feature_count)';
     ELSE
-      RETURN Format('SUM(%s*%s)/%s::' || column_type, qualified_column, feature_count, total_feature_count);
+      IF column_type = 'integer' AND _cdb_categorical_column(base_table, column_name) THEN
+        RETURN Format('CDB_Math_Mode(%s)::', qualified_column) || column_type;
+      ELSE
+        RETURN Format('SUM(%s*%s)/%s::' || column_type, qualified_column, feature_count, total_feature_count);
+      END IF;
     END IF;
   WHEN 'text', 'character varying', 'character' THEN
-    IF _cdb_unlimited_text_column(reloid, column_name) THEN
-      -- TODO: this should not be applied to columns containing largish text;
-      -- it is intended only to short names/identifiers
-      RETURN  'CASE WHEN count(distinct ' || qualified_column || ') = 1 THEN MIN(' || qualified_column || ') WHEN ' || total_feature_count || ' < 5 THEN string_agg(distinct ' || qualified_column || ','' / '') ELSE ''*'' END::' || column_type;
+    IF _cdb_categorical_column(base_table, column_name) THEN
+      RETURN Format('_cdb_mode(%s)::', qualified_column) || column_type;
     ELSE
-      RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+      IF _cdb_unlimited_text_column(base_table, column_name) THEN
+        -- TODO: this should not be applied to columns containing largish text;
+        -- it is intended only to short names/identifiers
+        RETURN  'CASE WHEN count(distinct ' || qualified_column || ') = 1 THEN MIN(' || qualified_column || ') WHEN ' || total_feature_count || ' < 5 THEN string_agg(distinct ' || qualified_column || ','' / '') ELSE ''*'' END::' || column_type;
+      ELSE
+        RETURN 'CASE count(*) WHEN 1 THEN MIN(' || qualified_column || ') ELSE NULL END::' || column_type;
+      END IF;
     END IF;
   WHEN 'boolean' THEN
     RETURN 'CASE count(*) WHEN 1 THEN BOOL_AND(' || qualified_column || ') ELSE NULL END::' || column_type;
@@ -652,7 +588,13 @@ AS $$
     overview_rel TEXT;
     reduction FLOAT8;
     base_name TEXT;
+    pixel_m FLOAT8;
     grid_m FLOAT8;
+    offset_m FLOAT8;
+    offset_x TEXT;
+    offset_y TEXT;
+    cell_x TEXT;
+    cell_y TEXT;
     aggr_attributes TEXT;
     attributes TEXT;
     columns TEXT;
@@ -678,8 +620,10 @@ AS $$
 
     SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
 
-    -- compute grid cell size using the overview_z dimension...
-    SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
+    -- pixel_m: size of a pixel in webmercator units (meters)
+    SELECT CDB_XYZ_Resolution(overview_z) INTO pixel_m;
+    -- grid size in meters
+    grid_m = grid_px * pixel_m;
 
     attributes := _CDB_Aggregable_Attributes_Expression(reloid);
     aggr_attributes := _CDB_Aggregated_Attributes_Expression(reloid);
@@ -690,7 +634,21 @@ AS $$
       aggr_attributes := aggr_attributes || ', ';
     END IF;
 
-    point_geom = Format('ST_SetSRID(ST_MakePoint(gx*%1$s + %2$s, gy*%1$s + %2$s), 3857)', grid_m, grid_m/2);
+    -- Center of each cell:
+    cell_x := Format('gx*%1$s + %2$s', grid_m, grid_m/2);
+    cell_y := Format('gy*%1$s + %2$s', grid_m, grid_m/2);
+
+    -- Displacement to the nearest pixel center:
+    IF MOD(grid_px::numeric, 1.0::numeric) = 0 THEN
+      offset_m := pixel_m/2 - MOD((grid_m/2)::numeric, pixel_m::numeric)::float8;
+      offset_x := Format('%s', offset_m);
+      offset_y := Format('%s', offset_m);
+    ELSE
+      offset_x := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_x, pixel_m);
+      offset_y := Format('%2$s/2 - MOD((%1$s)::numeric, (%2$s)::numeric)::float8', cell_y, pixel_m);
+    END IF;
+
+    point_geom := Format('ST_SetSRID(ST_MakePoint(%1$s + %3$s, %2$s + %4$s), 3857)', cell_x, cell_y, offset_x, offset_y);
 
     -- compute the resulting columns in the same order as in the base table
     WITH cols AS (

scripts-available/CDB_OverviewsSupport.sql

@@ -0,0 +1,173 @@
+-- Auxiliary overviews FUNCTIONS
+
+-- Maximum zoom level for which overviews may be created
+CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel()
+RETURNS INTEGER
+AS $$
+  BEGIN
+    -- Zoom level will be limited so that both tile coordinates
+    -- and gridding coordinates within a tile up to 1px
+    -- (i.e. tile coordinates / 256)
+    -- can be stored in a 32-bit signed integer.
+    -- We have 31 bits por positive numbers
+    -- For zoom level Z coordinates range from 0 to 2^Z-1, so they
+    -- need Z bits, and need 8 bits more to address pixels within a tile
+    -- (gridding), so we'll limit Z to a maximum of 31 - 8
+    RETURN 23;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Maximum zoom level usable with integer coordinates
+CREATE OR REPLACE FUNCTION _CDB_MaxZoomLevel()
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN 31;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Information about tables in a schema.
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT
+    c.oid::regclass AS table_regclass,
+    n.nspname::text AS schema_name,
+    c.relname::text AS table_relname
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
+  AND CASE WHEN schema_name IS NULL
+             THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
+           ELSE n.nspname = schema_name
+           END;
+$$ LANGUAGE 'sql';
+
+-- Pattern that can be used to detect overview tables and Extract
+-- the intended zoom level from the table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '\A_vovw_(\d+)_';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- substring(tablename from _CDB_OverviewTableDiscriminator())
+
+
+-- Pattern matched by the overview tables of a given base table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN _CDB_OverviewTableDiscriminator() || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
+
+-- Name of an overview table, given the base table name and the Z level
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '_vovw_' || z::text || '_' || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Condition to check if a tabla is an overview table of some base table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Extract the Z level from an overview table name
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Name of the base table corresponding to an overview table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
+      RETURN overview_table;
+    ELSE
+      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
+    END IF;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTable(overview_table REGCLASS)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    table_name TEXT;
+    schema_name TEXT;
+    base_name TEXT;
+    base_table REGCLASS;
+  BEGIN
+    SELECT * FROM _cdb_split_table_name(overview_table) INTO schema_name, table_name;
+    base_name := _CDB_OverviewBaseTableName(table_name);
+    IF base_name != table_name THEN
+      base_table := Format('%I.%I', schema_name, base_name)::regclass;
+    ELSE
+      base_table := overview_table;
+    END IF;
+    RETURN base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
+AS $$
+  BEGIN
+    SELECT n.nspname, c.relname
+    INTO STRICT schema_name, table_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
+RETURNS TEXT
+AS $$
+  DECLARE
+    schema_name TEXT;
+  BEGIN
+    SELECT n.nspname
+    INTO STRICT schema_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+    RETURN schema_name;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;

scripts-available/CDB_Quota.sql

@@ -35,7 +35,11 @@ BEGIN
   table_cat AS (
     SELECT
       table_name,
-      EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
+      (
+        EXISTS(select * from raster_tables where o_table_name = table_name)
+        OR
+        table_name SIMILAR TO _CDB_OverviewTableDiscriminator() || '[\w\d]*'
+      ) AS is_overview,
       EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
     FROM user_tables
   ),

scripts-available/CDB_XYZ.sql

@@ -6,7 +6,7 @@ CREATE OR REPLACE FUNCTION CDB_XYZ_Resolution(z INTEGER)
 RETURNS FLOAT8
 AS $$
   -- circumference divided by 256 is z0 resolution, then divide by 2^z
-  SELECT 40075017.0 / 256 / power(2, z);
+  SELECT 6378137.0*2.0*pi() / 256.0 / power(2.0, z);
 $$ LANGUAGE SQL IMMUTABLE STRICT;
 -- }
 

scripts-available/CDB_ZoomFromScale.sql

@@ -1,30 +1,36 @@
-CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
-BEGIN
-  CASE
-    WHEN scaleDenominator > 500000000 THEN RETURN 0;
-    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
-    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
-    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
-    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
-    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
-    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
-    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
-    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
-    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
-    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
-    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
-    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
-    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
-    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
-    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
-    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
-    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
-    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
-    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
-    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
-    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
-    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
-    WHEN scaleDenominator <= 100 THEN RETURN 23;
-  END CASE;
-END
-$$ LANGUAGE plpgsql IMMUTABLE;
+-- Maximum supported zoom level
+CREATE OR REPLACE FUNCTION _CDB_MaxSupportedZoom()
+RETURNS int
+LANGUAGE SQL
+IMMUTABLE
+AS $$
+  -- The maximum zoom level has to be limited for various reasons,
+  -- e.g. zoom levels greater than 31 would require tile coordinates
+  -- that would not fit in an INTEGER (which is signed, 32 bits long).
+  -- We'll choose 20 as a limit which is safe also when the JavaScript shift
+  -- operator (<<) is used for computing powers of two.
+  SELECT 29;
+$$;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric)
+RETURNS int
+LANGUAGE SQL
+IMMUTABLE
+AS $$
+  SELECT
+    CASE
+      WHEN scaleDenominator > 600000000 THEN
+        -- Scale is smaller than zoom level 0
+        NULL
+      WHEN scaleDenominator = 0 THEN
+        -- Actual zoom level would be infinite
+        _CDB_MaxSupportedZoom()
+      ELSE
+        CAST (
+          LEAST(
+            ROUND(LOG(2, 559082264.028/scaleDenominator)),
+            _CDB_MaxSupportedZoom()
+          )
+        AS INTEGER)
+    END;
+$$;

scripts-enabled/085-CDB_OverviewsSupport.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_OverviewsSupport.sql

test/extension/run_at_cartodb_schema.sql

@@ -4,4 +4,4 @@ SET SCHEMA 'cartodb';
 \i scripts-available/CDB_ColumnNames.sql
 \i scripts-available/CDB_ColumnType.sql
 \i scripts-available/CDB_AnalysisCatalog.sql
-SET SCHEMA 'public';
+SET SCHEMA 'public';

test/extension/test.sh

@@ -181,6 +181,7 @@ function setup_database() {
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_OverviewsSupport.sql
     # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
     ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
 }