Release 0.15.0

This release includes some overviews enhancements

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.14.4
+EXTVERSION = 0.15.0
 
 SED = sed
 
@@ -63,7 +63,8 @@ UPGRADABLE = \
   0.14.1 \
   0.14.2 \
   0.14.3 \
-	0.14.4 \
+  0.14.4 \
+  0.15.0 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,4 +1,14 @@
+0.15.0 (2016-04-05)
+
+* New function CDB_CreateOverviewsWithToleranceInPixels that adds tolerance parameter for overview creation
+  [#221](https://github.com/CartoDB/cartodb-postgresql/pull/221)
+* New default value for the overviews tolerance in pixels is 2 (used to be 7.5) (also in #221)
+* The feature density limit used to choose the reference Z level now depends on the tolerance in pixels (also in #221)
+* Tables that require an explicit schema can now be passed to overview functions
+  [#220](https://github.com/CartoDB/cartodb-postgresql/pull/220)
+
 0.14.4 (2016-03-29)
+
 * Fix creating overviews for tables with boolean columns
   [#214](https://github.com/CartoDB/cartodb-postgresql/pull/214)
 * Fix tests for some systems [#215](https://github.com/CartoDB/cartodb-postgresql/pull/215)

doc/CDB_Overviews.md

@@ -48,6 +48,13 @@ CDB_CreateOverviews(table_name, ref_z_strategy, reduction_strategy)
   - **base_z** integer, base Z level assigned to the base table.
   - **overview_z** integer, Z level for which to generate the overview.
 
+#### Tolerance / level of detail
+
+The level of detail to be representable by each overview layer can
+be specified as a tolerance in pixels (if different from the default of 2 pixels)
+with the function `CDB_CreateOverviewsWithToleranceInPixels`
+which has as a second additional argument the desired tolerance.
+
 ### CDB_Overviews
 
 Obtain overview metadata for a given table (existing overviews).

scripts-available/CDB_Overviews.sql

@@ -68,6 +68,41 @@ AS $$
   END;
 $$ LANGUAGE PLPGSQL IMMUTABLE;
 
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
+AS $$
+  BEGIN
+    SELECT n.nspname, c.relname
+    INTO STRICT schema_name, table_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
+RETURNS TEXT
+AS $$
+  DECLARE
+    schema_name TEXT;
+  BEGIN
+    SELECT n.nspname
+    INTO STRICT schema_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+    RETURN schema_name;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
 
 -- Remove a dataset's existing  overview tables.
 -- Scope: public
@@ -77,12 +112,15 @@ CREATE OR REPLACE FUNCTION CDB_DropOverviews(reloid REGCLASS)
 RETURNS void
 AS $$
 DECLARE
-    row     record;
+    row record;
+    schema_name TEXT;
+    table_name TEXT;
 BEGIN
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
     FOR row IN
         SELECT * FROM CDB_Overviews(reloid)
     LOOP
-        EXECUTE Format('DROP TABLE %s;', row.overview_table);
+        EXECUTE Format('DROP TABLE %I.%I;', schema_name, row.overview_table);
         RAISE NOTICE 'Dropped overview for level %: %', row.z, row.overview_table;
     END LOOP;
 END;
@@ -100,18 +138,21 @@ $$ LANGUAGE PLPGSQL VOLATILE;
 CREATE OR REPLACE FUNCTION CDB_Overviews(reloid REGCLASS)
 RETURNS TABLE(base_table REGCLASS, z integer, overview_table REGCLASS)
 AS $$
-  -- FIXME: this will fail if the overview tables
-  -- require a explicit schema name
-  -- possible solutions: return table names as text instead of regclass
-  -- or add schema of reloid before casting to regclass
-  SELECT
-    reloid AS base_table,
-    _CDB_OverviewTableZ(cdb_usertables) AS z,
-    cdb_usertables::regclass AS overview_table
-    FROM CDB_UserTables()
-    WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
-    ORDER BY z;
-$$ LANGUAGE SQL;
+  DECLARE
+    schema_name TEXT;
+    table_name TEXT;
+  BEGIN
+    -- TODO: review implementation of CDB_UserTables an suitability for this
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+    RETURN QUERY SELECT
+      reloid AS base_table,
+      _CDB_OverviewTableZ(cdb_usertables) AS z,
+      ('"' || schema_name|| '"."' ||cdb_usertables || '"')::regclass AS overview_table
+      FROM CDB_UserTables()
+      WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=reloid), cdb_usertables)
+      ORDER BY z;
+  END
+$$ LANGUAGE PLPGSQL;
 
 -- Return existing overviews (if any) for multiple dataset tables.
 -- Scope: public
@@ -128,29 +169,13 @@ AS $$
   SELECT
     base_table::regclass AS base_table,
     _CDB_OverviewTableZ(cdb_usertables) AS z,
-    cdb_usertables::regclass AS overview_table
+    ('"' || _cdb_schema_name(base_table::regclass) || '"."' || cdb_usertables || '"')::regclass AS overview_table
     FROM
       CDB_UserTables(), unnest(tables) base_table
     WHERE _CDB_IsOverviewTableOf((SELECT relname FROM pg_class WHERE oid=base_table), cdb_usertables)
     ORDER BY base_table, z;
 $$ LANGUAGE SQL;
 
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
-AS $$
-  BEGIN
-    SELECT n.nspname, c.relname
-    INTO STRICT schema_name, table_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
 -- Calculate the estimated extent of a cartodbfy'ed table.
 -- Scope: private.
 -- Parameters
@@ -266,22 +291,28 @@ $$ LANGUAGE PLPGSQL STABLE;
 -- Parameters:
 --   reloid: oid of the input table. It must be a cartodbfy'ed table.
 -- Return value: Z level as an integer
-CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
+CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS INTEGER
 AS $$
   DECLARE
-    lim FLOAT8 := 500; -- TODO: determine/parameterize this
+    lim FLOAT8;
     nz integer := 4;
     fd FLOAT8;
     c FLOAT8;
   BEGIN
+    IF (tolerance_px IS NULL) OR tolerance_px = 0 THEN
+      lim := 500;
+    ELSE
+      lim := floor(power(256/tolerance_px, 2))/2;
+    END IF;
+
     -- Compute fd as an estimation of the (maximum) number
     -- of features per unit of tile area (in webmercator squared meters)
     SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
     -- lim maximum number of (desiderable) features per tile
     -- we have c = 2*Pi*R = CDB_XYZ_Resolution(-8) (earth circumference)
-    -- ta(z): tile area = power(c*power(2,z), 2) = c*c*power(2,2*z)
-    -- => fd*ta(z) if the average number of features per tile at level z
+    -- ta(z): tile area = power(c*power(2,-z), 2) = c*c*power(2,-2*z)
+    -- => fd*ta(z) is the average number of features per tile at level z
     -- find minimum z so that fd*ta(z) <= lim
     -- compute a rough 'feature density' value
     SELECT CDB_XYZ_Resolution(-8) INTO c;
@@ -321,7 +352,7 @@ $$ LANGUAGE PLPGSQL IMMUTABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS REGCLASS
 AS $$
   DECLARE
@@ -330,12 +361,16 @@ AS $$
     base_name TEXT;
     class_info RECORD;
     num_samples INTEGER;
+    schema_name TEXT;
+    table_name TEXT;
   BEGIN
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+    -- TODO: compute fraction from tolerance_px if not NULL
     fraction := power(2, 2*(overview_z - ref_z));
 
-    -- FIXME: handle schema name for overview_rel if reloid requires it
-    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
+    EXECUTE Format('DROP TABLE IF EXISTS %I.%I CASCADE;', schema_name.overview_rel);
 
     -- Estimate number of rows
     SELECT reltuples, relpages FROM pg_class INTO STRICT class_info
@@ -547,19 +582,20 @@ $$ LANGUAGE PLPGSQL STABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, grid_px FLOAT8 DEFAULT NULL)
 RETURNS REGCLASS
 AS $$
   DECLARE
     overview_rel TEXT;
     reduction FLOAT8;
     base_name TEXT;
-    grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
     grid_m FLOAT8;
     aggr_attributes TEXT;
     attributes TEXT;
     columns TEXT;
     gtypes TEXT[];
+    schema_name TEXT;
+    table_name TEXT;
   BEGIN
     SELECT _CDB_GeometryTypes(reloid) INTO gtypes;
     IF array_upper(gtypes, 1) <> 1 OR gtypes[1] <> 'ST_Point' THEN
@@ -572,6 +608,13 @@ AS $$
 
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
 
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 7.5;
+    END IF;
+
+    SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
+
     -- compute grid cell size using the overview_z dimension...
     SELECT CDB_XYZ_Resolution(overview_z)*grid_px INTO grid_m;
 
@@ -605,8 +648,7 @@ AS $$
       SELECT * FROM cols
     ) AS s INTO columns;
 
-    -- FIXME: handle schema name for overview_rel if reloid requires it
-    EXECUTE Format('DROP TABLE IF EXISTS %I CASCADE;', overview_rel);
+    EXECUTE Format('DROP TABLE IF EXISTS %I.%I CASCADE;', schema_name, overview_rel);
 
     -- Now we cluster the data using a grid of size grid_m
     -- and selecte the centroid (average coordinates) of each cluster.
@@ -644,7 +686,20 @@ $$ LANGUAGE PLPGSQL;
 --                    created by the strategy must have the same columns
 --                    as the base table and in the same order.
 -- Return value: Array with the names of the generated overview tables
-CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
+CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8)'::regprocedure)
+RETURNS text[]
+AS $$
+DECLARE
+  tolerance_px FLOAT8;
+BEGIN
+  -- Use the default tolerance
+  tolerance_px := 2.0;
+  RETURN CDB_CreateOverviewsWithToleranceInPixels(reloid, tolerance_px, refscale_strategy, reduce_strategy);
+END;
+$$ LANGUAGE PLPGSQL;
+
+-- Create overviews with additional parameter to define the desired detail/tolerance in pixels
+CREATE OR REPLACE FUNCTION CDB_CreateOverviewsWithToleranceInPixels(reloid REGCLASS, tolerance_px FLOAT8, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT  '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8)'::regprocedure)
 RETURNS text[]
 AS $$
 DECLARE
@@ -657,7 +712,7 @@ DECLARE
   overviews_step integer := 1;
 BEGIN
   -- Determine the referece zoom level
-  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'', %s);', reloid, tolerance_px) INTO ref_z;
 
   -- Determine overlay zoom levels
   -- TODO: should be handled by the refscale_strategy?
@@ -671,7 +726,7 @@ BEGIN
   base_z := ref_z;
   base_rel := reloid;
   FOREACH overview_z IN ARRAY overviews_z LOOP
-    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s, %s);', base_rel, base_z, overview_z, tolerance_px) INTO base_rel;
     IF base_rel IS NULL THEN
       EXIT;
     END IF;
@@ -683,3 +738,10 @@ BEGIN
   RETURN overview_tables;
 END;
 $$ LANGUAGE PLPGSQL;
+
+-- Here are some older signatures of these functions, no longar in use.
+-- They must be droped here, after the (new) definition of the function `CDB_CreateOverviews`
+-- because that function used to contain references to them in the default argument values.
+DROP FUNCTION IF EXISTS _CDB_Feature_Density_Ref_Z_Strategy(REGCLASS);
+DROP FUNCTION IF EXISTS _CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);
+DROP FUNCTION IF EXISTS _CDB_Sampling_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);

test/CDB_OverviewsTest.sql

@@ -8,7 +8,7 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_bare_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_bare_t;
+SELECT count(*) FROM _vovw_2_base_bare_t;
 
 
 SELECT _CDB_Aggregable_Attributes_Expression('base_t'::regclass);
@@ -16,13 +16,14 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
 
 SELECT CDB_CreateOverviews('polyg_t'::regclass);
 
 SELECT CDB_CreateOverviews('column_types_t'::regclass);
 
 SELECT CDB_Overviews('base_t'::regclass);
+SELECT CDB_Overviews('"public"."base_t"'::regclass);
 SELECT CDB_Overviews(ARRAY['base_t'::regclass, 'base_bare_t'::regclass]);
 SELECT CDB_Overviews('polyg_t'::regclass);
 SELECT CDB_Overviews('column_types_t'::regclass);
@@ -30,7 +31,11 @@ SELECT CDB_Overviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('base_bare_t'::regclass);
 SELECT CDB_DropOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
+
+SELECT CDB_CreateOverviewsWithToleranceInPixels('base_t'::regclass, 7.5);
+SELECT count(*) FROM _vovw_2_base_t;
+SELECT CDB_DropOverviews('base_t'::regclass);
 
 DROP TABLE column_types_t;
 DROP TABLE base_bare_t;

test/CDB_OverviewsTest_expect

@@ -9,45 +9,44 @@ SELECT 1114
 
 
 
-{_vovw_5_base_bare_t,_vovw_4_base_bare_t,_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
-125
+{_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
+113
 number,int_number,name,start
 AVG(number)::double precision AS number,AVG(int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
 AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(tab.name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
-{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
-125
+{_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+113
 
-{_vovw_5_column_types_t,_vovw_4_column_types_t,_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+{_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+(base_t,0,_vovw_0_base_t)
+(base_t,1,_vovw_1_base_t)
+(base_t,2,_vovw_2_base_t)
+(base_t,3,_vovw_3_base_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (base_bare_t,0,_vovw_0_base_bare_t)
 (base_bare_t,1,_vovw_1_base_bare_t)
 (base_bare_t,2,_vovw_2_base_bare_t)
 (base_bare_t,3,_vovw_3_base_bare_t)
-(base_bare_t,4,_vovw_4_base_bare_t)
-(base_bare_t,5,_vovw_5_base_bare_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (column_types_t,0,_vovw_0_column_types_t)
 (column_types_t,1,_vovw_1_column_types_t)
 (column_types_t,2,_vovw_2_column_types_t)
 (column_types_t,3,_vovw_3_column_types_t)
-(column_types_t,4,_vovw_4_column_types_t)
-(column_types_t,5,_vovw_5_column_types_t)
 
 
 
-ERROR:  relation "_vovw_5_base_t" does not exist
-LINE 1: SELECT count(*) FROM _vovw_5_base_t;
+ERROR:  relation "_vovw_2_base_t" does not exist
+LINE 1: SELECT count(*) FROM _vovw_2_base_t;
                              ^
+{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+38
+
 DROP TABLE
 DROP TABLE
 DROP TABLE