Merge pull request #221 from CartoDB/219-overviews-tolerance

New function CDB_CreateOverviewsWithToleranceInPixels

doc/CDB_Overviews.md

@@ -48,6 +48,13 @@ CDB_CreateOverviews(table_name, ref_z_strategy, reduction_strategy)
   - **base_z** integer, base Z level assigned to the base table.
   - **overview_z** integer, Z level for which to generate the overview.
 
+#### Tolerance / level of detail
+
+The level of detail to be representable by each overview layer can
+be specified as a tolerance in pixels (if different from the default of 2 pixels)
+with the function `CDB_CreateOverviewsWithToleranceInPixels`
+which has as a second additional argument the desired tolerance.
+
 ### CDB_Overviews
 
 Obtain overview metadata for a given table (existing overviews).

scripts-available/CDB_Overviews.sql

@@ -291,15 +291,21 @@ $$ LANGUAGE PLPGSQL STABLE;
 -- Parameters:
 --   reloid: oid of the input table. It must be a cartodbfy'ed table.
 -- Return value: Z level as an integer
-CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS)
+CREATE OR REPLACE FUNCTION _CDB_Feature_Density_Ref_Z_Strategy(reloid REGCLASS, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS INTEGER
 AS $$
   DECLARE
-    lim FLOAT8 := 500; -- TODO: determine/parameterize this
+    lim FLOAT8;
     nz integer := 4;
     fd FLOAT8;
     c FLOAT8;
   BEGIN
+    IF (tolerance_px IS NULL) OR tolerance_px = 0 THEN
+      lim := 500;
+    ELSE
+      lim := floor(power(256/tolerance_px, 2))/2;
+    END IF;
+
     -- Compute fd as an estimation of the (maximum) number
     -- of features per unit of tile area (in webmercator squared meters)
     SELECT _CDB_Feature_Density(reloid, nz) INTO fd;
@@ -346,7 +352,7 @@ $$ LANGUAGE PLPGSQL IMMUTABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_Sampling_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, tolerance_px FLOAT8 DEFAULT NULL)
 RETURNS REGCLASS
 AS $$
   DECLARE
@@ -359,6 +365,7 @@ AS $$
     table_name TEXT;
   BEGIN
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
+    -- TODO: compute fraction from tolerance_px if not NULL
     fraction := power(2, 2*(overview_z - ref_z));
 
     SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
@@ -575,14 +582,13 @@ $$ LANGUAGE PLPGSQL STABLE;
 --   ref_z Z level assigned to the original table
 --   overview_z Z level of the overview to be generated, must be smaller than ref_z
 -- Return value: Name of the generated overview table
-CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER)
+CREATE OR REPLACE FUNCTION _CDB_GridCluster_Reduce_Strategy(reloid REGCLASS, ref_z INTEGER, overview_z INTEGER, grid_px FLOAT8 DEFAULT NULL)
 RETURNS REGCLASS
 AS $$
   DECLARE
     overview_rel TEXT;
     reduction FLOAT8;
     base_name TEXT;
-    grid_px FLOAT8 = 7.5; -- Grid size in pixels at Z level overview_z
     grid_m FLOAT8;
     aggr_attributes TEXT;
     attributes TEXT;
@@ -602,6 +608,11 @@ AS $$
 
     overview_rel := _CDB_Overview_Name(reloid, ref_z, overview_z);
 
+    -- Grid size in pixels at Z level overview_z
+    IF grid_px IS NULL THEN
+      grid_px := 7.5;
+    END IF;
+
     SELECT * FROM _cdb_split_table_name(reloid) INTO schema_name, table_name;
 
     -- compute grid cell size using the overview_z dimension...
@@ -675,7 +686,20 @@ $$ LANGUAGE PLPGSQL;
 --                    created by the strategy must have the same columns
 --                    as the base table and in the same order.
 -- Return value: Array with the names of the generated overview tables
-CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy'::regproc, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy'::regproc)
+CREATE OR REPLACE FUNCTION CDB_CreateOverviews(reloid REGCLASS, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8)'::regprocedure)
+RETURNS text[]
+AS $$
+DECLARE
+  tolerance_px FLOAT8;
+BEGIN
+  -- Use the default tolerance
+  tolerance_px := 2.0;
+  RETURN CDB_CreateOverviewsWithToleranceInPixels(reloid, tolerance_px, refscale_strategy, reduce_strategy);
+END;
+$$ LANGUAGE PLPGSQL;
+
+-- Create overviews with additional parameter to define the desired detail/tolerance in pixels
+CREATE OR REPLACE FUNCTION CDB_CreateOverviewsWithToleranceInPixels(reloid REGCLASS, tolerance_px FLOAT8, refscale_strategy regproc DEFAULT '_CDB_Feature_Density_Ref_Z_Strategy(REGCLASS,FLOAT8)'::regprocedure, reduce_strategy regproc DEFAULT  '_CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER,FLOAT8)'::regprocedure)
 RETURNS text[]
 AS $$
 DECLARE
@@ -688,7 +712,7 @@ DECLARE
   overviews_step integer := 1;
 BEGIN
   -- Determine the referece zoom level
-  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'');', reloid) INTO ref_z;
+  EXECUTE 'SELECT ' || quote_ident(refscale_strategy::text) || Format('(''%s'', %s);', reloid, tolerance_px) INTO ref_z;
 
   -- Determine overlay zoom levels
   -- TODO: should be handled by the refscale_strategy?
@@ -702,7 +726,7 @@ BEGIN
   base_z := ref_z;
   base_rel := reloid;
   FOREACH overview_z IN ARRAY overviews_z LOOP
-    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s);', base_rel, base_z, overview_z) INTO base_rel;
+    EXECUTE 'SELECT ' || quote_ident(reduce_strategy::text) || Format('(''%s'', %s, %s, %s);', base_rel, base_z, overview_z, tolerance_px) INTO base_rel;
     IF base_rel IS NULL THEN
       EXIT;
     END IF;
@@ -714,3 +738,10 @@ BEGIN
   RETURN overview_tables;
 END;
 $$ LANGUAGE PLPGSQL;
+
+-- Here are some older signatures of these functions, no longar in use.
+-- They must be droped here, after the (new) definition of the function `CDB_CreateOverviews`
+-- because that function used to contain references to them in the default argument values.
+DROP FUNCTION IF EXISTS _CDB_Feature_Density_Ref_Z_Strategy(REGCLASS);
+DROP FUNCTION IF EXISTS _CDB_GridCluster_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);
+DROP FUNCTION IF EXISTS _CDB_Sampling_Reduce_Strategy(REGCLASS,INTEGER,INTEGER);

test/CDB_OverviewsTest.sql

@@ -8,7 +8,7 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_bare_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_bare_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_bare_t;
+SELECT count(*) FROM _vovw_2_base_bare_t;
 
 
 SELECT _CDB_Aggregable_Attributes_Expression('base_t'::regclass);
@@ -16,7 +16,7 @@ SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass);
 SELECT _CDB_Aggregated_Attributes_Expression('base_t'::regclass, 'tab');
 
 SELECT CDB_CreateOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
 
 SELECT CDB_CreateOverviews('polyg_t'::regclass);
 
@@ -31,7 +31,11 @@ SELECT CDB_Overviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('column_types_t'::regclass);
 SELECT CDB_DropOverviews('base_bare_t'::regclass);
 SELECT CDB_DropOverviews('base_t'::regclass);
-SELECT count(*) FROM _vovw_5_base_t;
+SELECT count(*) FROM _vovw_2_base_t;
+
+SELECT CDB_CreateOverviewsWithToleranceInPixels('base_t'::regclass, 7.5);
+SELECT count(*) FROM _vovw_2_base_t;
+SELECT CDB_DropOverviews('base_t'::regclass);
 
 DROP TABLE column_types_t;
 DROP TABLE base_bare_t;

test/CDB_OverviewsTest_expect

@@ -9,51 +9,44 @@ SELECT 1114
 
 
 
-{_vovw_5_base_bare_t,_vovw_4_base_bare_t,_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
+{_vovw_3_base_bare_t,_vovw_2_base_bare_t,_vovw_1_base_bare_t,_vovw_0_base_bare_t}
-125
+113
 number,int_number,name,start
 AVG(number)::double precision AS number,AVG(int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(start) ELSE NULL END::date AS start
 AVG(tab.number)::double precision AS number,AVG(tab.int_number)::integer AS int_number,CASE count(*) WHEN 1 THEN MIN(tab.name) ELSE NULL END::text AS name,CASE count(*) WHEN 1 THEN MIN(tab.start) ELSE NULL END::date AS start
-{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+{_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
-125
+113
 
-{_vovw_5_column_types_t,_vovw_4_column_types_t,_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
+{_vovw_3_column_types_t,_vovw_2_column_types_t,_vovw_1_column_types_t,_vovw_0_column_types_t}
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (base_bare_t,0,_vovw_0_base_bare_t)
 (base_bare_t,1,_vovw_1_base_bare_t)
 (base_bare_t,2,_vovw_2_base_bare_t)
 (base_bare_t,3,_vovw_3_base_bare_t)
-(base_bare_t,4,_vovw_4_base_bare_t)
-(base_bare_t,5,_vovw_5_base_bare_t)
 (base_t,0,_vovw_0_base_t)
 (base_t,1,_vovw_1_base_t)
 (base_t,2,_vovw_2_base_t)
 (base_t,3,_vovw_3_base_t)
-(base_t,4,_vovw_4_base_t)
-(base_t,5,_vovw_5_base_t)
 (column_types_t,0,_vovw_0_column_types_t)
 (column_types_t,1,_vovw_1_column_types_t)
 (column_types_t,2,_vovw_2_column_types_t)
 (column_types_t,3,_vovw_3_column_types_t)
-(column_types_t,4,_vovw_4_column_types_t)
-(column_types_t,5,_vovw_5_column_types_t)
 
 
 
-ERROR:  relation "_vovw_5_base_t" does not exist
+ERROR:  relation "_vovw_2_base_t" does not exist
-LINE 1: SELECT count(*) FROM _vovw_5_base_t;
+LINE 1: SELECT count(*) FROM _vovw_2_base_t;
                              ^
+{_vovw_5_base_t,_vovw_4_base_t,_vovw_3_base_t,_vovw_2_base_t,_vovw_1_base_t,_vovw_0_base_t}
+38
+
 DROP TABLE
 DROP TABLE
 DROP TABLE