Release 0.18.2

Move "username" column to the last position on analysis_catalog - fixes #276

Makefile

@@ -1,7 +1,7 @@
 # cartodb/Makefile
 
 EXTENSION = cartodb
-EXTVERSION = 0.16.2
+EXTVERSION = 0.18.2
 
 SED = sed
 
@@ -69,6 +69,13 @@ UPGRADABLE = \
   0.16.0 \
   0.16.1 \
   0.16.2 \
+  0.16.3 \
+  0.16.4 \
+  0.17.0 \
+  0.17.1 \
+  0.18.0 \
+  0.18.1 \
+  0.18.2 \
   $(EXTVERSION)dev \
   $(EXTVERSION)next \
   $(END)

NEWS.md

@@ -1,3 +1,57 @@
+0.18.2 (2016-10-20)
+-------------------
+
+* Fix: cleanup inconsistent position of `username` column in analysis catalog after upgrades
+  [#285](https://github.com/cartodb/cartodb-postgresql/pull/285)
+
+0.18.1 (2016-10-19)
+-------------------
+
+* Increase analysis limit factor to 2 [#284](https://github.com/CartoDB/cartodb-postgresql/pull/284)
+
+0.18.0 (2016-10-17)
+-------------------
+
+* Fix: exclude NULL geometries when creating Overviews #269
+* Function to check analysis tables limits #279
+* Exclude analysis cache tables from the quota #281
+
+
+0.17.1 (2016-08-16)
+-------------------
+
+* Add cache_tables column to cdb_analysis_catalog table #274.
+
+
+0.17.0 (2016-07-04)
+-------------------
+
+* Add export config for cdb_analysis_catalog table #268.
+* Add some extra fields to cdb_analysis_catalog table. Track user, error_message for failures, and last entity modifying the node #267.
+* Exclude overviews from user data size #262.
+
+
+0.16.4 (2016-05-27)
+-------------------
+
+* Change CDB_ZoomFromScale() to use a formula and raise
+  maximum overview level from 23 to 29.
+  [#259](https://github.com/CartoDB/cartodb-postgresql/pull/259)
+
+* Fix bug in overview creating causing it to fail when `x` or
+  `y` columns exist with non-integer type. Prevent also
+  potential integer overflows limiting maximum overview level
+  to 23.
+  [#258](https://github.com/CartoDB/cartodb-postgresql/pull/258)
+
+
+0.16.3 (2016-05-09)
+-------------------
+
+* Fix overview creation problem for organization users
+  with names that require quoting:
+  [#253](https://github.com/CartoDB/cartodb-postgresql/pull/253)
+
 0.16.2 (2016-04-27)
 -------------------
 

doc/CDB_UserTables.md

@@ -3,7 +3,11 @@ List the name of available tables (only the usable ones)
 #### Using the function
 
 ```sql
---- Returns a row for each table having given permission with the table name
+--- Returns a row for each table having given permission with the table name.
+--- It also returns tables from others users if you've permission to see them. For example, consider the following scenario:
+--- User X and User Y at account C.
+--- User X has a public table T.
+--- User Y will see table T. 
 --- Currently accepted permissions are: 'public', 'private' or 'all'
 SELECT CDB_UserTables(perms)
 ```

scripts-available/CDB_AnalysisCatalog.sql

@@ -20,5 +20,76 @@ cartodb.cdb_analysis_catalog (
     -- should register the number of times the node was used
     hits NUMERIC DEFAULT 0,
     -- should register what was the last node using current node
-    last_used_from char(40)
+    last_used_from char(40),
+    -- last job modifying the node
+    last_modified_by uuid,
+    -- store error message for failures
+    last_error_message text,
+    -- cached tables involved in the analysis
+    cache_tables regclass[] NOT NULL DEFAULT '{}',
+    -- useful for multi account deployments
+    username text
 );
+
+-- This can only be called from an SQL script executed by CREATE EXTENSION
+DO LANGUAGE 'plpgsql' $$
+BEGIN
+    PERFORM pg_catalog.pg_extension_config_dump('cartodb.cdb_analysis_catalog', '');
+END
+$$;
+
+-- Migrations to add new columns from old versions.
+-- IMPORTANT: Those columns will be added in order of creation. To be consistent
+-- in column order, ensure that new columns are added at the end and in the same order.
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN last_modified_by uuid;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN last_error_message text;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN cache_tables regclass[] NOT NULL DEFAULT '{}';
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+DO $$
+    BEGIN
+        BEGIN
+            ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username text;
+        EXCEPTION
+            WHEN duplicate_column THEN END;
+    END;
+$$;
+
+-- We want the "username" column to be moved to the last position if it was on a position from other versions
+-- see https://github.com/CartoDB/cartodb-postgresql/issues/276
+DO LANGUAGE 'plpgsql' $$
+    DECLARE
+        column_index int;
+    BEGIN
+        SELECT ordinal_position FROM information_schema.columns WHERE table_name='cdb_analysis_catalog' AND table_schema='cartodb' AND column_name='username' INTO column_index;
+        IF column_index = 1 OR column_index = 10 THEN
+           ALTER TABLE cartodb.cdb_analysis_catalog ADD COLUMN username_final text;
+           UPDATE cartodb.cdb_analysis_catalog SET username_final = username;
+           ALTER TABLE cartodb.cdb_analysis_catalog DROP COLUMN username;
+           ALTER TABLE cartodb.cdb_analysis_catalog RENAME COLUMN username_final TO username;
+        END IF;
+    END;
+$$;

scripts-available/CDB_AnalysisCheck.sql

@@ -0,0 +1,62 @@
+-- Read configuration parameter analysis_quota_factor, making it
+-- accessible to regular users (which don't have access to cdb_conf)
+CREATE OR REPLACE FUNCTION _CDB_GetConfAnalysisQuotaFactor()
+RETURNS float8 AS
+$$
+BEGIN
+  RETURN CDB_Conf_GetConf('analysis_quota_factor')::text::float8;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE SECURITY DEFINER;
+
+
+-- Get the factor (fraction of the quota) for Camshaft cached analysis tables
+CREATE OR REPLACE FUNCTION _CDB_AnalysisQuotaFactor()
+RETURNS float8 AS
+$$
+DECLARE
+  factor float8;
+BEGIN
+  -- We use a floating point cdb_conf parameter
+  factor := _CDB_GetConfAnalysisQuotaFactor();
+  -- With a default value
+  IF factor IS NULL THEN
+    factor := 2;
+  END IF;
+  RETURN factor;
+END;
+$$
+LANGUAGE 'plpgsql' STABLE;
+
+-- This checks the space used up by Camshaft cached analysis tables.
+-- An exception will be raised if the limits are exceeded.
+-- The name of an analysis table is passed; this, in addition to the
+-- db role that executes this function is used to determined which
+-- analysis tables will be considered.
+CREATE OR REPLACE FUNCTION CDB_CheckAnalysisQuota(table_name TEXT)
+RETURNS void AS
+$$
+DECLARE
+  schema_name TEXT;
+  user_name TEXT;
+  nominal_quota int8;
+  cache_size float8;
+BEGIN
+  -- We rely on the search_path to determine the user's schema and
+  -- check for all analysis tables in that schema.
+  -- An alternative would be to use cdb_analysis_catalog to
+  -- select analysis tables (cache_tables) from the same user, analysis or node.
+  -- For example:
+  --   SELECT unnest(cache_tables) FROM cdb_analysis_catalog
+  --     WHERE username IN (SELECT username FROM cdb_analysis_catalog
+  --       WHERE table_name::regclass = ANY (cache_tables));
+  -- At the moment we're not using the provided table_name.
+
+  SELECT current_schema() INTO schema_name;
+  EXECUTE FORMAT('SELECT %I._CDB_UserQuotaInBytes();', schema_name) INTO nominal_quota;
+  IF nominal_quota*_CDB_AnalysisQuotaFactor() < _CDB_AnalysisDataSize(schema_name) THEN
+    -- The limit is defined by a factor applied to the total space quota for the user
+    RAISE EXCEPTION 'Analysis cache space limits exceeded';
+  END IF;
+END;
+$$ LANGUAGE PLPGSQL;

scripts-available/CDB_AnalysisSupport.sql

@@ -0,0 +1,55 @@
+-- Internal auxiliar functions to deal with [Camshaft](https://github.com/cartodb/camshaft) cached analysis tables.
+
+-- This function returns TRUE if a given table name corresponds to a Camshaft cached analysis table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsAnalysisTableName(table_name TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN table_name SIMILAR TO '\Aanalysis_[0-9a-f]{10}_[0-9a-f]{40}\Z';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- This function returns a relation of Camshaft cached analysis tables in the given schema.
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_AnalysisTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE _CDB_IsAnalysisTableName(table_name);
+$$ LANGUAGE 'sql';
+
+-- This function returns a relation user tables excluding analysis tables
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_NonAnalysisTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT * FROM _CDB_UserTablesInSchema(schema_name) WHERE Not _CDB_IsAnalysisTableName(table_name);
+$$ LANGUAGE 'sql';
+
+-- Total spaced used up by Camshaft cached analysis tables in the given schema.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_AnalysisDataSize(schema_name TEXT DEFAULT NULL)
+RETURNS bigint AS
+$$
+DECLARE
+  total_size bigint;
+BEGIN
+  WITH analysis_tables AS (
+    SELECT t.schema_name, t.table_name FROM _CDB_AnalysisTablesInSchema(schema_name) t
+  )
+  SELECT COALESCE(INT8(SUM(_CDB_total_relation_size(analysis_tables.schema_name, analysis_tables.table_name))))::int8
+    INTO total_size FROM analysis_tables;
+  IF total_size IS NOT NULL THEN
+    RETURN total_size;
+  ELSE
+    RETURN 0;
+  END IF;
+END;
+$$
+LANGUAGE 'plpgsql' VOLATILE;

scripts-available/CDB_Overviews.sql

@@ -1,149 +1,3 @@
--- Information about tables in a schema.
--- If the schema name parameter is NULL, then tables from all schemas
--- that may contain user tables are returned.
--- For each table, the regclass, schema name and table name are returned.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
-RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
-AS $$
-  SELECT
-    c.oid::regclass AS table_regclass,
-    n.nspname::text AS schema_name,
-    c.relname::text AS table_relname
-  FROM pg_class c
-  JOIN pg_namespace n ON n.oid = c.relnamespace
-  WHERE c.relkind = 'r'
-  AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
-  AND CASE WHEN schema_name IS NULL
-             THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
-           ELSE n.nspname = schema_name
-           END;
-$$ LANGUAGE 'sql';
-
--- Pattern that can be used to detect overview tables and Extract
--- the intended zoom level from the table name.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN '\A_vovw_(\d+)_';
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
--- substring(tablename from _CDB_OverviewTableDiscriminator())
-
-
--- Pattern matched by the overview tables of a given base table name.
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN _CDB_OverviewTableDiscriminator() || base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
--- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
-
--- Name of an overview table, given the base table name and the Z level
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
-RETURNS TEXT
-AS $$
-  BEGIN
-    RETURN '_vovw_' || z::text || '_' || base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Condition to check if a tabla is an overview table of some base table
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
-RETURNS BOOLEAN
-AS $$
-  BEGIN
-    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Extract the Z level from an overview table name
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
-RETURNS INTEGER
-AS $$
-  BEGIN
-    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Name of the base table corresponding to an overview table
--- Scope: private.
-CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
-RETURNS TEXT
-AS $$
-  BEGIN
-    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
-      RETURN overview_table;
-    ELSE
-      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
-    END IF;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
-CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTable(overview_table REGCLASS)
-RETURNS REGCLASS
-AS $$
-  DECLARE
-    table_name TEXT;
-    schema_name TEXT;
-    base_name TEXT;
-    base_table REGCLASS;
-  BEGIN
-    SELECT * FROM _cdb_split_table_name(overview_table) INTO schema_name, table_name;
-    base_name := _CDB_OverviewBaseTableName(table_name);
-    IF base_name != table_name THEN
-      base_table := Format('%I.%I', schema_name, base_name)::regclass;
-    ELSE
-      base_table := overview_table;
-    END IF;
-    RETURN base_table;
-  END;
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
-AS $$
-  BEGIN
-    SELECT n.nspname, c.relname
-    INTO STRICT schema_name, table_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
--- Schema and relation names of a table given its reloid
--- Scope: private.
--- Parameters
---   reloid: oid of the table.
--- Return (schema_name, table_name)
--- note that returned names will be quoted if necessary
-CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
-RETURNS TEXT
-AS $$
-  DECLARE
-    schema_name TEXT;
-  BEGIN
-    SELECT n.nspname
-    INTO STRICT schema_name
-    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
-    WHERE c.oid = reloid;
-    RETURN schema_name;
-  END
-$$ LANGUAGE PLPGSQL IMMUTABLE;
-
 -- Remove a dataset's existing  overview tables.
 -- Scope: public
 -- Parameters:
@@ -235,7 +89,7 @@ AS $$
       FROM pg_class c JOIN pg_namespace n on n.oid = c.relnamespace WHERE c.oid = reloid::oid;
 
     ext_query = format(
-      'SELECT ST_EstimatedExtent(''%1$I'', ''%2$I'', ''%3$I'');',
+      'SELECT ST_EstimatedExtent(''%1$s'', ''%2$s'', ''%3$s'');',
       table_id.schema_name, table_id.table_name, 'the_geom_webmercator'
     );
 
@@ -297,7 +151,11 @@ AS $$
     WITH RECURSIVE t(x, y, z, e) AS (
       WITH ext AS (SELECT _cdb_estimated_extent(%6$s) as g),
       base AS (
-        SELECT (-floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)))::integer z
+        SELECT
+          least(
+           -floor(log(2, (greatest(ST_XMax(ext.g)-ST_XMin(ext.g), ST_YMax(ext.g)-ST_YMin(ext.g))/(%4$s*%5$s))::numeric)),
+           _CDB_MaxOverviewLevel()+1
+          )::integer z
         FROM ext
       ),
       lim AS (
@@ -319,10 +177,10 @@ AS $$
       UNION ALL
       SELECT x*2 + xx, y*2 + yy, t.z+1, (
         SELECT count(*) FROM %1$s
-          WHERE the_geom_webmercator && CDB_XYZ_Extent(x*2 + xx, y*2 + yy, t.z+1)
+          WHERE the_geom_webmercator && CDB_XYZ_Extent(t.x*2 + c.xx, t.y*2 + c.yy, t.z+1)
       )
       FROM t, base, (VALUES (0, 0), (0, 1), (1, 1), (1, 0)) AS c(xx, yy)
-      WHERE t.e > %2$s AND t.z < (base.z + %3$s)
+      WHERE t.e > %2$s AND t.z < least(base.z + %3$s, _CDB_MaxZoomLevel())
     )
     SELECT MAX(e/ST_Area(CDB_XYZ_Extent(x,y,z))) FROM t where e > 0;
   ', reloid::text, min_features, nz, n, c, reloid::oid)
@@ -363,7 +221,7 @@ AS $$
     -- find minimum z so that fd*ta(z) <= lim
     -- compute a rough 'feature density' value
     SELECT CDB_XYZ_Resolution(-8) INTO c;
-    RETURN ceil(log(2.0, (c*c*fd/lim)::numeric)/2);
+    RETURN least(_CDB_MaxOverviewLevel()+1, ceil(log(2.0, (c*c*fd/lim)::numeric)/2));
   END;
 $$ LANGUAGE PLPGSQL STABLE;
 
@@ -591,7 +449,7 @@ $$
 $$
 LANGUAGE SQL IMMUTABLE;
 
--- Tell Postgres how to use our aggregate
+DROP AGGREGATE IF EXISTS _cdb_mode(anyelement);
 CREATE AGGREGATE _cdb_mode(anyelement) (
   SFUNC=array_append,
   STYPE=anyarray,
@@ -829,6 +687,7 @@ AS $$
              Floor(ST_Y(f.the_geom_webmercator)/%2$s)::int AS gy,
              MIN(cartodb_id) AS cartodb_id
           FROM %1$s f
+          WHERE f.the_geom_webmercator IS NOT NULL
           GROUP BY gx, gy
          )
          SELECT %6$s FROM clusters

scripts-available/CDB_OverviewsSupport.sql

@@ -0,0 +1,173 @@
+-- Auxiliary overviews FUNCTIONS
+
+-- Maximum zoom level for which overviews may be created
+CREATE OR REPLACE FUNCTION _CDB_MaxOverviewLevel()
+RETURNS INTEGER
+AS $$
+  BEGIN
+    -- Zoom level will be limited so that both tile coordinates
+    -- and gridding coordinates within a tile up to 1px
+    -- (i.e. tile coordinates / 256)
+    -- can be stored in a 32-bit signed integer.
+    -- We have 31 bits por positive numbers
+    -- For zoom level Z coordinates range from 0 to 2^Z-1, so they
+    -- need Z bits, and need 8 bits more to address pixels within a tile
+    -- (gridding), so we'll limit Z to a maximum of 31 - 8
+    RETURN 23;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Maximum zoom level usable with integer coordinates
+CREATE OR REPLACE FUNCTION _CDB_MaxZoomLevel()
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN 31;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Information about tables in a schema.
+-- If the schema name parameter is NULL, then tables from all schemas
+-- that may contain user tables are returned.
+-- For each table, the regclass, schema name and table name are returned.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_UserTablesInSchema(schema_name text DEFAULT NULL)
+RETURNS TABLE(table_regclass REGCLASS, schema_name TEXT, table_name TEXT)
+AS $$
+  SELECT
+    c.oid::regclass AS table_regclass,
+    n.nspname::text AS schema_name,
+    c.relname::text AS table_relname
+  FROM pg_class c
+  JOIN pg_namespace n ON n.oid = c.relnamespace
+  WHERE c.relkind = 'r'
+  AND c.relname NOT IN ('cdb_tablemetadata', 'spatial_ref_sys')
+  AND CASE WHEN schema_name IS NULL
+             THEN n.nspname NOT IN ('pg_catalog', 'information_schema', 'topology', 'cartodb')
+           ELSE n.nspname = schema_name
+           END;
+$$ LANGUAGE 'sql';
+
+-- Pattern that can be used to detect overview tables and Extract
+-- the intended zoom level from the table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableDiscriminator()
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '\A_vovw_(\d+)_';
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- substring(tablename from _CDB_OverviewTableDiscriminator())
+
+
+-- Pattern matched by the overview tables of a given base table name.
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTablePattern(base_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN _CDB_OverviewTableDiscriminator() || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+-- tablename SIMILAR TO _CDB_OverviewTablePattern(base_table)
+
+-- Name of an overview table, given the base table name and the Z level
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableName(base_table TEXT, z INTEGER)
+RETURNS TEXT
+AS $$
+  BEGIN
+    RETURN '_vovw_' || z::text || '_' || base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Condition to check if a tabla is an overview table of some base table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_IsOverviewTableOf(base_table TEXT, otable TEXT)
+RETURNS BOOLEAN
+AS $$
+  BEGIN
+    RETURN otable SIMILAR TO _CDB_OverviewTablePattern(base_table);
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Extract the Z level from an overview table name
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewTableZ(otable TEXT)
+RETURNS INTEGER
+AS $$
+  BEGIN
+    RETURN substring(otable from _CDB_OverviewTableDiscriminator())::integer;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Name of the base table corresponding to an overview table
+-- Scope: private.
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTableName(overview_table TEXT)
+RETURNS TEXT
+AS $$
+  BEGIN
+    IF _CDB_OverviewTableZ(overview_table) IS NULL THEN
+      RETURN overview_table;
+    ELSE
+      RETURN regexp_replace(overview_table, _CDB_OverviewTableDiscriminator(), '');
+    END IF;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION _CDB_OverviewBaseTable(overview_table REGCLASS)
+RETURNS REGCLASS
+AS $$
+  DECLARE
+    table_name TEXT;
+    schema_name TEXT;
+    base_name TEXT;
+    base_table REGCLASS;
+  BEGIN
+    SELECT * FROM _cdb_split_table_name(overview_table) INTO schema_name, table_name;
+    base_name := _CDB_OverviewBaseTableName(table_name);
+    IF base_name != table_name THEN
+      base_table := Format('%I.%I', schema_name, base_name)::regclass;
+    ELSE
+      base_table := overview_table;
+    END IF;
+    RETURN base_table;
+  END;
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_split_table_name(reloid REGCLASS, OUT schema_name TEXT, OUT table_name TEXT)
+AS $$
+  BEGIN
+    SELECT n.nspname, c.relname
+    INTO STRICT schema_name, table_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;
+
+-- Schema and relation names of a table given its reloid
+-- Scope: private.
+-- Parameters
+--   reloid: oid of the table.
+-- Return (schema_name, table_name)
+-- note that returned names will be quoted if necessary
+CREATE OR REPLACE FUNCTION _cdb_schema_name(reloid REGCLASS)
+RETURNS TEXT
+AS $$
+  DECLARE
+    schema_name TEXT;
+  BEGIN
+    SELECT n.nspname
+    INTO STRICT schema_name
+    FROM pg_class c JOIN pg_namespace n ON c.relnamespace = n.oid
+    WHERE c.oid = reloid;
+    RETURN schema_name;
+  END
+$$ LANGUAGE PLPGSQL IMMUTABLE;

scripts-available/CDB_Quota.sql

@@ -35,7 +35,11 @@ BEGIN
   table_cat AS (
     SELECT
       table_name,
-      EXISTS(select * from raster_tables where o_table_name = table_name) AS is_overview,
+      (
+        EXISTS(select * from raster_tables where o_table_name = table_name)
+        OR
+        table_name SIMILAR TO _CDB_OverviewTableDiscriminator() || '[\w\d]*'
+      ) AS is_overview,
       EXISTS(SELECT * FROM raster_tables WHERE r_table_name = table_name) AS is_raster
     FROM user_tables
   ),

scripts-available/CDB_ZoomFromScale.sql

@@ -1,30 +1,36 @@
-CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
-BEGIN
-  CASE
-    WHEN scaleDenominator > 500000000 THEN RETURN 0;
-    WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
-    WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
-    WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
-    WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
-    WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
-    WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
-    WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
-    WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
-    WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
-    WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
-    WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
-    WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
-    WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
-    WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
-    WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
-    WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
-    WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
-    WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
-    WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
-    WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
-    WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
-    WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
-    WHEN scaleDenominator <= 100 THEN RETURN 23;
-  END CASE;
-END
-$$ LANGUAGE plpgsql IMMUTABLE;
+-- Maximum supported zoom level
+CREATE OR REPLACE FUNCTION _CDB_MaxSupportedZoom()
+RETURNS int
+LANGUAGE SQL
+IMMUTABLE
+AS $$
+  -- The maximum zoom level has to be limited for various reasons,
+  -- e.g. zoom levels greater than 31 would require tile coordinates
+  -- that would not fit in an INTEGER (which is signed, 32 bits long).
+  -- We'll choose 20 as a limit which is safe also when the JavaScript shift
+  -- operator (<<) is used for computing powers of two.
+  SELECT 29;
+$$;
+
+CREATE OR REPLACE FUNCTION cartodb.CDB_ZoomFromScale(scaleDenominator numeric)
+RETURNS int
+LANGUAGE SQL
+IMMUTABLE
+AS $$
+  SELECT
+    CASE
+      WHEN scaleDenominator > 600000000 THEN
+        -- Scale is smaller than zoom level 0
+        NULL
+      WHEN scaleDenominator = 0 THEN
+        -- Actual zoom level would be infinite
+        _CDB_MaxSupportedZoom()
+      ELSE
+        CAST (
+          LEAST(
+            ROUND(LOG(2, 559082264.028/scaleDenominator)),
+            _CDB_MaxSupportedZoom()
+          )
+        AS INTEGER)
+    END;
+$$;

scripts-enabled/085-CDB_OverviewsSupport.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_OverviewsSupport.sql

scripts-enabled/270-CDB_AnalysisSupport.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisSupport.sql

scripts-enabled/275-CDB_AnalysisCheck.sql

@@ -0,0 +1 @@
+../scripts-available/CDB_AnalysisCheck.sql

test/CDB_AnalysisCheckTest.sql

@@ -0,0 +1,20 @@
+SET client_min_messages TO error;
+\set VERBOSITY terse
+
+SELECT CDB_SetUserQuotaInBytes(1000000);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+CREATE TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94(id int);
+CREATE TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9(id int);
+SELECT _CDB_AnalysisTablesInSchema('public');
+SELECT _CDB_AnalysisDataSize('public');
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+SELECT CDB_SetUserQuotaInBytes(1);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+INSERT INTO analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5(id) VALUES (1),(2),(3),(4),(5);
+SELECT CDB_CheckAnalysisQuota('analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94');
+DROP TABLE analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94;
+DROP TABLE analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da9;
+DROP FUNCTION "public"._CDB_UserQuotaInBytes();

test/CDB_AnalysisCheckTest_expect

@@ -0,0 +1,18 @@
+SET
+1000000
+0
+CREATE TABLE
+CREATE TABLE
+CREATE TABLE
+(analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5,public,analysis_2f13a3dbd7_41bd92976fc6dd97072afe4ee450054f4c0715d5)
+(analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94,public,analysis_2f13a3dbd7_f00cee44e9e6152b450bde3a92eb9ae0d099da94)
+0
+
+1
+
+INSERT 0 5
+ERROR:  Analysis cache space limits exceeded
+DROP TABLE
+DROP TABLE
+DROP TABLE
+DROP FUNCTION

test/extension/run_at_cartodb_schema.sql

@@ -4,4 +4,4 @@ SET SCHEMA 'cartodb';
 \i scripts-available/CDB_ColumnNames.sql
 \i scripts-available/CDB_ColumnType.sql
 \i scripts-available/CDB_AnalysisCatalog.sql
-SET SCHEMA 'public';
+SET SCHEMA 'public';

test/extension/test.sh

@@ -181,6 +181,7 @@ function setup_database() {
 
     log_info "########################### BOOTSTRAP ###########################"
     ${CMD} -d ${DATABASE} -f scripts-available/CDB_Organizations.sql
+    ${CMD} -d ${DATABASE} -f scripts-available/CDB_OverviewsSupport.sql
     # trick to allow forcing a schema when loading SQL files (see: http://bit.ly/1HeLnhL)
     ${CMD} -d ${DATABASE} -f test/extension/run_at_cartodb_schema.sql
 }