Updated NEWS.md

* Force png tile generation for static maps

If the user tries to generate a static map requesting JPG it will fail
because is going to try to generate the tiles using JPG as format which
is not supported by now, this bug was introduced in the version 4.0.1

So we now force, again, the tiles to be generated as PNG but we pass
the requested format, JPG, to windshaft to generate the final image as
the user reqests

* Added support to define image format in the image assertions

* Added test for JPEG static image generation

Also I've added support for:

- JPEG images
- Different tolerance based on the file type, it seems that due to
  different compression we need different tolerance for JPG images

NEWS.md

@@ -1,5 +1,22 @@
 # Changelog
 
+## 4.3.1
+Released 2017-12-12
+
+Bug fix:
+  - Fixed bug introduced in version 4.0.1 that brokes the static map generation using JPG as format
+
+## 4.3.0
+Released 2017-12-11
+
+Announcements:
+- Optimize Formula queries.
+- Optimize Formula queries in overviews.
+- Optimize Numeric Histogram queries.
+- Optimize Date Histogram queries.
+- Date Histograms: Now returns the same value for max/min/avg/timestamp per bin.
+- Date Histograms: Now it should return the same no matter the DB/Client time zone.
+
 ## 4.2.0
 Released 2017-12-04
 

lib/cartodb/controllers/layergroup.js

@@ -357,7 +357,9 @@ LayergroupController.prototype.center = function(req, res, next) {
 
 LayergroupController.prototype.staticMap = function(req, res, width, height, zoom /* bounds */, center, next) {
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    req.params.format = req.params.format || 'png';
+    // We force always the tile to be generated using PNG because
+    // is the only format we support by now
+    res.locals.format = 'png';
     res.locals.layer = res.locals.layer || 'all';
 
     var self = this;
@@ -507,4 +509,4 @@ function validateLayerRouteMiddleware(req, res, next) {
     }
 
     next();
-}
+}

lib/cartodb/controllers/named_maps.js

@@ -28,7 +28,7 @@ NamedMapsController.prototype.register = function(app) {
         userMiddleware,
         this.prepareContext,
         this.tile.bind(this),
-        vectorError()        
+        vectorError()
     );
 
     app.get(
@@ -121,7 +121,9 @@ NamedMapsController.prototype.staticMap = function(req, res, next) {
     var cdbUser = res.locals.user;
 
     var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
-    res.locals.format = req.params.format || 'png';
+    // We force always the tile to be generated using PNG because
+    // is the only format we support by now
+    res.locals.format = 'png';
     res.locals.layer = res.locals.layer || 'all';
 
     var namedMapProvider;

lib/cartodb/models/dataview/formula.js

@@ -1,34 +1,14 @@
 const BaseDataview = require('./base');
 const debug = require('debug')('windshaft:dataview:formula');
+const utils = require('../../utils/query-utils');
 
-const countInfinitiesQueryTpl = ctx => `
-    SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities
-    WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float
-`;
-
-const countNansQueryTpl = ctx => `
-    SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans
-    WHERE ${ctx.column} = 'NaN'::float
-`;
-
-const filterOutSpecialNumericValuesTpl = ctx => `
-    WHERE
-        ${ctx.column} != 'infinity'::float
-    AND
-        ${ctx.column} != '-infinity'::float
-    AND
-        ${ctx.column} != 'NaN'::float
-`;
-
-const formulaQueryTpl = ctx => `
-    SELECT
-        ${ctx.operation}(${ctx.column}) AS result,
-        (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count
-        ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''}
-        ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''}
-    FROM (${ctx.query}) __cdb_formula
-    ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''}
-`;
+const formulaQueryTpl = ctx =>
+`SELECT
+    ${ctx.operation}(${utils.handleFloatColumn(ctx)}) AS result,
+    ${utils.countNULLs(ctx)} AS nulls_count
+    ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
+    ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
+FROM (${ctx.query}) __cdb_formula`;
 
 const VALID_OPERATIONS = {
     count: true,

lib/cartodb/models/dataview/histograms/date-histogram.js

@@ -1,5 +1,102 @@
 const BaseHistogram = require('./base-histogram');
 const debug = require('debug')('windshaft:dataview:date-histogram');
+const utils = require('../../../utils/query-utils');
+
+/**
+ * Gets the name of a timezone with the same offset as the required
+ * using the pg_timezone_names table. We do this because it's simpler to pass
+ * the name than to pass the offset itself as PostgreSQL uses different
+ * sign convention. For example: TIME ZONE 'CET' is equal to TIME ZONE 'UTC-1',
+ * not 'UTC+1' which would be expected.
+ * Gives priority to Etc/GMT±N timezones but still support odd offsets like 8.5
+ * hours for Asia/Pyongyang.
+ * It also makes it easier to, in the future, support the input of expected timezone
+ * instead of the offset; that is using 'Europe/Madrid' instead of
+ * '+3600' or '+7200'. The daylight saving status can be handled by postgres.
+ */
+const offsetNameQueryTpl = ctx => `
+WITH __wd_tz AS
+(
+    SELECT name
+    FROM pg_timezone_names
+    WHERE utc_offset = interval '${ctx.offset} hours'
+    ORDER BY CASE WHEN name LIKE 'Etc/GMT%' THEN 0 ELSE 1 END
+    LIMIT 1
+),`;
+
+/**
+ * Function to get the subquery that places each row in its bin depending on
+ * the aggregation. Since the data stored is in epoch we need to adapt it to
+ * our timezone so when calling date_trunc it falls into the correct bin
+ */
+function dataBucketsQuery(ctx) {
+    var condition_str = '';
+
+    if (ctx.start !== 0) {
+        condition_str = `WHERE ${ctx.column} >= to_timestamp(${ctx.start})`;
+    }
+    if (ctx.end !== 0) {
+        if (condition_str === '') {
+            condition_str = `WHERE ${ctx.column} <= to_timestamp(${ctx.end})`;
+        }
+        else {
+            condition_str += ` and ${ctx.column} <= to_timestamp(${ctx.end})`;
+        }
+    }
+
+    return `
+__wd_buckets AS
+(
+    SELECT
+        date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, ${ctx.column}::timestamptz)) as timestamp,
+        count(*) as freq,
+        ${utils.countNULLs(ctx)} as nulls_count
+    FROM
+    (
+        ${ctx.query}
+    ) __source, __wd_tz
+    ${condition_str}
+    GROUP BY timestamp, __wd_tz.name
+),`;
+}
+
+/**
+ * Function that generates an array with all the possible bins between the
+ * start and end date. If not provided we use the min and max generated from
+ * the dataBucketsQuery
+ */
+function allBucketsArrayQuery(ctx) {
+    var extra_from = ``;
+    var series_start = ``;
+    var series_end = ``;
+
+    if (ctx.start === 0) {
+        extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
+        series_start = `min(__wd_buckets.timestamp)`;
+    } else {
+        series_start = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.start})))`;
+    }
+
+    if (ctx.end === 0) {
+        extra_from = `, __wd_buckets GROUP BY __wd_tz.name`;
+        series_end = `max(__wd_buckets.timestamp)`;
+    } else {
+        series_end = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.end})))`;
+    }
+
+    return `
+__wd_all_buckets AS
+(
+    SELECT ARRAY(
+        SELECT
+            generate_series(
+                    ${series_start},
+                    ${series_end},
+                    interval '${ctx.interval}') as bin_start
+            FROM __wd_tz${extra_from}
+    ) as bins
+)`;
+}
 
 const dateIntervalQueryTpl = ctx => `
     WITH
@@ -41,107 +138,6 @@ const dateIntervalQueryTpl = ctx => `
     FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds
 `;
 
-const nullsQueryTpl = ctx => `
-    __cdb_nulls AS (
-        SELECT
-            count(*) AS __cdb_nulls_count
-        FROM (${ctx.query}) __cdb_histogram_nulls
-        WHERE ${ctx.column} IS NULL
-    )
-`;
-
-const dateBasicsQueryTpl = ctx => `
-    __cdb_basics AS (
-        SELECT
-            max(date_part('epoch', ${ctx.column})) AS __cdb_max_val,
-            min(date_part('epoch', ${ctx.column})) AS __cdb_min_val,
-            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
-            min(
-                date_trunc(
-                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
-                )
-            ) AS __cdb_start_date,
-            max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date,
-            count(1) AS __cdb_total_rows
-        FROM (${ctx.query}) __cdb_basics_query
-    )
-`;
-
-const dateOverrideBasicsQueryTpl = ctx => `
-    __cdb_basics AS (
-        SELECT
-            max(${ctx.end})::float AS __cdb_max_val,
-            min(${ctx.start})::float AS __cdb_min_val,
-            avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val,
-            min(
-                date_trunc(
-                    '${ctx.aggregation}',
-                    TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}'
-                )
-            ) AS __cdb_start_date,
-            max(
-                TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}'
-            ) AS __cdb_end_date,
-            count(1) AS __cdb_total_rows
-        FROM (${ctx.query}) __cdb_basics_query
-    )
-`;
-
-const dateBinsQueryTpl = ctx => `
-    __cdb_bins AS (
-        SELECT
-            __cdb_bins_array,
-            ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number
-        FROM (
-            SELECT
-                ARRAY(
-                    SELECT GENERATE_SERIES(
-                        __cdb_start_date::timestamptz,
-                        __cdb_end_date::timestamptz,
-                        ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`}
-                    )
-                ) AS __cdb_bins_array
-            FROM __cdb_basics
-        ) __cdb_bins_array_query
-    )
-`;
-
-const dateHistogramQueryTpl = ctx => `
-    SELECT
-        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
-        __cdb_bins_number AS bins_number,
-        __cdb_nulls_count AS nulls_count,
-        CASE WHEN __cdb_min_val = __cdb_max_val
-        THEN 0
-        ELSE GREATEST(
-            1,
-            LEAST(
-                WIDTH_BUCKET(
-                    ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}',
-                    __cdb_bins_array
-                ),
-                __cdb_bins_number
-            )
-        ) - 1
-        END AS bin,
-        min(
-            date_part(
-                'epoch',
-                date_trunc(
-                    '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}'
-                ) AT TIME ZONE '${ctx.offset}'
-            )
-        )::numeric AS timestamp,
-        date_part('epoch', __cdb_start_date)::numeric AS timestamp_start,
-        min(date_part('epoch', ${ctx.column}))::numeric AS min,
-        max(date_part('epoch', ${ctx.column}))::numeric AS max,
-        avg(date_part('epoch', ${ctx.column}))::numeric AS avg,
-        count(*) AS freq
-    FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls
-    WHERE date_part('epoch', ${ctx.column}) IS NOT NULL
-    GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start
-    ORDER BY bin
-`;
 
 const MAX_INTERVAL_VALUE = 366;
 
@@ -176,12 +172,21 @@ module.exports = class DateHistogram extends BaseHistogram {
 
     _buildQueryTpl (ctx) {
         return `
-            WITH
-            ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)},
-            ${dateBinsQueryTpl(ctx)},
-            ${nullsQueryTpl(ctx)}
-            ${dateHistogramQueryTpl(ctx)}
-        `;
+${offsetNameQueryTpl(ctx)}
+${dataBucketsQuery(ctx)}
+${allBucketsArrayQuery(ctx)}
+SELECT
+    array_position(__wd_all_buckets.bins, __wd_buckets.timestamp) - 1 as bin,
+    date_part('epoch', timezone(__wd_tz.name, __wd_buckets.timestamp)) AS timestamp,
+    __wd_buckets.freq as freq,
+    date_part('epoch', timezone(__wd_tz.name, (__wd_all_buckets.bins)[1])) as timestamp_start,
+    array_length(__wd_all_buckets.bins, 1) as bins_number,
+    date_part('epoch', interval '${ctx.interval}') as bin_width,
+    __wd_buckets.nulls_count as nulls_count
+FROM __wd_buckets, __wd_all_buckets, __wd_tz
+GROUP BY __wd_tz.name, __wd_all_buckets.bins, __wd_buckets.timestamp, __wd_buckets.nulls_count, __wd_buckets.freq
+ORDER BY bin ASC;
+`;
     }
 
     _buildQuery (psql, override, callback) {
@@ -204,6 +209,9 @@ module.exports = class DateHistogram extends BaseHistogram {
             return null;
         }
 
+        var interval = this._getAggregation(override) === 'quarter' ?
+            '3 months' : '1 ' + this._getAggregation(override);
+
         const histogramSql = this._buildQueryTpl({
             override: override,
             query: this.query,
@@ -211,7 +219,8 @@ module.exports = class DateHistogram extends BaseHistogram {
             aggregation: this._getAggregation(override),
             start: this._getBinStart(override),
             end: this._getBinEnd(override),
-            offset: this._parseOffset(override)
+            offset: this._parseOffset(override),
+            interval: interval
         });
 
         debug(histogramSql);
@@ -264,8 +273,8 @@ module.exports = class DateHistogram extends BaseHistogram {
             offset: this._getOffset(override),
             timestamp_start: firstRow.timestamp_start,
 
-            bin_width: firstRow.bin_width,
-            bins_count: firstRow.bins_number,
+            bin_width: firstRow.bin_width || 0,
+            bins_count: firstRow.bins_number || 0,
             bins_start: firstRow.timestamp,
             nulls: firstRow.nulls_count,
             infinities: firstRow.infinities_count,
@@ -275,6 +284,10 @@ module.exports = class DateHistogram extends BaseHistogram {
     }
 
     _getBuckets (result) {
+        result.rows.forEach(function(row) {
+            row.min = row.max = row.avg = row.timestamp;
+        });
+
         return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp }));
     }
 

lib/cartodb/models/dataview/histograms/numeric-histogram.js

@@ -1,44 +1,25 @@
 const BaseHistogram = require('./base-histogram');
 const debug = require('debug')('windshaft:dataview:numeric-histogram');
+const utils = require('../../../utils/query-utils');
 
-const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`;
-
-const filterOutSpecialNumericValues = ctx => `
-        ${ctx.column} != 'infinity'::float
-    AND
-        ${ctx.column} != '-infinity'::float
-    AND
-        ${ctx.column} != 'NaN'::float
-`;
-
-const filteredQueryTpl = ctx => `
+/** Query to get min and max values from the query */
+const irqQueryTpl = ctx => `
     __cdb_filtered_source AS (
         SELECT *
         FROM (${ctx.query}) __cdb_filtered_source_query
-        WHERE ${ctx.column} IS NOT NULL
-        ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''}
-    )
-`;
-
-const basicsQueryTpl = ctx => `
+        WHERE ${utils.handleFloatColumn(ctx)} IS NOT NULL
+    ),
     __cdb_basics AS (
         SELECT
-            max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val,
-            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
+            max(${ctx.column}) AS __cdb_max_val,
+            min(${ctx.column}) AS __cdb_min_val,
+            count(1) AS __cdb_total_rows
         FROM __cdb_filtered_source
     )
 `;
 
-const overrideBasicsQueryTpl = ctx => `
-    __cdb_basics AS (
-        SELECT
-            max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val,
-            avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows
-        FROM __cdb_filtered_source
-    )
-`;
-
-const iqrQueryTpl = ctx => `
+/* Query to calculate the number of bins (needs irqQueryTpl before it*/
+const binsQueryTpl = ctx => `
     __cdb_iqrange AS (
         SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr
         FROM (
@@ -49,10 +30,7 @@ const iqrQueryTpl = ctx => `
             WHERE quartile = 1 or quartile = 3
             GROUP BY quartile
         ) __cdb_iqr
-    )
-`;
-
-const binsQueryTpl = ctx => `
+    ),
     __cdb_bins AS (
         SELECT
             CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0
@@ -70,83 +48,6 @@ const binsQueryTpl = ctx => `
     )
 `;
 
-const overrideBinsQueryTpl = ctx => `
-    __cdb_bins AS (
-        SELECT ${ctx.override.bins} AS __cdb_bins_number
-    )
-`;
-
-const nullsQueryTpl = ctx => `
-    __cdb_nulls AS (
-        SELECT
-            count(*) AS __cdb_nulls_count
-        FROM (${ctx.query}) __cdb_histogram_nulls
-        WHERE ${ctx.column} IS NULL
-    )
-`;
-
-const infinitiesQueryTpl = ctx => `
-    __cdb_infinities AS (
-        SELECT
-            count(*) AS __cdb_infinities_count
-        FROM (${ctx.query}) __cdb_infinities_query
-        WHERE
-            ${ctx.column} = 'infinity'::float
-        OR
-            ${ctx.column} = '-infinity'::float
-    )
-`;
-
-const nansQueryTpl = ctx => `
-    __cdb_nans AS (
-        SELECT
-            count(*) AS __cdb_nans_count
-        FROM (${ctx.query}) __cdb_nans_query
-        WHERE ${ctx.column} = 'NaN'::float
-    )
-`;
-
-const specialNumericValuesColumnDefinitionTpl = () => `
-    __cdb_infinities_count AS infinities_count,
-    __cdb_nans_count AS nans_count
-`;
-
-const specialNumericValuesCTETpl = () => `
-    __cdb_infinities, __cdb_nans
-`;
-
-const specialNumericValuesColumnTpl = () => `
-    infinities_count, nans_count
-`;
-
-const histogramQueryTpl = ctx => `
-    SELECT
-        (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width,
-        __cdb_bins_number AS bins_number,
-        __cdb_nulls_count AS nulls_count,
-        ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''}
-        __cdb_avg_val AS avg_val,
-        CASE WHEN __cdb_min_val = __cdb_max_val
-            THEN 0
-            ELSE GREATEST(
-                1,
-                LEAST(
-                    WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number),
-                    __cdb_bins_number
-                )
-            ) - 1
-        END AS bin,
-        min(${ctx.column})::numeric AS min,
-        max(${ctx.column})::numeric AS max,
-        avg(${ctx.column})::numeric AS avg,
-        count(*) AS freq
-    FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins
-        ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''}
-    GROUP BY bin, bins_number, bin_width, nulls_count, avg_val
-        ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''}
-    ORDER BY bin
-`;
-
 const BIN_MIN_NUMBER = 6;
 const BIN_MAX_NUMBER = 48;
 
@@ -167,14 +68,14 @@ module.exports = class NumericHistogram extends BaseHistogram {
 
     _buildQuery (psql, override, callback) {
         const histogramSql = this._buildQueryTpl({
-            override: override,
-            column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column,
+            column: this._columnType === 'date' ? utils.columnCastTpl({ column: this.column }) : this.column,
             isFloatColumn: this._columnType === 'float',
             query: this.query,
             start: this._getBinStart(override),
             end: this._getBinEnd(override),
+            bins: this._getBinsCount(override),
             minBins: BIN_MIN_NUMBER,
-            maxBins: BIN_MAX_NUMBER,
+            maxBins: BIN_MAX_NUMBER
         });
 
         debug(histogramSql);
@@ -182,19 +83,62 @@ module.exports = class NumericHistogram extends BaseHistogram {
         return callback(null, histogramSql);
     }
 
+
+/**
+ * ctx: Object with the following values
+ * ctx.column -- Column for the histogram
+ * ctx.isFloatColumn - Whether the column is float or not
+ * ctx.query -- Subquery to extract data
+ * ctx.start -- Start value for the bins. [>= end to force calculation]
+ * ctx.end -- End value for the bins.
+ * ctx.bins -- Numbers of bins to generate [<0 to force calculation]
+ * ctx.minBins - If !full min bins to calculate [Optional]
+ * ctx.maxBins - If !full max bins to calculate [Optional]
+ */
     _buildQueryTpl (ctx) {
+        var extra_tables = ``;
+        var extra_queries = ``;
+        var extra_groupby = ``;
+
+        if (ctx.start >= ctx.end) {
+            ctx.end = `__cdb_basics.__cdb_max_val`;
+            ctx.start = `__cdb_basics.__cdb_min_val`;
+            extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`;
+            extra_tables = `, __cdb_basics`;
+            extra_queries = `WITH ${irqQueryTpl(ctx)}`;
+        }
+
+        if (ctx.bins <= 0) {
+            ctx.bins = `__cdb_bins.__cdb_bins_number`;
+            extra_groupby += `, __cdb_bins.__cdb_bins_number`;
+            extra_tables += `, __cdb_bins`;
+            extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`;
+        }
+
         return `
-            WITH
-            ${filteredQueryTpl(ctx)},
-            ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)},
-            ${this._hasOverridenBins(ctx.override) ?
-                overrideBinsQueryTpl(ctx) :
-                `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`
-            },
-            ${nullsQueryTpl(ctx)}
-            ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''}
-            ${histogramQueryTpl(ctx)}
-        `;
+${extra_queries}
+SELECT
+    (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width,
+    ${ctx.bins} as bins_number,
+    ${utils.countNULLs(ctx)} AS nulls_count,
+    ${utils.countInfinites(ctx)} AS infinities_count,
+    ${utils.countNaNs(ctx)} AS nans_count,
+    min(${utils.handleFloatColumn(ctx)}) AS min,
+    max(${utils.handleFloatColumn(ctx)}) AS max,
+    avg(${utils.handleFloatColumn(ctx)}) AS avg,
+    sum(CASE WHEN (${utils.handleFloatColumn(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq,
+    CASE WHEN ${ctx.start} = ${ctx.end}
+        THEN 0
+        ELSE GREATEST(1, LEAST(
+                    ${ctx.bins},
+                    WIDTH_BUCKET(${utils.handleFloatColumn(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1
+    END AS bin
+FROM
+(
+    ${ctx.query}
+) __cdb_filtered_source_query${extra_tables}
+GROUP BY bin${extra_groupby}
+ORDER BY bin;`;
     }
 
     _hasOverridenBins (override) {
@@ -204,14 +148,31 @@ module.exports = class NumericHistogram extends BaseHistogram {
     _getSummary (result, override) {
         const firstRow = result.rows[0] || {};
 
+        var total_nulls = 0;
+        var total_infinities = 0;
+        var total_nans = 0;
+        var total_avg = 0;
+        var total_count = 0;
+
+        result.rows.forEach(function(row) {
+            total_nulls += row.nulls_count;
+            total_infinities += row.infinities_count;
+            total_nans += row.nans_count;
+            total_avg += row.avg * row.freq;
+            total_count += row.freq;
+        });
+        if (total_count !== 0) {
+            total_avg /= total_count;
+        }
+
         return {
             bin_width: firstRow.bin_width,
             bins_count: firstRow.bins_number,
             bins_start: this._populateBinStart(firstRow, override),
-            nulls: firstRow.nulls_count,
-            infinities: firstRow.infinities_count,
-            nans: firstRow.nans_count,
-            avg: firstRow.avg_val,
+            nulls: total_nulls,
+            infinities: total_infinities,
+            nans: total_nans,
+            avg: total_avg
         };
     }
 

lib/cartodb/models/dataview/overviews/formula.js

@@ -1,55 +1,38 @@
 var BaseOverviewsDataview = require('./base');
 var BaseDataview = require('../formula');
 var debug = require('debug')('windshaft:widget:formula:overview');
+const utils = require('../../../utils/query-utils');
 
 var dot = require('dot');
 dot.templateSettings.strip = false;
 
-var formulaQueryTpls = {
-    'count': dot.template([
-        'SELECT',
-        'sum(_feature_count) AS result,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
-        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
-        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-        'FROM ({{=it._query}}) _cdb_formula'
-    ].join('\n')),
-    'sum': dot.template([
-        'SELECT',
-        'sum({{=it._column}}*_feature_count) AS result,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
-        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
-        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
-        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-        'FROM ({{=it._query}}) _cdb_formula',
-        '{{?it._isFloatColumn}}WHERE',
-        '  {{=it._column}} != \'infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'-infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'NaN\'::float{{?}}'
-    ].join('\n')),
-    'avg': dot.template([
-        'SELECT',
-        'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,',
-        '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count',
-        '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities',
-        '  WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count',
-        ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans',
-        '  WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}',
-        'FROM ({{=it._query}}) _cdb_formula',
-        '{{?it._isFloatColumn}}WHERE',
-        '  {{=it._column}} != \'infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'-infinity\'::float',
-        'AND',
-        '  {{=it._column}} != \'NaN\'::float{{?}}'
-    ].join('\n')),
+const VALID_OPERATIONS = {
+    count: true,
+    sum: true,
+    avg: true
 };
 
+/** Formulae to calculate the end result using _feature_count from the overview table*/
+function dataviewResult(ctx) {
+    switch (ctx.operation) {
+        case 'count':
+            return `sum(_feature_count)`;
+        case 'sum':
+            return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)`;
+        case 'avg':
+            return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)/sum(_feature_count) `;
+    }
+    return `${ctx.operation}(${utils.handleFloatColumn(ctx)})`;
+}
+
+const formulaQueryTpl = ctx =>
+`SELECT
+    ${dataviewResult(ctx)} AS result,
+    ${utils.countNULLs(ctx)} AS nulls_count
+    ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``}
+    ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``}
+FROM (${ctx.query}) __cdb_formula`;
+
 function Formula(query, options, queryRewriter, queryRewriteData, params, queries) {
     BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries);
     this.column = options.column || '1';
@@ -65,36 +48,31 @@ module.exports = Formula;
 
 Formula.prototype.sql = function (psql, override, callback) {
     var self = this;
-    var formulaQueryTpl = formulaQueryTpls[this.operation];
-
-    if (formulaQueryTpl) {
-        // supported formula for use with overviews
-        if (this._isFloatColumn === null) {
-            this._isFloatColumn = false;
-            this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
-                if (!err && !!type) {
-                    self._isFloatColumn = type.float;
-                }
-                self.sql(psql, override, callback);
-            });
-            return null;
-        }
-
-        var formulaSql = formulaQueryTpl({
-            _isFloatColumn: this._isFloatColumn,
-            _query: this.rewrittenQuery(this.query),
-            _operation: this.operation,
-            _column: this.column
-        });
-
-        callback = callback || override;
-
-        debug(formulaSql);
-
-        return callback(null, formulaSql);
+    if (!VALID_OPERATIONS[this.operation]) {
+        return this.defaultSql(psql, override, callback);
     }
 
+    if (this._isFloatColumn === null) {
+        this._isFloatColumn = false;
+        this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) {
+            if (!err && !!type) {
+                self._isFloatColumn = type.float;
+            }
+            self.sql(psql, override, callback);
+        });
+        return null;
+    }
 
-    // default behaviour
-    return this.defaultSql(psql, override, callback);
+    var formulaSql = formulaQueryTpl({
+        isFloatColumn: this._isFloatColumn,
+        query: this.rewrittenQuery(this.query),
+        operation: this.operation,
+        column: this.column
+    });
+
+    callback = callback || override;
+
+    debug(formulaSql);
+
+    return callback(null, formulaSql);
 };

lib/cartodb/utils/query-utils.js

@@ -22,5 +22,36 @@ module.exports.extractTableNames = function extractTableNames(query) {
 };
 
 module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
-    return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
+    return 'select CDB_EstimateRowCount($$' + query + '$$) as rows';
 };
+
+/** Cast the column to epoch */
+module.exports.columnCastTpl = function columnCastTpl(ctx) {
+    return `date_part('epoch', ${ctx.column})`;
+};
+
+/** If the column type is float, ignore any non numeric result (infinity / NaN) */
+module.exports.handleFloatColumn = function handleFloatColumn(ctx) {
+    return `${!ctx.isFloatColumn ? `${ctx.column}` :
+        `nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)`
+    }`;
+};
+
+/** Count NULL appearances */
+module.exports.countNULLs= function countNULLs(ctx) {
+    return `sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END)`;
+};
+
+/** Count only infinity (positive and negative) appearances */
+module.exports.countInfinites = function countInfinites(ctx) {
+    return `${!ctx.isFloatColumn ? `0` :
+        `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)`
+    }`;
+};
+
+/** Count only NaNs appearances*/
+module.exports.countNaNs = function countNaNs(ctx) {
+    return `${!ctx.isFloatColumn ? `0` :
+        `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
+    }`;
+};

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "4.2.0",
+  "version": "4.3.1",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"

test/acceptance/dataviews/histogram.js

@@ -186,7 +186,7 @@ describe('histogram-dataview for date column type', function() {
             },
             minute_histogram: {
                 source: {
-                    id: 'minute-histogram-source'
+                    id: 'minute-histogram-source-tz'
                 },
                 type: 'histogram',
                 options: {
@@ -214,8 +214,8 @@ describe('histogram-dataview for date column type', function() {
                 "params": {
                     "query": [
                         "select null::geometry the_geom_webmercator, date AS d",
-                        "from generate_series(",
-                            "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval",
+                        "from generate_series('2007-02-15 01:00:00+00'::timestamptz,",
+                        "'2008-04-09 01:00:00+00'::timestamptz, '1 day'::interval",
                         ") date"
                     ].join(' ')
                 }
@@ -233,13 +233,13 @@ describe('histogram-dataview for date column type', function() {
                 }
             },
             {
-                "id": "minute-histogram-source",
+                "id": "minute-histogram-source-tz",
                 "type": "source",
                 "params": {
                     "query": [
                         "select null::geometry the_geom_webmercator, date AS d",
-                        "from generate_series(",
-                            "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval",
+                        "from generate_series('2007-02-15 23:50:00+00'::timestamptz,",
+                        "'2007-02-16 00:10:00+00'::timestamptz, '1 minute'::interval",
                         ") date"
                     ].join(' ')
                 }
@@ -256,6 +256,7 @@ describe('histogram-dataview for date column type', function() {
     }];
 
     dateHistogramsUseCases.forEach(function (test) {
+
         it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) {
             var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes
 
@@ -323,7 +324,7 @@ describe('histogram-dataview for date column type', function() {
                 assert.ok(!err, err);
                 assert.equal(dataview.type, 'histogram');
                 assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width);
-                assert.equal(dataview.bins.length, 6);
+                assert.equal(dataview.bins_count, 6);
                 dataview.bins.forEach(function (bin) {
                     assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin));
                 });
@@ -335,7 +336,7 @@ describe('histogram-dataview for date column type', function() {
         it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) {
             var params = {
                 start: -2145916800,
-                end: 1009843199
+                end: 1193792400
             };
 
             this.testClient = new TestClient(mapConfig, 1234);
@@ -348,27 +349,6 @@ describe('histogram-dataview for date column type', function() {
             });
         });
 
-        it('should return same histogram ' + test.desc, function (done) {
-            var params = {
-                start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
-                end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
-            };
-
-            this.testClient = new TestClient(mapConfig, 1234);
-            this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) {
-                assert.ok(!err, err);
-
-                this.testClient = new TestClient(mapConfig, 1234);
-                this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) {
-                    assert.ok(!err, err);
-
-                    assert.deepEqual(dataview, filteredDataview);
-                    done();
-                });
-            });
-        });
-
-
         it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) {
             var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time)
             var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time)
@@ -533,6 +513,26 @@ describe('histogram-dataview for date column type', function() {
         });
     });
 
+    it('should return same histogram ', function (done) {
+        var params = {
+            start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum)
+            end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum)
+        };
+
+        this.testClient = new TestClient(mapConfig, 1234);
+        this.testClient.getDataview('datetime_histogram_tz', {}, function (err, dataview) {
+            assert.ok(!err, err);
+
+            this.testClient = new TestClient(mapConfig, 1234);
+            this.testClient.getDataview('datetime_histogram_tz', params, function (err, filteredDataview) {
+                assert.ok(!err, err);
+
+                assert.deepEqual(dataview, filteredDataview);
+                done();
+            });
+        });
+    });
+
     it('should find the best aggregation (automatic mode) to build the histogram', function (done) {
         var params = {};
         this.testClient = new TestClient(mapConfig, 1234);
@@ -640,7 +640,7 @@ describe('histogram-dataview for date column type', function() {
 
         var dataviewWithDailyAggFixture = {
             aggregation: 'day',
-            bin_width: 600,
+            bin_width: 86400,
             bins_count: 2,
             bins_start: 1171497600,
             timestamp_start: 1171497600,
@@ -650,17 +650,17 @@ describe('histogram-dataview for date column type', function() {
             [{
                 bin: 0,
                 timestamp: 1171497600,
-                min: 1171583400,
-                max: 1171583940,
-                avg: 1171583670,
+                min: 1171497600,
+                max: 1171497600,
+                avg: 1171497600,
                 freq: 10
             },
             {
                 bin: 1,
                 timestamp: 1171584000,
                 min: 1171584000,
-                max: 1171584600,
-                avg: 1171584300,
+                max: 1171584000,
+                avg: 1171584000,
                 freq: 11
             }],
             type: 'histogram'
@@ -687,19 +687,19 @@ describe('histogram-dataview for date column type', function() {
 
         var dataviewWithDailyAggAndOffsetFixture = {
             aggregation: 'day',
-            bin_width: 1200,
+            bin_width: 86400,
             bins_count: 1,
             bins_start: 1171501200,
-            timestamp_start: 1171497600,
+            timestamp_start: 1171501200,
             nulls: 0,
             offset: -3600,
             bins:
             [{
                 bin: 0,
                 timestamp: 1171501200,
-                min: 1171583400,
-                max: 1171584600,
-                avg: 1171584000,
+                min: 1171501200,
+                max: 1171501200,
+                avg: 1171501200,
                 freq: 21
             }],
             type: 'histogram'

test/acceptance/named_maps_static_view.js

@@ -19,7 +19,8 @@ describe('named maps static view', function() {
     var username = 'localhost';
     var templateName = 'template_with_view';
 
-    var IMAGE_TOLERANCE = 20;
+    var PNG_IMAGE_TOLERANCE = 20;
+    var JPG_IMAGE_TOLERANCE = 100;
 
     function createTemplate(view, layers) {
         return {
@@ -92,8 +93,8 @@ describe('named maps static view', function() {
         });
     }
 
-    function previewFixture(version) {
-        return './test/fixtures/previews/populated_places_simple_reduced-' + version + '.png';
+    function previewFixture(version, format='png') {
+        return './test/fixtures/previews/populated_places_simple_reduced-' + version + '.' + format;
     }
 
     it('should return an image estimating its bounds based on dataset', function (done) {
@@ -103,7 +104,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('estimated'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('estimated'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -122,7 +123,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -142,7 +143,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('bounds'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -167,7 +168,7 @@ describe('named maps static view', function() {
             }
             getStaticMap(function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('zoom-center'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -192,7 +193,7 @@ describe('named maps static view', function() {
             }
             getStaticMap({ zoom: 3 }, function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('override-zoom'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -217,7 +218,7 @@ describe('named maps static view', function() {
             }
             getStaticMap({ bbox: '0,45,90,45' }, function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('override-bbox'), PNG_IMAGE_TOLERANCE, done);
             });
         });
     });
@@ -256,7 +257,27 @@ describe('named maps static view', function() {
             }
             getStaticMap({ layer: 0 }, function(err, img) {
                 assert.ok(!err);
-                assert.imageIsSimilarToFile(img, previewFixture('bounds'), IMAGE_TOLERANCE, done);
+                assert.imageIsSimilarToFile(img, previewFixture('bounds'), PNG_IMAGE_TOLERANCE, done);
+            });
+        });
+    });
+
+    it('should return jpg static map', function (done) {
+        var view = {
+            zoom: 4,
+            center: {
+                lng: 40,
+                lat: 20
+            }
+        };
+        templateMaps.addTemplate(username, createTemplate(view), function (err) {
+            if (err) {
+                return done(err);
+            }
+            getStaticMap({ format: 'jpeg' }, function(err, img) {
+                assert.ok(!err);
+                assert.imageIsSimilarToFile(img, previewFixture('zoom-center', 'jpeg'),
+                                            JPG_IMAGE_TOLERANCE, done, 'jpeg');
             });
         });
     });

test/results/jpeg/.gitignore

@@ -0,0 +1,3 @@
+*.jpeg
+*.jpg
+*.js

test/support/assert.js

@@ -37,7 +37,7 @@ assert.imageBuffersAreSimilar = function(bufferA, bufferB, tolerance, callback)
     imagesAreSimilar(testImage, referenceImage, tolerance, callback);
 };
 
-assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback) {
+assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath, tolerance, callback, format='png') {
     callback = callback || function(err) { assert.ifError(err); };
 
     var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath);
@@ -46,19 +46,23 @@ assert.imageIsSimilarToFile = function(testImage, referenceImageRelativeFilePath
 
     imagesAreSimilar(testImage, referenceImage, tolerance, function(err) {
         if (err) {
-            var testImageFilePath = randomImagePath();
-            testImage.save(testImageFilePath);
+            var testImageFilePath = randomImagePath(format);
+            testImage.save(testImageFilePath, format);
         }
         callback(err);
-    });
+    }, format);
 };
 
-function imagesAreSimilar(testImage, referenceImage, tolerance, callback) {
+function imagesAreSimilar(testImage, referenceImage, tolerance, callback, format='png') {
     if (testImage.width() !== referenceImage.width() || testImage.height() !== referenceImage.height()) {
         return callback(new Error('Images are not the same size'));
     }
 
-    var pixelsDifference = referenceImage.compare(testImage);
+    var options = {};
+    if (format === 'jpeg') {
+        options.alpha = false;
+    }
+    var pixelsDifference = referenceImage.compare(testImage, options);
     var similarity = pixelsDifference / (referenceImage.width() * referenceImage.height());
     var tolerancePerMil = (tolerance / 1000);
 
@@ -73,8 +77,8 @@ function imagesAreSimilar(testImage, referenceImage, tolerance, callback) {
     }
 }
 
-function randomImagePath() {
-    return path.resolve('test/results/png/image-test-' + Date.now() + '.png');
+function randomImagePath(format='png') {
+    return path.resolve('test/results/' + format + '/image-test-' + Date.now() + '.' + format);
 }
 
 assert.response = function(server, req, res, callback) {