diff --git a/NEWS.md b/NEWS.md index 588166b9..20b8f06a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,16 @@ # Changelog -## 6.0.1 +## 6.1.0 Released 2018-mm-dd +New features: +- Aggreation filters + +Bug Fixes: +- Non-default aggregation selected the wrong columns (e.g. for vector tiles) +- Aggregation dimensions with alias where broken +- cartodb_id was not unique accross aggregated vector tiles + ## 6.0.0 Released 2018-03-19 Backward incompatible changes: diff --git a/docs/MapConfig-Aggregation-extension.md b/docs/MapConfig-Aggregation-extension.md index 850b179a..bf2172f2 100644 --- a/docs/MapConfig-Aggregation-extension.md +++ b/docs/MapConfig-Aggregation-extension.md @@ -29,7 +29,7 @@ The value of this attribute can be `false` to explicitly disable aggregation for // object, defines the columns of the aggregated datasets. Each property corresponds to a columns name and // should contain an object with two properties: "aggregate_function" (one of "sum", "max", "min", "avg", "mode" or "count"), // and "aggregated_column" (the name of a column of the original layer query or "*") - // A column defined as `"_cdb_features_count": {"aggregate_function": "count", aggregated_column: "*"}` + // A column defined as `"_cdb_feature_count": {"aggregate_function": "count", aggregated_column: "*"}` // is always generated in addition to the defined columns. // The column names `cartodb_id`, `the_geom`, `the_geom_webmercator` and `_cdb_feature_count` cannot be used // for aggregated columns, as they correspond to columns always present in the result. diff --git a/docs/aggregation.md b/docs/aggregation.md index 2fb361c3..6920776f 100644 --- a/docs/aggregation.md +++ b/docs/aggregation.md @@ -10,7 +10,7 @@ Aggregation is available only for point geometries. During aggregation the point When no placement or columns are specified a special default aggregation is performed. -This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_features_count` with the number of features in the group. +This special mode performs only spatial aggregation (using a grid defined by the requested tile and the resolution, parameter, as all the other cases), and returns a _random_ record from each group (grid cell) with all its columns and an additional `_cdb_feature_count` with the number of features in the group. Regarding the randomness of the sample: currently we use the row with the minimum `cartodb_id` value in each group. @@ -18,7 +18,7 @@ The rationale behind having this special aggregation with all the original colum ### User defined aggregations -When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_features_count`, which is always present. +When either a explicit placement or columns are requested we no longer use the special, query; we use one determined by the placement (which will default to "centroid"), and it will have as columns only the aggregated columns specified, in addition to `_cdb_feature_count`, which is always present. We might decide in the future to allow sampling column values for any of the different placement modes. @@ -185,3 +185,80 @@ This is the minimum number of (estimated) rows in the dataset (query results) fo ] } ``` + +### `filters` + +Aggregated data can be filtered by imposing filtering conditions on the aggregated columns. + +Each condition is represented by one or more parameters: + +* `{ "equal": V }` selects an specific value of the aggregated column. +* `{ "not_equal": V }` selects values different from the one specified. +* `{ "in": [v1, v2, v3] }` selects any value from a list. +* `{ "not_in": [v1, v2, v3] }` selects any value not in a list. +* `{ "less_than": v }` selects values strictly less than the one given. +* `{ "less_than_or_equal_to": v }` selects values less than or equal to the one given. +* `{ "greater_than": v }` selects values strictly greater than the one given. +* `{ "greater_than_or_equal_to": v }` selects values greater than or equal to the one given. + +One of the *less* conditions can be combined with one of the *greater* conditions to select a range of values, for example: +* `{ "greater_than": v1, "less_than": v2 }` +* `{ "greater_than_or_equal_to": v1, "less_than": v2 }` +* `{ "greater_than": v1, "less_than_or_equal_to": v2 }` +* `{ "greater_than_or_equal_to": v1, "less_than_or_equal_to": v2 }` + +For a given column, multiple conditions can be passed in an array; the conditions will logically ORed (any of the conditions have to be verifid for the value to be selected): + +* `"myvalue": [ { "equal": 10 }, { "less_than": 0 }]` will select values of the column `myvalue` which are equal to 10 **or** less than 0. + +In addition, the filters applied to different columns are logically combined with AND (all the conditions have to be satisfied for an element to be selected); for example with the following `filters` parameter we'll select aggregated records which have a `total_value` > 100 **and** a category equal to "a". + +```json +{ + "total_value": { "greater_than": 100 }, + "category": { "equal": "a" } +} +``` + +Note that the filtered columns have to be defined with the `columns` parameter, except for `_cdb_feature_count`, which is always implicitly defined and can be filtered too. + +#### Example + +```json +{ + "version": "1.7.0", + "extent": [-20037508.5, -20037508.5, 20037508.5, 20037508.5], + "srid": 3857, + "maxzoom": 18, + "minzoom": 3, + "layers": [ + { + "type": "mapnik", + "options": { + "sql": "select * from table", + "cartocss": "#table { marker-width: [total]; marker-fill: ramp(value, (red, green, blue), jenks); }", + "cartocss_version": "2.3.0", + "aggregation": { + "placement": "centroid", + "columns": { + "total_value": { + "aggregate_function": "sum", + "aggregated_column": "value" + }, + "category": { + "aggregate_function": "mode", + "aggregated_column": "category" + } + }, + "filters" : { + "total_value": { "greater_than": 100 }, + "category": { "equal": "a" } + }, + "resolution": 2, + "threshold": 500000 + } + } + } + ] +} +``` diff --git a/lib/cartodb/api/auth_api.js b/lib/cartodb/api/auth_api.js index e9f10262..ff42c9df 100644 --- a/lib/cartodb/api/auth_api.js +++ b/lib/cartodb/api/auth_api.js @@ -25,7 +25,7 @@ module.exports = AuthApi; // null if the request is not signed by anyone // or will be a string cartodb username otherwise. // -AuthApi.prototype.authorizedBySigner = function(res, callback) { +AuthApi.prototype.authorizedBySigner = function(req, res, callback) { if ( ! res.locals.token || ! res.locals.signer ) { return callback(null, false); // no signer requested } @@ -33,7 +33,7 @@ AuthApi.prototype.authorizedBySigner = function(res, callback) { var self = this; var layergroup_id = res.locals.token; - var auth_token = res.locals.auth_token; + var auth_token = req.query.auth_token; this.mapStore.load(layergroup_id, function(err, mapConfig) { if (err) { @@ -109,7 +109,7 @@ AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) { return callback(error); } - return callback(null, true); + return callback(null, true, apikey); }); }; @@ -180,7 +180,7 @@ AuthApi.prototype.authorize = function(req, res, callback) { }); } - this.authorizedBySigner(res, (err, isAuthorizedBySigner) => { + this.authorizedBySigner(req, res, (err, isAuthorizedBySigner) => { if (err) { return callback(err); } diff --git a/lib/cartodb/backends/analysis-status.js b/lib/cartodb/backends/analysis-status.js index 97f851d2..0060f36b 100644 --- a/lib/cartodb/backends/analysis-status.js +++ b/lib/cartodb/backends/analysis-status.js @@ -5,16 +5,14 @@ function AnalysisStatusBackend() { module.exports = AnalysisStatusBackend; - -AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { - var nodeId = params.nodeId; - +AnalysisStatusBackend.prototype.getNodeStatus = function (nodeId, dbParams, callback) { var statusQuery = [ 'SELECT node_id, status, updated_at, last_error_message as error_message', 'FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\'' ].join(' '); - var pg = new PSQL(dbParamsFromReqParams(params)); + var pg = new PSQL(dbParams); + pg.query(statusQuery, function(err, result) { if (err) { return callback(err, result); @@ -36,23 +34,3 @@ AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { return callback(null, statusResponse); }, true); // use read-only transaction }; - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index c110f328..4eebb2a2 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -1,13 +1,11 @@ var assert = require('assert'); - var _ = require('underscore'); var PSQL = require('cartodb-psql'); var step = require('step'); - var BBoxFilter = require('../models/filter/bbox'); - var DataviewFactory = require('../models/dataview/factory'); var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory'); +const dbParamsFromReqParams = require('../utils/database-params'); var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' @@ -48,7 +46,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param } var pg = new PSQL(dbParamsFromReqParams(params)); - + var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters); if (params.bbox) { var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox}); @@ -170,23 +168,3 @@ function getDataviewDefinition(mapConfig, dataviewName) { var dataviews = mapConfig.dataviews || {}; return dataviews[dataviewName]; } - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index ebafbbac..e0850822 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -6,12 +6,20 @@ var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function NamedMapProviderCache( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.mapConfigAdapter = mapConfigAdapter; + this.affectedTablesCache = affectedTablesCache; this.providerCache = new LruCache({ max: 2000 }); } @@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.metadataBackend, this.userLimitsApi, this.mapConfigAdapter, + this.affectedTablesCache, user, templateId, config, diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 791f147d..2d89a091 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -1,14 +1,15 @@ const PSQL = require('cartodb-psql'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); -const locals = require('../middleware/locals'); const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const authorize = require('../middleware/authorize'); const dbConnSetup = require('../middleware/db-conn-setup'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const cacheControlHeader = require('../middleware/cache-control-header'); +const sendResponse = require('../middleware/send-response'); +const dbParamsFromResLocals = require('../utils/database-params'); function AnalysesController(pgConnection, authApi, userLimitsApi) { this.pgConnection = pgConnection; @@ -24,19 +25,17 @@ AnalysesController.prototype.register = function (app) { app.get( `${mapconfigBasePath}/analyses/catalog`, cors(), - cleanUpQueryParams(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), - layergroupToken(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG), + cleanUpQueryParams(), createPGClient(), getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), prepareResponse(), - setCacheControlHeader(), + cacheControlHeader({ ttl: 10, revalidate: true }), sendResponse(), unauthorizedError() ); @@ -44,7 +43,10 @@ AnalysesController.prototype.register = function (app) { function createPGClient () { return function createPGClientMiddleware (req, res, next) { - res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); + const dbParams = dbParamsFromResLocals(res.locals); + + res.locals.pg = new PSQL(dbParams); + next(); }; } @@ -111,25 +113,6 @@ function prepareResponse () { }; } -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - }; -} - function unauthorizedError () { return function unathorizedErrorMiddleware(err, req, res, next) { if (err.message.match(/permission\sdenied/)) { @@ -163,23 +146,3 @@ var tablesQueryTpl = ctx => ` FROM analysis_tables ORDER BY size DESC `; - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js deleted file mode 100644 index eb45c022..00000000 --- a/lib/cartodb/controllers/layergroup.js +++ /dev/null @@ -1,716 +0,0 @@ -const cors = require('../middleware/cors'); -const user = require('../middleware/user'); -const vectorError = require('../middleware/vector-error'); -const locals = require('../middleware/locals'); -const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const layergroupToken = require('../middleware/layergroup-token'); -const credentials = require('../middleware/credentials'); -const dbConnSetup = require('../middleware/db-conn-setup'); -const authorize = require('../middleware/authorize'); -const rateLimit = require('../middleware/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; -const DataviewBackend = require('../backends/dataview'); -const AnalysisStatusBackend = require('../backends/analysis-status'); -const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); -const QueryTables = require('cartodb-query-tables'); -const SUPPORTED_FORMATS = { - grid_json: true, - json_torque: true, - torque_json: true, - png: true, - png32: true, - mvt: true -}; - -/** - * @param {prepareContext} prepareContext - * @param {PgConnection} pgConnection - * @param {MapStore} mapStore - * @param {TileBackend} tileBackend - * @param {PreviewBackend} previewBackend - * @param {AttributesBackend} attributesBackend - * @param {SurrogateKeysCache} surrogateKeysCache - * @param {UserLimitsApi} userLimitsApi - * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {AnalysisBackend} analysisBackend - * @constructor - */ -function LayergroupController( - pgConnection, - mapStore, - tileBackend, - previewBackend, - attributesBackend, - surrogateKeysCache, - userLimitsApi, - layergroupAffectedTables, - analysisBackend, - authApi -) { - this.pgConnection = pgConnection; - this.mapStore = mapStore; - this.tileBackend = tileBackend; - this.previewBackend = previewBackend; - this.attributesBackend = attributesBackend; - this.surrogateKeysCache = surrogateKeysCache; - this.userLimitsApi = userLimitsApi; - this.layergroupAffectedTables = layergroupAffectedTables; - - this.dataviewBackend = new DataviewBackend(analysisBackend); - this.analysisStatusBackend = new AnalysisStatusBackend(); - this.authApi = authApi; -} - -module.exports = LayergroupController; - -LayergroupController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath } = app; - - app.get( - `${mapconfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, - cors(), - cleanUpQueryParams(), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - incrementSuccessMetrics(global.statsClient), - sendResponse(), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - app.get( - `${mapconfigBasePath}/:token/:z/:x/:y.:format`, - cors(), - cleanUpQueryParams(), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getTile(this.tileBackend, 'map_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - incrementSuccessMetrics(global.statsClient), - sendResponse(), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - app.get( - `${mapconfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, - distinguishLayergroupFromStaticRoute(), - cors(), - cleanUpQueryParams(), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getTile(this.tileBackend, 'maplayer_tile'), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - incrementSuccessMetrics(global.statsClient), - sendResponse(), - incrementErrorMetrics(global.statsClient), - tileError(), - vectorError() - ); - - app.get( - `${mapconfigBasePath}/:token/:layer/attributes/:fid`, - cors(), - cleanUpQueryParams(), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getFeatureAttributes(this.attributesBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - const forcedFormat = 'png'; - - app.get( - `${mapconfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, - cors(), - cleanUpQueryParams(['layer']), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), - getPreviewImageByCenter(this.previewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${mapconfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, - cors(), - cleanUpQueryParams(['layer']), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat), - getPreviewImageByBoundingBox(this.previewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - // Undocumented/non-supported API endpoint methods. - // Use at your own peril. - - const allowedDataviewQueryParams = [ - 'filters', // json - 'own_filter', // 0, 1 - 'no_filters', // 0, 1 - 'bbox', // w,s,e,n - 'start', // number - 'end', // number - 'column_type', // string - 'bins', // number - 'aggregation', //string - 'offset', // number - 'q', // widgets search - 'categories', // number - ]; - - app.get( - `${mapconfigBasePath}/:token/dataview/:dataviewName`, - cors(), - cleanUpQueryParams(allowedDataviewQueryParams), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getDataview(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${mapconfigBasePath}/:token/:layer/widget/:dataviewName`, - cors(), - cleanUpQueryParams(allowedDataviewQueryParams), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - getDataview(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${mapconfigBasePath}/:token/dataview/:dataviewName/search`, - cors(), - cleanUpQueryParams(allowedDataviewQueryParams), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${mapconfigBasePath}/:token/:layer/widget/:dataviewName/search`, - cors(), - cleanUpQueryParams(allowedDataviewQueryParams), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi), - dataviewSearch(this.dataviewBackend), - setCacheControlHeader(), - setLastModifiedHeader(), - getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore), - setCacheChannelHeader(), - setSurrogateKeyHeader(this.surrogateKeysCache), - sendResponse() - ); - - app.get( - `${mapconfigBasePath}/:token/analysis/node/:nodeId`, - cors(), - cleanUpQueryParams(), - locals(), - user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), - layergroupToken(), - credentials(), - authorize(this.authApi), - dbConnSetup(this.pgConnection), - analysisNodeStatus(this.analysisStatusBackend), - sendResponse() - ); -}; - -function distinguishLayergroupFromStaticRoute () { - return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { - if (req.params.token === 'static') { - return next('route'); - } - - next(); - }; -} - -function analysisNodeStatus (analysisStatusBackend) { - return function analysisNodeStatusMiddleware(req, res, next) { - analysisStatusBackend.getNodeStatus(res.locals, (err, nodeStatus, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET NODE STATUS'; - return next(err); - } - - res.set({ - 'Cache-Control': 'public,max-age=5', - 'Last-Modified': new Date().toUTCString() - }); - - res.body = nodeStatus; - - next(); - }); - }; -} - -function getRequestParams(locals) { - const params = Object.assign({}, locals); - - delete params.mapConfigProvider; - delete params.allowedQueryParams; - - return params; -} - -function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) { - return function createMapStoreMapConfigProviderMiddleware (req, res, next) { - const { user } = res.locals; - - const params = getRequestParams(res.locals); - - if (forcedFormat) { - params.format = forcedFormat; - params.layer = params.layer || 'all'; - } - - res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params); - - next(); - }; -} - -function getDataview (dataviewBackend) { - return function getDataviewMiddleware (req, res, next) { - const { user, mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET DATAVIEW'; - return next(err); - } - - res.body = dataview; - - next(); - }); - }; -} - -function dataviewSearch (dataviewBackend) { - return function dataviewSearchMiddleware (req, res, next) { - const { user, dataviewName, mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET DATAVIEW SEARCH'; - return next(err); - } - - res.body = searchResult; - - next(); - }); - }; -} - -function getFeatureAttributes (attributesBackend) { - return function getFeatureAttributesMiddleware (req, res, next) { - req.profiler.start('windshaft.maplayer_attribute'); - - const { mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => { - req.profiler.add(stats); - - if (err) { - err.label = 'GET ATTRIBUTES'; - return next(err); - } - - res.body = tile; - - next(); - }); - }; -} - -function getStatusCode(tile, format){ - return tile.length === 0 && format === 'mvt'? 204 : 200; -} - -function parseFormat (format = '') { - const prettyFormat = format.replace('.', '_'); - return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; -} - -function getTile (tileBackend, profileLabel = 'tile') { - return function getTileMiddleware (req, res, next) { - req.profiler.start(`windshaft.${profileLabel}`); - - const { mapConfigProvider } = res.locals; - const params = getRequestParams(res.locals); - - tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { - req.profiler.add(stats); - - if (err) { - return next(err); - } - - if (headers) { - res.set(headers); - } - - const formatStat = parseFormat(req.params.format); - - res.statusCode = getStatusCode(tile, formatStat); - res.body = tile; - - next(); - }); - }; -} - -function getPreviewImageByCenter (previewBackend) { - return function getPreviewImageByCenterMiddleware (req, res, next) { - const width = +req.params.width; - const height = +req.params.height; - const zoom = +req.params.z; - const center = { - lng: +req.params.lng, - lat: +req.params.lat - }; - - const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - const { mapConfigProvider: provider } = res.locals; - - previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => { - req.profiler.done(`render-${format}`); - req.profiler.add(stats); - - if (err) { - err.label = 'STATIC_MAP'; - return next(err); - } - - if (headers) { - res.set(headers); - } - - res.set('Content-Type', headers['Content-Type'] || `image/${format}`); - - res.body = image; - - next(); - }); - }; -} - -function getPreviewImageByBoundingBox (previewBackend) { - return function getPreviewImageByBoundingBoxMiddleware (req, res, next) { - const width = +req.params.width; - const height = +req.params.height; - const bounds = { - west: +req.params.west, - north: +req.params.north, - east: +req.params.east, - south: +req.params.south - }; - const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; - const { mapConfigProvider: provider } = res.locals; - - previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => { - req.profiler.done(`render-${format}`); - req.profiler.add(stats); - - if (err) { - err.label = 'STATIC_MAP'; - return next(err); - } - - if (headers) { - res.set(headers); - } - - res.set('Content-Type', headers['Content-Type'] || `image/${format}`); - - res.body = image; - - next(); - }); - }; -} - -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware (req, res, next) { - let { cache_buster: cacheBuster } = res.locals; - - cacheBuster = parseInt(cacheBuster, 10); - - const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date(); - - res.set('Last-Modified', lastUpdated.toUTCString()); - - next(); - }; -} - -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware (req, res, next) { - res.set('Cache-Control', 'public,max-age=31536000'); - - next(); - }; -} - -function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) { - return function getAffectedTablesMiddleware (req, res, next) { - const { user, dbname, token } = res.locals; - - if (layergroupAffectedTables.hasAffectedTables(dbname, token)) { - res.locals.affectedTables = layergroupAffectedTables.get(dbname, token); - return next(); - } - - mapStore.load(token, (err, mapconfig) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - const queries = []; - mapconfig.getLayers().forEach(function(layer) { - queries.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - queries.push(`SELECT * FROM ${table} LIMIT 0`); - }); - } - }); - - const sql = queries.length ? queries.join(';') : null; - - if (!sql) { - global.logger.warn('ERROR generating cache channel:' + - ' this request doesn\'t need an X-Cache-Channel generated'); - return next(); - } - - pgConnection.getConnection(user, (err, connection) => { - if (err) { - global.logger.warn('ERROR generating cache channel:', err); - return next(); - } - - QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - global.logger.warn('ERROR generating cache channel: ', err); - return next(); - } - - // feed affected tables cache so it can be reused from, for instance, map controller - layergroupAffectedTables.set(dbname, token, affectedTables); - - res.locals.affectedTables = affectedTables; - - next(); - }); - }); - }); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (affectedTables) { - surrogateKeysCache.tag(res, affectedTables); - } - - next(); - }; -} - -function incrementSuccessMetrics (statsClient) { - return function incrementSuccessMetricsMiddleware (req, res, next) { - const formatStat = parseFormat(req.params.format); - - statsClient.increment('windshaft.tiles.success'); - statsClient.increment(`windshaft.tiles.${formatStat}.success`); - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - - res.status(res.statusCode || 200); - - if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(res.body); - } else { - res.json(res.body); - } - } else { - res.send(res.body); - } - }; -} - -function incrementErrorMetrics (statsClient) { - return function incrementErrorMetricsMiddleware (err, req, res, next) { - const formatStat = parseFormat(req.params.format); - - statsClient.increment('windshaft.tiles.error'); - statsClient.increment(`windshaft.tiles.${formatStat}.error`); - - next(err); - }; -} - -function tileError () { - return function tileErrorMiddleware (err, req, res, next) { - // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 - let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); - - // Rewrite mapnik parsing errors to start with layer number - const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); - - if (matches) { - errMsg = `style${matches[2]}: ${matches[1]}`; - } - - err.message = errMsg; - err.label = 'TILE RENDER'; - - next(err); - }; -} diff --git a/lib/cartodb/controllers/layergroup/analysis.js b/lib/cartodb/controllers/layergroup/analysis.js new file mode 100644 index 00000000..3022bce3 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/analysis.js @@ -0,0 +1,75 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const sendResponse = require('../../middleware/send-response'); +const dbParamsFromResLocals = require('../../utils/database-params'); + +module.exports = class AnalysisController { + constructor ( + analysisStatusBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.analysisStatusBackend = analysisStatusBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + app.get( + `${mapConfigBasePath}/:token/analysis/node/:nodeId`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS), + cleanUpQueryParams(), + analysisNodeStatus(this.analysisStatusBackend), + sendResponse() + ); + + } +}; + +function analysisNodeStatus (analysisStatusBackend) { + return function analysisNodeStatusMiddleware(req, res, next) { + const { nodeId } = req.params; + const dbParams = dbParamsFromResLocals(res.locals); + + analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET NODE STATUS'; + return next(err); + } + + res.set({ + 'Cache-Control': 'public,max-age=5', + 'Last-Modified': new Date().toUTCString() + }); + + res.body = nodeStatus; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/attributes.js b/lib/cartodb/controllers/layergroup/attributes.js new file mode 100644 index 00000000..07ae5da1 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/attributes.js @@ -0,0 +1,93 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); + +module.exports = class AttribitesController { + constructor ( + attributesBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.attributesBackend = attributesBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + app.get( + `${mapConfigBasePath}/:token/:layer/attributes/:fid`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getFeatureAttributes(this.attributesBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + } +}; + +function getFeatureAttributes (attributesBackend) { + return function getFeatureAttributesMiddleware (req, res, next) { + req.profiler.start('windshaft.maplayer_attribute'); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer, fid } = req.params; + + const params = { + token, + dbuser, dbname, dbpassword, dbhost, dbport, + layer, fid + }; + + attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET ATTRIBUTES'; + return next(err); + } + + res.body = tile; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/dataview.js b/lib/cartodb/controllers/layergroup/dataview.js new file mode 100644 index 00000000..bca27f1e --- /dev/null +++ b/lib/cartodb/controllers/layergroup/dataview.js @@ -0,0 +1,199 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); + +const ALLOWED_DATAVIEW_QUERY_PARAMS = [ + 'filters', // json + 'own_filter', // 0, 1 + 'no_filters', // 0, 1 + 'bbox', // w,s,e,n + 'start', // number + 'end', // number + 'column_type', // string + 'bins', // number + 'aggregation', //string + 'offset', // number + 'q', // widgets search + 'categories', // number +]; + +module.exports = class DataviewController { + constructor ( + dataviewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.dataviewBackend = dataviewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + // Undocumented/non-supported API endpoint methods. + // Use at your own peril. + + app.get( + `${mapConfigBasePath}/:token/dataview/:dataviewName`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getDataview(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getDataview(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/dataview/:dataviewName/search`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + dataviewSearch(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH), + cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + dataviewSearch(this.dataviewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + } +}; + +function getDataview (dataviewBackend) { + return function getDataviewMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW'; + return next(err); + } + + res.body = dataview; + + next(); + }); + }; +} + +function dataviewSearch (dataviewBackend) { + return function dataviewSearchMiddleware (req, res, next) { + const { user, mapConfigProvider } = res.locals; + const { dataviewName } = req.params; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => { + req.profiler.add(stats); + + if (err) { + err.label = 'GET DATAVIEW SEARCH'; + return next(err); + } + + res.body = searchResult; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/index.js b/lib/cartodb/controllers/layergroup/index.js new file mode 100644 index 00000000..e4c1770c --- /dev/null +++ b/lib/cartodb/controllers/layergroup/index.js @@ -0,0 +1,114 @@ +const DataviewBackend = require('../../backends/dataview'); +const AnalysisStatusBackend = require('../../backends/analysis-status'); + +const TileController = require('./tile'); +const AttributesController = require('./attributes'); +const StaticController = require('./static'); +const DataviewController = require('./dataview'); +const AnalysisController = require('./analysis'); + +/** + * @param {prepareContext} prepareContext + * @param {PgConnection} pgConnection + * @param {MapStore} mapStore + * @param {TileBackend} tileBackend + * @param {PreviewBackend} previewBackend + * @param {AttributesBackend} attributesBackend + * @param {SurrogateKeysCache} surrogateKeysCache + * @param {UserLimitsApi} userLimitsApi + * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {AnalysisBackend} analysisBackend + * @constructor + */ +function LayergroupController( + pgConnection, + mapStore, + tileBackend, + previewBackend, + attributesBackend, + surrogateKeysCache, + userLimitsApi, + layergroupAffectedTablesCache, + analysisBackend, + authApi +) { + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.tileBackend = tileBackend; + this.previewBackend = previewBackend; + this.attributesBackend = attributesBackend; + this.surrogateKeysCache = surrogateKeysCache; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + + this.dataviewBackend = new DataviewBackend(analysisBackend); + this.analysisStatusBackend = new AnalysisStatusBackend(); + this.authApi = authApi; +} + +module.exports = LayergroupController; + +LayergroupController.prototype.register = function(app) { + + const tileController = new TileController( + this.tileBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + tileController.register(app); + + const attributesController = new AttributesController( + this.attributesBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + attributesController.register(app); + + const staticController = new StaticController( + this.previewBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + staticController.register(app); + + const dataviewController = new DataviewController( + this.dataviewBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + dataviewController.register(app); + + const analysisController = new AnalysisController( + this.analysisStatusBackend, + this.pgConnection, + this.mapStore, + this.userLimitsApi, + this.layergroupAffectedTablesCache, + this.authApi, + this.surrogateKeysCache + ); + + analysisController.register(app); + + +}; diff --git a/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js b/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js new file mode 100644 index 00000000..7d64cb01 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js @@ -0,0 +1,37 @@ +const MapStoreMapConfigProvider = require('../../../models/mapconfig/provider/map-store-provider'); + +module.exports = function createMapStoreMapConfigProvider ( + mapStore, + userLimitsApi, + pgConnection, + affectedTablesCache, + forcedFormat = null +) { + return function createMapStoreMapConfigProviderMiddleware (req, res, next) { + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { layer, z, x, y, scale_factor, format } = req.params; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + layer, z, x, y, scale_factor, format + }; + + if (forcedFormat) { + params.format = forcedFormat; + params.layer = params.layer || 'all'; + } + + res.locals.mapConfigProvider = new MapStoreMapConfigProvider( + mapStore, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + params + ); + + next(); + }; +}; diff --git a/lib/cartodb/controllers/layergroup/static.js b/lib/cartodb/controllers/layergroup/static.js new file mode 100644 index 00000000..00510780 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/static.js @@ -0,0 +1,161 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); + +module.exports = class StaticController { + constructor ( + previewBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.previewBackend = previewBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + const forcedFormat = 'png'; + + app.get( + `${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + cleanUpQueryParams(['layer']), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), + getPreviewImageByCenter(this.previewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC), + cleanUpQueryParams(['layer']), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache, + forcedFormat + ), + getPreviewImageByBoundingBox(this.previewBackend), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + sendResponse() + ); + } +}; + +function getPreviewImageByCenter (previewBackend) { + return function getPreviewImageByCenterMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const zoom = +req.params.z; + const center = { + lng: +req.params.lng, + lat: +req.params.lat + }; + + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.body = image; + + next(); + }); + }; +} + +function getPreviewImageByBoundingBox (previewBackend) { + return function getPreviewImageByBoundingBoxMiddleware (req, res, next) { + const width = +req.params.width; + const height = +req.params.height; + const bounds = { + west: +req.params.west, + north: +req.params.north, + east: +req.params.east, + south: +req.params.south + }; + const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; + const { mapConfigProvider: provider } = res.locals; + + previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => { + req.profiler.done(`render-${format}`); + req.profiler.add(stats); + + if (err) { + err.label = 'STATIC_MAP'; + return next(err); + } + + if (headers) { + res.set(headers); + } + + res.set('Content-Type', headers['Content-Type'] || `image/${format}`); + + res.body = image; + + next(); + }); + }; +} diff --git a/lib/cartodb/controllers/layergroup/tile.js b/lib/cartodb/controllers/layergroup/tile.js new file mode 100644 index 00000000..07727236 --- /dev/null +++ b/lib/cartodb/controllers/layergroup/tile.js @@ -0,0 +1,230 @@ +const cors = require('../../middleware/cors'); +const user = require('../../middleware/user'); +const layergroupToken = require('../../middleware/layergroup-token'); +const cleanUpQueryParams = require('../../middleware/clean-up-query-params'); +const credentials = require('../../middleware/credentials'); +const dbConnSetup = require('../../middleware/db-conn-setup'); +const authorize = require('../../middleware/authorize'); +const rateLimit = require('../../middleware/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider'); +const cacheControlHeader = require('../../middleware/cache-control-header'); +const cacheChannelHeader = require('../../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../../middleware/last-modified-header'); +const sendResponse = require('../../middleware/send-response'); +const vectorError = require('../../middleware/vector-error'); + +const SUPPORTED_FORMATS = { + grid_json: true, + json_torque: true, + torque_json: true, + png: true, + png32: true, + mvt: true +}; + +module.exports = class TileController { + constructor ( + tileBackend, + pgConnection, + mapStore, + userLimitsApi, + layergroupAffectedTablesCache, + authApi, + surrogateKeysCache + ) { + this.tileBackend = tileBackend; + this.pgConnection = pgConnection; + this.mapStore = mapStore; + this.userLimitsApi = userLimitsApi; + this.layergroupAffectedTablesCache = layergroupAffectedTablesCache; + this.authApi = authApi; + this.surrogateKeysCache = surrogateKeysCache; + } + + register (app) { + const { base_url_mapconfig: mapConfigBasePath } = app; + + app.get( + `${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend, 'map_tile'), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:z/:x/:y.:format`, + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend, 'map_tile'), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError(), + sendResponse() + ); + + app.get( + `${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`, + distinguishLayergroupFromStaticRoute(), + cors(), + user(), + layergroupToken(), + credentials(), + authorize(this.authApi), + dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE), + cleanUpQueryParams(), + createMapStoreMapConfigProvider( + this.mapStore, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTablesCache + ), + getTile(this.tileBackend, 'maplayer_tile'), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), + incrementSuccessMetrics(global.statsClient), + incrementErrorMetrics(global.statsClient), + tileError(), + vectorError(), + sendResponse() + ); + } +}; + +function distinguishLayergroupFromStaticRoute () { + return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) { + if (req.params.token === 'static') { + return next('route'); + } + + next(); + }; +} + +function parseFormat (format = '') { + const prettyFormat = format.replace('.', '_'); + return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid'; +} + +function getStatusCode(tile, format){ + return tile.length === 0 && format === 'mvt' ? 204 : 200; +} + +function getTile (tileBackend, profileLabel = 'tile') { + return function getTileMiddleware (req, res, next) { + req.profiler.start(`windshaft.${profileLabel}`); + + const { mapConfigProvider } = res.locals; + const { token } = res.locals; + const { layer, z, x, y, format } = req.params; + + const params = { token, layer, z, x, y, format }; + + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => { + req.profiler.add(stats); + + if (err) { + return next(err); + } + + if (headers) { + res.set(headers); + } + + const formatStat = parseFormat(req.params.format); + + res.statusCode = getStatusCode(tile, formatStat); + res.body = tile; + + next(); + }); + }; +} + +function incrementSuccessMetrics (statsClient) { + return function incrementSuccessMetricsMiddleware (req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.success'); + statsClient.increment(`windshaft.tiles.${formatStat}.success`); + + next(); + }; +} + +function incrementErrorMetrics (statsClient) { + return function incrementErrorMetricsMiddleware (err, req, res, next) { + const formatStat = parseFormat(req.params.format); + + statsClient.increment('windshaft.tiles.error'); + statsClient.increment(`windshaft.tiles.${formatStat}.error`); + + next(err); + }; +} + +function tileError () { + return function tileErrorMiddleware (err, req, res, next) { + if (err.message === 'Tile does not exist' && req.params.format === 'mvt') { + res.statusCode = 204; + return next(); + } + + // See https://github.com/Vizzuality/Windshaft-cartodb/issues/68 + let errMsg = err.message ? ( '' + err.message ) : ( '' + err ); + + // Rewrite mapnik parsing errors to start with layer number + const matches = errMsg.match("(.*) in style 'layer([0-9]+)'"); + + if (matches) { + errMsg = `style${matches[2]}: ${matches[1]}`; + } + + err.message = errMsg; + err.label = 'TILE RENDER'; + + next(err); + }; +} diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 71229c51..004dc0c1 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,17 +2,18 @@ const _ = require('underscore'); const windshaft = require('windshaft'); const MapConfig = windshaft.model.MapConfig; const Datasource = windshaft.model.Datasource; -const QueryTables = require('cartodb-query-tables'); const ResourceLocator = require('../models/resource-locator'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); -const locals = require('../middleware/locals'); const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); const LayergroupMetadata = require('../utils/layergroup-metadata'); @@ -63,15 +64,15 @@ function MapController ( module.exports = MapController; MapController.prototype.register = function(app) { - const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app; + const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app; app.get( - `${mapconfigBasePath}`, + `${mapConfigBasePath}`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) ); app.post( - `${mapconfigBasePath}`, + `${mapConfigBasePath}`, this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) ); @@ -87,7 +88,7 @@ MapController.prototype.register = function(app) { this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate) ); - app.options(app.base_url_mapconfig, cors('Content-Type')); + app.options(`${mapConfigBasePath}`, cors('Content-Type')); }; MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) { @@ -99,24 +100,22 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us return [ cors(), - cleanUpQueryParams(['aggregation']), - locals(), user(), - rateLimit(this.userLimitsApi, endpointGroup), - layergroupToken(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, endpointGroup), + cleanUpQueryParams(['aggregation']), initProfiler(isTemplateInstantiation), checkJsonContentType(), this.getCreateMapMiddlewares(useTemplate), incrementMapViewCount(this.metadataBackend), augmentLayergroupData(), - getAffectedTables(this.pgConnection, this.layergroupAffectedTables), - setCacheChannel(), - setLastModified(), + cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader({ now: true }), setLastUpdatedTimeToLayergroup(), - setCacheControl(), setLayerStats(this.pgConnection, this.statsBackend), setLayergroupIdHeader(this.templateMaps ,useTemplateHash), setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata), @@ -124,7 +123,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us setTurboCartoMetadataToLayergroup(this.layergroupMetadata), setAggregationMetadataToLayergroup(this.layergroupMetadata), setTilejsonMetadataToLayergroup(this.layergroupMetadata), - setSurrogateKeyHeader(this.surrogateKeysCache), sendResponse(), augmentError({ label, addContext }) ]; @@ -139,16 +137,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) { this.pgConnection, this.metadataBackend, this.userLimitsApi, - this.mapConfigAdapter + this.mapConfigAdapter, + this.layergroupAffectedTables ), - instantiateLayergroup(this.mapBackend, this.userLimitsApi) + instantiateLayergroup( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; } return [ checkCreateLayergroup(), prepareAdapterMapConfig(this.mapConfigAdapter), - createLayergroup (this.mapBackend, this.userLimitsApi) + createLayergroup ( + this.mapBackend, + this.userLimitsApi, + this.pgConnection, + this.layergroupAffectedTables + ) ]; }; @@ -201,7 +210,7 @@ function checkInstantiteLayergroup () { function checkCreateLayergroup () { return function checkCreateLayergroupMiddleware (req, res, next) { if (req.method === 'GET') { - const { config } = res.locals; + const { config } = req.query; if (!config) { return next(new Error('layergroup GET needs a "config" parameter')); @@ -219,33 +228,45 @@ function checkCreateLayergroup () { }; } -function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { +function getTemplate ( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache +) { return function getTemplateMiddleware (req, res, next) { const templateParams = req.body; - const { user } = res.locals; + const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id } = req.params; + const { auth_token } = req.query; - const mapconfigProvider = new NamedMapMapConfigProvider( + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); + + const mapConfigProvider = new NamedMapMapConfigProvider( templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, + affectedTablesCache, user, - req.params.template_id, + template_id, templateParams, - res.locals.auth_token, - res.locals + auth_token, + params ); - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => { req.profiler.done('named.getMapConfig'); if (err) { return next(err); } - res.locals.mapconfig = mapconfig; + res.locals.mapConfig = mapConfig; res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); @@ -255,7 +276,10 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi function prepareAdapterMapConfig (mapConfigAdapter) { return function prepareAdapterMapConfigMiddleware(req, res, next) { const requestMapConfig = req.body; - const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; + + const { user, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query); const context = { analysisConfiguration: { @@ -274,7 +298,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) { } }; - mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { + mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => { req.profiler.done('anonymous.getMapConfig'); if (err) { return next(err); @@ -288,51 +312,75 @@ function prepareAdapterMapConfig (mapConfigAdapter) { }; } -function createLayergroup (mapBackend, userLimitsApi) { +function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function createLayergroupMiddleware (req, res, next) { const requestMapConfig = req.body; - const { context, user } = res.locals; - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals); - res.locals.mapconfig = mapconfig; + const { context } = res.locals; + const { user, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + + const params = { + cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport + }; + + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapConfig = new MapConfig(requestMapConfig, datasource); + + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + params + ); + + res.locals.mapConfig = mapConfig; res.locals.analysesResults = context.analysesResults; - mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; + + mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; + res.locals.mapConfigProvider = mapConfigProvider; next(); }); }; } -function instantiateLayergroup (mapBackend, userLimitsApi) { +function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) { return function instantiateLayergroupMiddleware (req, res, next) { - const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams); + const { user, mapConfig, rendererParams } = res.locals; + const mapConfigProvider = new CreateLayergroupMapConfigProvider( + mapConfig, + user, + userLimitsApi, + pgConnection, + affectedTablesCache, + rendererParams + ); - mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => { req.profiler.done('createLayergroup'); if (err) { return next(err); } - res.locals.layergroup = layergroup; + res.body = layergroup; - const { mapconfigProvider } = res.locals; + const { mapConfigProvider } = res.locals; - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; next(); }); @@ -341,10 +389,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) { function incrementMapViewCount (metadataBackend) { return function incrementMapViewCountMiddleware(req, res, next) { - const { mapconfig, user } = res.locals; + const { mapConfig, user } = res.locals; // Error won't blow up, just be logged. - metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { + metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => { req.profiler.done('incMapviewCount'); if (err) { @@ -358,7 +406,7 @@ function incrementMapViewCount (metadataBackend) { function augmentLayergroupData () { return function augmentLayergroupDataMiddleware (req, res, next) { - const { layergroup } = res.locals; + const layergroup = res.body; // include in layergroup response the variables in serverMedata // those variables are useful to send to the client information @@ -369,80 +417,33 @@ function augmentLayergroupData () { }; } -function getAffectedTables (pgConnection, layergroupAffectedTables) { - return function getAffectedTablesMiddleware (req, res, next) { - const { dbname, layergroup, user, mapconfig } = res.locals; +function setLastUpdatedTimeToLayergroup () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { mapConfigProvider, analysesResults } = res.locals; + const layergroup = res.body; - pgConnection.getConnection(user, (err, connection) => { + mapConfigProvider.createAffectedTables((err, affectedTables) => { if (err) { return next(err); } - const sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); - } - }); + if (!affectedTables) { + return next(); + } - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { - req.profiler.done('getAffectedTablesFromQuery'); - if (err) { - return next(err); - } + var lastUpdateTime = affectedTables.getLastUpdatedAt(); - // feed affected tables cache so it can be reused from, for instance, layergroup controller - layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - res.locals.affectedTables = affectedTables; + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - next(); - }); + next(); }); }; } -function setCacheChannel () { - return function setCacheChannelMiddleware (req, res, next) { - const { affectedTables } = res.locals; - - if (req.method === 'GET') { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } - - next(); - }; -} - -function setLastModified () { - return function setLastModifiedMiddleware (req, res, next) { - if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); - } - - next(); - }; -} - -function setLastUpdatedTimeToLayergroup () { - return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { - const { affectedTables, layergroup, analysesResults } = res.locals; - - var lastUpdateTime = affectedTables.getLastUpdatedAt(); - - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - next(); - }; -} - function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; @@ -456,27 +457,17 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) { }, lastUpdateTime); } -function setCacheControl () { - return function setCacheControlMiddleware (req, res, next) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - } - - next(); - }; -} - function setLayerStats (pgConnection, statsBackend) { return function setLayerStatsMiddleware(req, res, next) { - const { user, mapconfig, layergroup } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; pgConnection.getConnection(user, (err, connection) => { if (err) { return next(err); } - statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + statsBackend.getStats(mapConfig, connection, function(err, layersStats) { if (err) { return next(err); } @@ -495,7 +486,8 @@ function setLayerStats (pgConnection, statsBackend) { function setLayergroupIdHeader (templateMaps, useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { - const { layergroup, user, template } = res.locals; + const { user, template } = res.locals; + const layergroup = res.body; if (useTemplateHash) { var templateHash = templateMaps.fingerPrint(template).substring(0, 8); @@ -510,9 +502,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) { function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; - layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj()); next(); }; @@ -520,7 +513,8 @@ function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) { function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, user, analysesResults = [] } = res.locals; + const { user, analysesResults = [] } = res.locals; + const layergroup = res.body; layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); @@ -530,9 +524,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) { function setTurboCartoMetadataToLayergroup (layergroupMetadata) { return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; + const layergroup = res.body; - layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -540,9 +535,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) { function setAggregationMetadataToLayergroup (layergroupMetadata) { return function setAggregationMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; + const { mapConfig, context } = res.locals; + const layergroup = res.body; - layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context); + layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context); next(); }; @@ -550,54 +546,24 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) { function setTilejsonMetadataToLayergroup (layergroupMetadata) { return function augmentLayergroupTilejsonMiddleware (req, res, next) { - const { layergroup, user, mapconfig } = res.locals; + const { user, mapConfig } = res.locals; + const layergroup = res.body; - layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig); + layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig); next(); }; } -function setSurrogateKeyHeader (surrogateKeysCache) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - req.profiler.done('res'); - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } - }; -} - function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; return function augmentErrorMiddleware (err, req, res, next) { req.profiler.done('error'); - const { mapconfig } = res.locals; + const { mapConfig } = res.locals; if (addContext) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; } err.label = label; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 815c68c8..93e56bbe 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -1,12 +1,14 @@ -const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); -const locals = require('../middleware/locals'); const cleanUpQueryParams = require('../middleware/clean-up-query-params'); -const layergroupToken = require('../middleware/layergroup-token'); const credentials = require('../middleware/credentials'); const dbConnSetup = require('../middleware/db-conn-setup'); const authorize = require('../middleware/authorize'); +const cacheControlHeader = require('../middleware/cache-control-header'); +const cacheChannelHeader = require('../middleware/cache-channel-header'); +const surrogateKeyHeader = require('../middleware/surrogate-key-header'); +const lastModifiedHeader = require('../middleware/last-modified-header'); +const sendResponse = require('../middleware/send-response'); const vectorError = require('../middleware/vector-error'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; @@ -23,17 +25,6 @@ function numMapper(n) { return +n; } -function getRequestParams(locals) { - const params = Object.assign({}, locals); - - delete params.template; - delete params.affectedTablesAndLastUpdate; - delete params.namedMapProvider; - delete params.allowedQueryParams; - - return params; -} - function NamedMapsController ( namedMapProviderCache, tileBackend, @@ -64,28 +55,25 @@ NamedMapsController.prototype.register = function(app) { app.get( `${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`, cors(), - cleanUpQueryParams(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), - layergroupToken(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES), + cleanUpQueryParams(), getNamedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, label: 'NAMED_MAP_TILE' }), - getAffectedTables(), getTile({ tileBackend: this.tileBackend, label: 'NAMED_MAP_TILE' }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse(), vectorError() ); @@ -93,19 +81,16 @@ NamedMapsController.prototype.register = function(app) { app.get( `${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`, cors(), - cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), - layergroupToken(), credentials(), authorize(this.authApi), dbConnSetup(this.pgConnection), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED), + cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']), getNamedMapProvider({ namedMapProviderCache: this.namedMapProviderCache, label: 'STATIC_VIZ_MAP', forcedFormat: 'png' }), - getAffectedTables(), getTemplate({ label: 'STATIC_VIZ_MAP' }), prepareLayerFilterFromPreviewLayers({ namedMapProviderCache: this.namedMapProviderCache, @@ -113,28 +98,35 @@ NamedMapsController.prototype.register = function(app) { }), getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }), getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }), - incrementMapViews({ metadataBackend: this.metadataBackend }), - setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), - setCacheChannelHeader(), - setLastModifiedHeader(), - setCacheControlHeader(), setContentTypeHeader(), + incrementMapViews({ metadataBackend: this.metadataBackend }), + cacheControlHeader(), + cacheChannelHeader(), + surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }), + lastModifiedHeader(), sendResponse() ); }; function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) { return function getNamedMapProviderMiddleware (req, res, next) { - const { user } = res.locals; - const { config, auth_token } = req.query; - const { template_id } = req.params; + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, layer: layerFromParams, z, x, y, format } = req.params; + const { layer: layerFromQuery } = req.query; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format + }; if (forcedFormat) { - res.locals.format = forcedFormat; - res.locals.layer = res.locals.layer || 'all'; + params.format = forcedFormat; + params.layer = params.layer || 'all'; } - const params = getRequestParams(res.locals); + const { config, auth_token } = req.query; namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => { if (err) { @@ -142,25 +134,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul return next(err); } - res.locals.namedMapProvider = namedMapProvider; - - next(); - }); - }; -} - -function getAffectedTables () { - return function getAffectedTables (req, res, next) { - const { namedMapProvider } = res.locals; - - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { - req.profiler.done('affectedTables'); - - if (err) { - return next(err); - } - - res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate; + res.locals.mapConfigProvider = namedMapProvider; next(); }); @@ -169,9 +143,9 @@ function getAffectedTables () { function getTemplate ({ label }) { return function getTemplateMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider } = res.locals; - namedMapProvider.getTemplate((err, template) => { + mapConfigProvider.getTemplate((err, template) => { if (err) { err.label = label; return next(err); @@ -186,8 +160,7 @@ function getTemplate ({ label }) { function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) { return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) { - const { user, template } = res.locals; - const { template_id } = req.params; + const { template } = res.locals; const { config, auth_token } = req.query; if (!template || !template.view || !template.view.preview_layers) { @@ -207,7 +180,15 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) return next(); } - const params = getRequestParams(res.locals); + const { user, token, cache_buster, api_key } = res.locals; + const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; + const { template_id, format } = req.params; + + const params = { + user, token, cache_buster, api_key, + dbuser, dbname, dbpassword, dbhost, dbport, + template_id, format + }; // overwrites 'all' default filter params.layer = layerVisibilityFilter.join(','); @@ -219,7 +200,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) return next(err); } - res.locals.namedMapProvider = provider; + res.locals.mapConfigProvider = provider; next(); }); @@ -228,10 +209,13 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) function getTile ({ tileBackend, label }) { return function getTileMiddleware (req, res, next) { - const { namedMapProvider } = res.locals; + const { mapConfigProvider } = res.locals; + const { layer, z, x, y, format } = req.params; + const params = { layer, z, x, y, format }; - tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => { + tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; @@ -251,9 +235,11 @@ function getTile ({ tileBackend, label }) { function getStaticImageOptions ({ tablesExtentApi }) { return function getStaticImageOptionsMiddleware(req, res, next) { - const { user, namedMapProvider, template } = res.locals; + const { user, mapConfigProvider, template } = res.locals; + const { zoom, lon, lat, bbox } = req.query; + const params = { zoom, lon, lat, bbox }; - const imageOpts = getImageOptions(res.locals, template); + const imageOpts = getImageOptions(params, template); if (imageOpts) { res.locals.imageOpts = imageOpts; @@ -262,18 +248,18 @@ function getStaticImageOptions ({ tablesExtentApi }) { res.locals.imageOpts = DEFAULT_ZOOM_CENTER; - namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => { + mapConfigProvider.createAffectedTables((err, affectedTables) => { if (err) { return next(); } - var affectedTables = affectedTablesAndLastUpdate.tables || []; + var tables = affectedTables.tables || []; - if (affectedTables.length === 0) { + if (tables.length === 0) { return next(); } - tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => { + tablesExtentApi.getBounds(user, tables, (err, bounds) => { if (err) { return next(); } @@ -353,7 +339,7 @@ function getImageOptionsFromBoundingBox (bbox = '') { function getImage({ previewBackend, label }) { return function getImageMiddleware (req, res, next) { - const { imageOpts, namedMapProvider } = res.locals; + const { imageOpts, mapConfigProvider } = res.locals; const { zoom, center, bounds } = imageOpts; let { width, height } = req.params; @@ -364,7 +350,7 @@ function getImage({ previewBackend, label }) { const format = req.params.format === 'jpg' ? 'jpeg' : 'png'; if (zoom !== undefined && center) { - return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center, + return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center, (err, image, headers, stats) => { req.profiler.add(stats); @@ -383,8 +369,9 @@ function getImage({ previewBackend, label }) { }); } - previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => { + previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => { req.profiler.add(stats); + req.profiler.done('render-' + format); if (err) { err.label = label; @@ -402,15 +389,23 @@ function getImage({ previewBackend, label }) { }; } +function setContentTypeHeader () { + return function setContentTypeHeaderMiddleware(req, res, next) { + res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); + + next(); + }; +} + function incrementMapViewsError (ctx) { return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`; } function incrementMapViews ({ metadataBackend }) { return function incrementMapViewsMiddleware(req, res, next) { - const { user, namedMapProvider } = res.locals; + const { user, mapConfigProvider } = res.locals; - namedMapProvider.getMapConfig((err, mapConfig) => { + mapConfigProvider.getMapConfig((err, mapConfig) => { if (err) { global.logger.log(incrementMapViewsError({ user, err })); return next(); @@ -458,84 +453,3 @@ function templateBounds(view) { } return false; } - -function setSurrogateKeyHeader ({ surrogateKeysCache }) { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals; - - surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName())); - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - if (affectedTablesAndLastUpdate.tables.length > 0) { - surrogateKeysCache.tag(res, affectedTablesAndLastUpdate); - } - } - - next(); - }; -} - -function setCacheChannelHeader () { - return function setCacheChannelHeaderMiddleware (req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel()); - } - - next(); - }; -} - -function setLastModifiedHeader () { - return function setLastModifiedHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - var lastModifiedDate; - if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) { - lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); - } - - res.set('Last-Modified', lastModifiedDate.toUTCString()); - } - - next(); - }; - } - -function setCacheControlHeader () { - return function setCacheControlHeaderMiddleware(req, res, next) { - const { affectedTablesAndLastUpdate } = res.locals; - - res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - - if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - } - - next(); - }; - } - -function setContentTypeHeader () { - return function setContentTypeHeaderMiddleware(req, res, next) { - - res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png'); - - next(); - }; -} - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - const { format } = res.locals; - - req.profiler.done('render-' + format); - - res.status(200); - res.send(res.body); - }; -} diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index db503fb7..df3ac766 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -1,10 +1,10 @@ const { templateName } = require('../backends/template_maps'); const cors = require('../middleware/cors'); const user = require('../middleware/user'); -const locals = require('../middleware/locals'); const credentials = require('../middleware/credentials'); const rateLimit = require('../middleware/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const sendResponse = require('../middleware/send-response'); /** * @param {AuthApi} authApi @@ -26,12 +26,11 @@ NamedMapsAdminController.prototype.register = function (app) { app.post( `${templateBasePath}/`, cors(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), credentials(), checkContentType({ action: 'POST', label: 'POST TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE), createTemplate({ templateMaps: this.templateMaps }), sendResponse() ); @@ -39,12 +38,11 @@ NamedMapsAdminController.prototype.register = function (app) { app.put( `${templateBasePath}/:template_id`, cors(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), credentials(), checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }), authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE), updateTemplate({ templateMaps: this.templateMaps }), sendResponse() ); @@ -52,11 +50,10 @@ NamedMapsAdminController.prototype.register = function (app) { app.get( `${templateBasePath}/:template_id`, cors(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET), retrieveTemplate({ templateMaps: this.templateMaps }), sendResponse() ); @@ -64,11 +61,10 @@ NamedMapsAdminController.prototype.register = function (app) { app.delete( `${templateBasePath}/:template_id`, cors(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE), destroyTemplate({ templateMaps: this.templateMaps }), sendResponse() ); @@ -76,11 +72,10 @@ NamedMapsAdminController.prototype.register = function (app) { app.get( `${templateBasePath}/`, cors(), - locals(), user(), - rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), credentials(), authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }), + rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST), listTemplates({ templateMaps: this.templateMaps }), sendResponse() ); @@ -107,7 +102,7 @@ function authorizedByAPIKey ({ authApi, action, label }) { return function authorizedByAPIKeyMiddleware (req, res, next) { const { user } = res.locals; - authApi.authorizedByAPIKey(user, res, (err, authenticated) => { + authApi.authorizedByAPIKey(user, res, (err, authenticated, apikey) => { if (err) { return next(err); } @@ -119,6 +114,15 @@ function authorizedByAPIKey ({ authApi, action, label }) { return next(error); } + if (apikey.type !== 'master') { + const error = new Error('Forbidden'); + error.type = 'auth'; + error.subtype = 'api-key-does-not-grant-access'; + error.http_status = 403; + + return next(error); + } + next(); }); }; @@ -224,12 +228,3 @@ function listTemplates ({ templateMaps }) { }); }; } - -function sendResponse () { - return function sendResponseMiddleware (req, res) { - res.status(res.statusCode || 200); - - const method = req.query.callback ? 'jsonp' : 'json'; - res[method](res.body); - }; -} diff --git a/lib/cartodb/middleware/cache-channel-header.js b/lib/cartodb/middleware/cache-channel-header.js new file mode 100644 index 00000000..d6bf394b --- /dev/null +++ b/lib/cartodb/middleware/cache-channel-header.js @@ -0,0 +1,24 @@ +module.exports = function setCacheChannelHeader () { + return function setCacheChannelHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider } = res.locals; + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Cache Channel Header:', err); + return next(); + } + + if (!affectedTables) { + return next(); + } + + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/cache-control-header.js b/lib/cartodb/middleware/cache-control-header.js new file mode 100644 index 00000000..574ae3b5 --- /dev/null +++ b/lib/cartodb/middleware/cache-control-header.js @@ -0,0 +1,19 @@ +const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365; + +module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) { + return function setCacheControlHeaderMiddleware (req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const directives = [ 'public', `max-age=${ttl}` ]; + + if (revalidate) { + directives.push('must-revalidate'); + } + + res.set('Cache-Control', directives.join(',')); + + next(); + }; +}; diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/middleware/error-middleware.js index 85f38936..c02974b4 100644 --- a/lib/cartodb/middleware/error-middleware.js +++ b/lib/cartodb/middleware/error-middleware.js @@ -15,10 +15,6 @@ module.exports = function errorMiddleware (/* options */) { var statusCode = findStatusCode(err); - if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') { - statusCode = 204; - } - setErrorHeader(allErrors, statusCode, res); debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); @@ -186,15 +182,15 @@ function setErrorHeader(errors, statusCode, res) { subtype: error.subtype }; }); - + res.set('X-Tiler-Errors', stringifyForLogs(errorsLog)); } /** - * Remove problematic nested characters + * Remove problematic nested characters * from object for logs RegEx - * - * @param {Object} object + * + * @param {Object} object */ function stringifyForLogs(object) { Object.keys(object).map(key => { diff --git a/lib/cartodb/middleware/last-modified-header.js b/lib/cartodb/middleware/last-modified-header.js new file mode 100644 index 00000000..18c0d961 --- /dev/null +++ b/lib/cartodb/middleware/last-modified-header.js @@ -0,0 +1,45 @@ +module.exports = function setLastModifiedHeader ({ now = false } = {}) { + return function setLastModifiedHeaderMiddleware(req, res, next) { + if (req.method !== 'GET') { + return next(); + } + + const { mapConfigProvider, cache_buster } = res.locals; + + if (cache_buster) { + const cacheBuster = parseInt(cache_buster, 10); + const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + return next(); + } + + // REVIEW: to keep 100% compatibility with maps controller + if (now) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Last Modified Header:', err); + return next(); + } + + if (!affectedTables) { + res.set('Last-Modified', new Date().toUTCString()); + + return next(); + } + + const lastUpdatedAt = affectedTables.getLastUpdatedAt(); + const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date(); + + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + next(); + }); + }; +}; diff --git a/lib/cartodb/middleware/layergroup-token.js b/lib/cartodb/middleware/layergroup-token.js index 797b1b3d..1a32e413 100644 --- a/lib/cartodb/middleware/layergroup-token.js +++ b/lib/cartodb/middleware/layergroup-token.js @@ -5,13 +5,8 @@ const authErrorMessageTemplate = function (signer, user) { module.exports = function layergroupToken () { return function layergroupTokenMiddleware (req, res, next) { - if (!res.locals.token) { - return next(); - } - const user = res.locals.user; - - const layergroupToken = LayergroupToken.parse(res.locals.token); + const layergroupToken = LayergroupToken.parse(req.params.token); res.locals.token = layergroupToken.token; res.locals.cache_buster = layergroupToken.cacheBuster; diff --git a/lib/cartodb/middleware/locals.js b/lib/cartodb/middleware/locals.js deleted file mode 100644 index 2629767e..00000000 --- a/lib/cartodb/middleware/locals.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = function locals () { - return function localsMiddleware (req, res, next) { - res.locals = Object.assign({}, req.query, req.params); - - next(); - }; -}; diff --git a/lib/cartodb/middleware/send-response.js b/lib/cartodb/middleware/send-response.js new file mode 100644 index 00000000..469cf0a7 --- /dev/null +++ b/lib/cartodb/middleware/send-response.js @@ -0,0 +1,17 @@ +module.exports = function sendResponse () { + return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); + + res.status(res.statusCode || 200); + + if (Buffer.isBuffer(res.body)) { + return res.send(res.body); + } + + if (req.query.callback) { + return res.jsonp(res.body); + } + + res.json(res.body); + }; +}; diff --git a/lib/cartodb/middleware/surrogate-key-header.js b/lib/cartodb/middleware/surrogate-key-header.js new file mode 100644 index 00000000..51cec1c1 --- /dev/null +++ b/lib/cartodb/middleware/surrogate-key-header.js @@ -0,0 +1,31 @@ +const NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); +const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); + +module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { user, mapConfigProvider } = res.locals; + + if (mapConfigProvider instanceof NamedMapMapConfigProvider) { + surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName())); + } + + if (req.method !== 'GET') { + return next(); + } + + mapConfigProvider.getAffectedTables((err, affectedTables) => { + if (err) { + global.logger.warn('ERROR generating Surrogate Key Header:', err); + return next(); + } + + if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) { + return next(); + } + + surrogateKeysCache.tag(res, affectedTables); + + next(); + }); + }; +}; diff --git a/lib/cartodb/models/aggregation/aggregation-mapconfig.js b/lib/cartodb/models/aggregation/aggregation-mapconfig.js index 50e4dcfe..548f95d3 100644 --- a/lib/cartodb/models/aggregation/aggregation-mapconfig.js +++ b/lib/cartodb/models/aggregation/aggregation-mapconfig.js @@ -4,7 +4,8 @@ const aggregationValidator = require('./aggregation-validator'); const { createPositiveNumberValidator, createIncludesValueValidator, - createAggregationColumnsValidator + createAggregationColumnsValidator, + createAggregationFiltersValidator } = aggregationValidator; const SubstitutionTokens = require('../../utils/substitution-tokens'); @@ -43,6 +44,18 @@ module.exports = class AggregationMapConfig extends MapConfig { ]; } + static get FILTER_PARAMETERS () { + return [ + // TODO: valid combinations of parameters: + // * Except for less/greater params, only one parameter allowed per filter. + // * Any less parameter can be combined with one of the greater paramters. (to define a range) + 'less_than', 'less_than_or_equal_to', + 'greater_than', 'greater_than_or_equal_to', + 'equal', 'not_equal', + 'between', 'in', 'not_in' + ]; + } + static supportsGeometryType(geometryType) { return AggregationMapConfig.SUPPORTED_GEOMETRY_TYPES.includes(geometryType); } @@ -58,11 +71,15 @@ module.exports = class AggregationMapConfig extends MapConfig { const positiveNumberValidator = createPositiveNumberValidator(this); const includesValidPlacementsValidator = createIncludesValueValidator(this, AggregationMapConfig.PLACEMENTS); const aggregationColumnsValidator = createAggregationColumnsValidator(this, AggregationMapConfig.AGGREGATIONS); + const aggregationFiltersValidator = createAggregationFiltersValidator( + this, AggregationMapConfig.FILTER_PARAMETERS + ); validate('resolution', positiveNumberValidator); validate('placement', includesValidPlacementsValidator); validate('threshold', positiveNumberValidator); validate('columns', aggregationColumnsValidator); + validate('filters', aggregationFiltersValidator); this.user = user; this.pgConnection = connection; @@ -77,7 +94,8 @@ module.exports = class AggregationMapConfig extends MapConfig { threshold = AggregationMapConfig.THRESHOLD, placement, columns = {}, - dimensions = {} + dimensions = {}, + filters = {} } = this.getAggregation(index); return aggregationQuery({ @@ -87,6 +105,7 @@ module.exports = class AggregationMapConfig extends MapConfig { placement, columns, dimensions, + filters, isDefaultAggregation: this._isDefaultLayerAggregation(index) }); } @@ -152,21 +171,19 @@ module.exports = class AggregationMapConfig extends MapConfig { _getLayerAggregationRequiredColumns (index) { const { columns, dimensions } = this.getAggregation(index); + let finalColumns = ['cartodb_id', '_cdb_feature_count']; + let aggregatedColumns = []; if (columns) { - aggregatedColumns = Object.keys(columns) - .map(key => columns[key].aggregated_column) - .filter(aggregatedColumn => typeof aggregatedColumn === 'string'); + aggregatedColumns = Object.keys(columns); } let dimensionsColumns = []; if (dimensions) { - dimensionsColumns = Object.keys(dimensions) - .map(key => dimensions[key]) - .filter(dimension => typeof dimension === 'string'); + dimensionsColumns = Object.keys(dimensions); } - return removeDuplicates(aggregatedColumns.concat(dimensionsColumns)); + return removeDuplicates(finalColumns.concat(aggregatedColumns).concat(dimensionsColumns)); } doesLayerReachThreshold(index, featureCount) { @@ -220,7 +237,8 @@ module.exports = class AggregationMapConfig extends MapConfig { _isDefaultAggregation (aggregation) { return aggregation.placement === undefined && aggregation.columns === undefined && - this._isEmptyParameter(aggregation.dimensions); + this._isEmptyParameter(aggregation.dimensions) && + this._isEmptyParameter(aggregation.filters); } _isEmptyParameter(parameter) { diff --git a/lib/cartodb/models/aggregation/aggregation-query.js b/lib/cartodb/models/aggregation/aggregation-query.js index 16e897ab..cfb9855a 100644 --- a/lib/cartodb/models/aggregation/aggregation-query.js +++ b/lib/cartodb/models/aggregation/aggregation-query.js @@ -42,7 +42,8 @@ const queryForOptions = (options) => templateForOptions(options)({ sourceQuery: options.query, res: 256/options.resolution, columns: options.columns, - dimensions: options.dimensions + dimensions: options.dimensions, + filters: options.filters }); module.exports = queryForOptions; @@ -93,20 +94,23 @@ const aggregateColumnNames = (ctx, table) => { return sep(Object.keys(columns)); }; +const aggregateExpression = (column_name, column_parameters) => { + const aggregate_function = column_parameters.aggregate_function || 'count'; + const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; + if (!aggregate_definition) { + throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); + } + return aggregate_definition.sql(column_name, column_parameters); +}; + const aggregateColumnDefs = ctx => { let columns = aggregateColumns(ctx); return sep(Object.keys(columns).map(column_name => { - const aggregate_function = columns[column_name].aggregate_function || 'count'; - const aggregate_definition = SUPPORTED_AGGREGATE_FUNCTIONS[aggregate_function]; - if (!aggregate_definition) { - throw new Error("Invalid Aggregate function: '" + aggregate_function + "'"); - } - const aggregate_expression = aggregate_definition.sql(column_name, columns[column_name]); + const aggregate_expression = aggregateExpression(column_name, columns[column_name]); return `${aggregate_expression} AS ${column_name}`; })); }; - const aggregateDimensions = ctx => ctx.dimensions || {}; const dimensionNames = (ctx, table) => { @@ -127,6 +131,111 @@ const dimensionDefs = ctx => { })); }; +const aggregateFilters = ctx => ctx.filters || {}; + +const filterConditionSQL = (expr, filter) => { + // TODO: validate filter parameters (e.g. cannot have both greater_than and greater_than or equal to) + + if (filter) { + if (!Array.isArray(filter)) { + filter = [filter]; + } + if (filter.length > 0) { + return filter.map(f => filterSingleConditionSQL(expr, f)).join(' OR '); + } + } +}; + +const filterSingleConditionSQL = (expr, filter) => { + let cond; + Object.keys(FILTERS).some(f => { + cond = FILTERS[f](expr, filter); + return cond; + }); + return cond; +}; + +const sqlQ = (value) => { + if (isFinite(value)) { + return String(value); + } + return `'${value}'`; // TODO: escape single quotes! (by doubling them) +}; + +/* jshint eqeqeq: false */ +/* x != null is used to check for both null and undefined; triple !== wouldn't do the trick */ + +const FILTERS = { + between: (expr, filter) => { + const lo = filter.greater_than_or_equal_to, hi = filter.less_than_or_equal_to; + if (lo != null && hi != null) { + return `(${expr} BETWEEN ${sqlQ(lo)} AND ${sqlQ(hi)})`; + } + }, + in: (expr, filter) => { + if (filter.in != null) { + return `(${expr} IN (${filter.in.map(v => sqlQ(v)).join(',')}))`; + } + }, + notin: (expr, filter) => { + if (filter.not_in != null) { + return `(${expr} NOT IN (${filter.not_in.map(v => sqlQ(v)).join(',')}))`; + } + }, + equal: (expr, filter) => { + if (filter.equal != null) { + return `(${expr} = ${sqlQ(filter.equal)})`; + } + }, + not_equal: (expr, filter) => { + if (filter.not_equal != null) { + return `(${expr} <> ${sqlQ(filter.not_equal)})`; + } + }, + range: (expr, filter) => { + let conds = []; + if (filter.greater_than_or_equal_to != null) { + conds.push(`(${expr} >= ${sqlQ(filter.greater_than_or_equal_to)})`); + } + if (filter.greater_than != null) { + conds.push(`(${expr} > ${sqlQ(filter.greater_than)})`); + } + if (filter.less_than_or_equal_to != null) { + conds.push(`(${expr} <= ${sqlQ(filter.less_than_or_equal_to)})`); + } + if (filter.less_than != null) { + conds.push(`(${expr} < ${sqlQ(filter.less_than)})`); + } + if (conds.length > 0) { + return conds.join(' AND '); + } + } +}; + +const filterConditions = ctx => { + let columns = aggregateColumns(ctx); + let dimensions = aggregateDimensions(ctx); + let filters = aggregateFilters(ctx); + return Object.keys(filters).map(filtered_column => { + let filtered_expr; + if (columns[filtered_column]) { + filtered_expr = aggregateExpression(filtered_column, columns[filtered_column]); + } + else if (dimensions[filtered_column]) { + filtered_expr = dimensions[filtered_column]; + } + if (!filtered_expr) { + throw new Error("Invalid filtered column: '" + filtered_column + "'"); + } + return filterConditionSQL(filtered_expr, filters[filtered_column]); + }).join(' AND '); +}; + +const havingClause = ctx => { + let cond = filterConditions(ctx); + return cond ? `HAVING ${cond}` : ''; +}; + // SQL expression to compute the aggregation resolution (grid cell size). // This is equivalent to `${256/ctx.res}*CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!))` // This is defined by the ctx.res parameter, which is the number of grid cells per tile linear dimension @@ -181,7 +290,7 @@ const aggregationQueryTemplates = { !bbox! AS bbox ) SELECT - row_number() over() AS cartodb_id, + MIN(_cdb_query.cartodb_id) AS cartodb_id, ST_SetSRID( ST_MakePoint( AVG(ST_X(_cdb_query.the_geom_webmercator)), @@ -196,6 +305,7 @@ const aggregationQueryTemplates = { Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} `, 'point-grid': ctx => ` @@ -207,6 +317,7 @@ const aggregationQueryTemplates = { ), _cdb_clusters AS ( SELECT + MIN(_cdb_query.cartodb_id) AS cartodb_id, Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gx, Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res)::int AS _cdb_gy ${dimensionDefs(ctx)} @@ -214,9 +325,10 @@ const aggregationQueryTemplates = { FROM (${ctx.sourceQuery}) _cdb_query, _cdb_params WHERE the_geom_webmercator && _cdb_params.bbox GROUP BY _cdb_gx, _cdb_gy ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT - row_number() over() AS cartodb_id, + _cdb_clusters.cartodb_id AS cartodb_id, ST_SetSRID(ST_MakePoint((_cdb_gx+0.5)*res, (_cdb_gy+0.5)*res), 3857) AS the_geom_webmercator ${dimensionNames(ctx)} ${aggregateColumnNames(ctx)} @@ -241,11 +353,12 @@ const aggregationQueryTemplates = { Floor(ST_X(_cdb_query.the_geom_webmercator)/_cdb_params.res), Floor(ST_Y(_cdb_query.the_geom_webmercator)/_cdb_params.res) ${dimensionNames(ctx)} + ${havingClause(ctx)} ) SELECT _cdb_clusters.cartodb_id, the_geom, the_geom_webmercator - ${dimensionNames(ctx, '_cdb_query')} + ${dimensionNames(ctx, '_cdb_clusters')} ${aggregateColumnNames(ctx, '_cdb_clusters')} FROM _cdb_clusters INNER JOIN (${ctx.sourceQuery}) _cdb_query diff --git a/lib/cartodb/models/aggregation/aggregation-validator.js b/lib/cartodb/models/aggregation/aggregation-validator.js index d0dc24c2..ce038f05 100644 --- a/lib/cartodb/models/aggregation/aggregation-validator.js +++ b/lib/cartodb/models/aggregation/aggregation-validator.js @@ -42,6 +42,38 @@ module.exports.createAggregationColumnsValidator = function (mapconfig, validAgg }; }; +module.exports.createAggregationFiltersValidator = function (mapconfig, validParameters) { + return function validateAggregationFilters (value, key, index) { + const dims = mapconfig.getAggregation(index).dimensions || {}; + const cols = mapconfig.getAggregation(index).columns || {}; + const validKeys = Object.keys(dims).concat(Object.keys(cols)); + Object.keys(value).forEach((filteredName) => { + // filteredName must be the name of either an aggregated column or a dimension in the same layer + if (!validKeys.includes(filteredName)) { + const message = `Invalid filtered column: ${filteredName}`; + throw createLayerError(message, mapconfig, index); + } + // The filter parameters must be valid + let filters = value[filteredName]; + // a single filter or an array of filters (to be OR-combined) are accepted + if (!Array.isArray(filters)) { + filters = [filters]; + } + filters.forEach(params => { + Object.keys(params).forEach(paramName => { + if (!validParameters.includes(paramName)) { + const message = `Invalid filter parameter name: ${paramName}`; + throw createLayerError(message, mapconfig, index); + } + }); + // TODO: check parameter value (params[paramName]) to be of the correct type + }); + // TODO: if multiple parameters within params check the combination is valid, + // i.e. one of the *less* parameters and one of the *greater* parameters. + }); + }; +}; + function createAggregationColumnNamesValidator(mapconfig) { return function validateAggregationColumnNames (value, key, index) { Object.keys(value).forEach((columnName) => { diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js index b9487095..2bd30152 100644 --- a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -11,6 +11,8 @@ var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasou var MapConfig = require('windshaft').model.MapConfig; +const dbParamsFromReqParams = require('../../../utils/database-params'); + function TurboCartoAdapter() { } @@ -158,23 +160,3 @@ TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) { function shouldParseLayerCartocss(layer) { return layer && layer.options && layer.options.cartocss && layer.options.sql; } - -function dbParamsFromReqParams(params) { - var dbParams = {}; - if ( params.dbuser ) { - dbParams.user = params.dbuser; - } - if ( params.dbpassword ) { - dbParams.pass = params.dbpassword; - } - if ( params.dbhost ) { - dbParams.host = params.dbhost; - } - if ( params.dbport ) { - dbParams.port = params.dbport; - } - if ( params.dbname ) { - dbParams.dbname = params.dbname; - } - return dbParams; -} diff --git a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 340073b5..1f19d395 100644 --- a/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -2,6 +2,7 @@ var assert = require('assert'); var step = require('step'); var MapStoreMapConfigProvider = require('./map-store-provider'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapConfig} mapConfig @@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider'); * @constructor * @type {CreateLayergroupMapConfigProvider} */ -function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) { + +function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapConfig = mapConfig; this.user = user; this.userLimitsApi = userLimitsApi; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.params = params; this.cacheBuster = params.cache_buster || 0; } @@ -23,7 +27,13 @@ module.exports = CreateLayergroupMapConfigProvider; CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig && this.params && this.context) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -31,6 +41,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) { function handleRenderLimits(err, renderLimits) { assert.ifError(err); context.limits = renderLimits; + self.context = context; return null; }, function finish(err) { @@ -46,3 +57,65 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter; CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey; + +CreateLayergroupMapConfigProvider.prototype.createAffectedTables = function (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + const queries = []; + + this.mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(null, affectedTables); + }); + }); + }); +}; + +CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + return this.createAffectedTables(callback); + }); +}; diff --git a/lib/cartodb/models/mapconfig/provider/map-store-provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js index 177322d4..d9f9da83 100644 --- a/lib/cartodb/models/mapconfig/provider/map-store-provider.js +++ b/lib/cartodb/models/mapconfig/provider/map-store-provider.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var dot = require('dot'); var step = require('step'); +const QueryTables = require('cartodb-query-tables'); /** * @param {MapStore} mapStore @@ -11,20 +12,30 @@ var step = require('step'); * @constructor * @type {MapStoreMapConfigProvider} */ -function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) { +function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) { this.mapStore = mapStore; this.user = user; this.userLimitsApi = userLimitsApi; - this.params = params; + this.pgConnection = pgConnection; + this.affectedTablesCache = affectedTablesCache; this.token = params.token; this.cacheBuster = params.cache_buster || 0; + this.mapConfig = null; + this.params = params; + this.context = null; } module.exports = MapStoreMapConfigProvider; MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; + + if (this.mapConfig !== null) { + return callback(null, this.mapConfig, this.params, this.context); + } + var context = {}; + step( function prepareContextLimits() { self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this); @@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) { self.mapStore.load(self.token, this); }, function finish(err, mapConfig) { + self.mapConfig = mapConfig; + self.context = context; return callback(err, mapConfig, self.params, context); } ); @@ -74,4 +87,66 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) { scale_factor: 1 }); return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues); -}; \ No newline at end of file +}; + +MapStoreMapConfigProvider.prototype.createAffectedTables = function(callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.user, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); +}; + +MapStoreMapConfigProvider.prototype.getAffectedTables = function (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + return this.createAffectedTables(callback); + }); +}; diff --git a/lib/cartodb/models/mapconfig/provider/named-map-provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js index 9b085dbc..065d20f4 100644 --- a/lib/cartodb/models/mapconfig/provider/named-map-provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables'); * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, - owner, templateId, config, authToken, params) { +function NamedMapMapConfigProvider( + templateMaps, + pgConnection, + metadataBackend, + userLimitsApi, + mapConfigAdapter, + affectedTablesCache, + owner, + templateId, + config, + authToken, + params +) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; @@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, // use template after call to mapConfig this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTablesCache = affectedTablesCache; // providing this.err = null; @@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() { NamedMapMapConfigProvider.prototype.reset = function() { this.template = null; - this.affectedTablesAndLastUpdate = null; + this.affectedTables = null; this.err = null; this.mapConfig = null; @@ -251,39 +262,64 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() { return this.templateName; }; -NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) { - var self = this; - - if (this.affectedTablesAndLastUpdate !== null) { - return callback(null, this.affectedTablesAndLastUpdate); - } - - step( - function getMapConfig() { - self.getMapConfig(this); - }, - function getSql(err, mapConfig) { - assert.ifError(err); - return mapConfig.getLayers().map(function(layer) { - return layer.options.sql; - }).join(';'); - }, - function getAffectedTables(err, sql) { - assert.ifError(err); - step( - function getConnection() { - self.pgConnection.getConnection(self.owner, this); - }, - function getAffectedTables(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql, this); - }, - this - ); - }, - function finish(err, result) { - self.affectedTablesAndLastUpdate = result; - return callback(err, result); +NamedMapMapConfigProvider.prototype.createAffectedTables = function(callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); } - ); + + const { dbname } = this.rendererParams; + const token = mapConfig.id(); + + const queries = []; + + mapConfig.getLayers().forEach(layer => { + queries.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(table => { + queries.push(`SELECT * FROM ${table} LIMIT 0`); + }); + } + }); + + const sql = queries.length ? queries.join(';') : null; + + if (!sql) { + return callback(); + } + + this.pgConnection.getConnection(this.owner, (err, connection) => { + if (err) { + return callback(err); + } + + QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => { + if (err) { + return callback(err); + } + + this.affectedTablesCache.set(dbname, token, affectedTables); + + callback(err, affectedTables); + }); + }); + }); +}; + +NamedMapMapConfigProvider.prototype.getAffectedTables = function (callback) { + this.getMapConfig((err, mapConfig) => { + if (err) { + return callback(err); + } + + const { dbname } = this.params; + const token = mapConfig.id(); + + if (this.affectedTablesCache.hasAffectedTables(dbname, token)) { + const affectedTables = this.affectedTablesCache.get(dbname, token); + return callback(null, affectedTables); + } + + return this.createAffectedTables(callback); + }); }; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 7f7e2712..8535783d 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -200,7 +200,8 @@ module.exports = function(serverOptions) { pgConnection, metadataBackend, userLimitsApi, - mapConfigAdapter + mapConfigAdapter, + layergroupAffectedTablesCache ); ['update', 'delete'].forEach(function(eventType) { diff --git a/lib/cartodb/utils/database-params.js b/lib/cartodb/utils/database-params.js new file mode 100644 index 00000000..a2176eaf --- /dev/null +++ b/lib/cartodb/utils/database-params.js @@ -0,0 +1,25 @@ +module.exports = function getDatabaseConnectionParams (params) { + const dbParams = {}; + + if (params.dbuser) { + dbParams.user = params.dbuser; + } + + if (params.dbpassword) { + dbParams.pass = params.dbpassword; + } + + if (params.dbhost) { + dbParams.host = params.dbhost; + } + + if (params.dbport) { + dbParams.port = params.dbport; + } + + if (params.dbname) { + dbParams.dbname = params.dbname; + } + + return dbParams; +}; diff --git a/test/acceptance/aggregation.js b/test/acceptance/aggregation.js index d8870c8a..6658eb81 100644 --- a/test/acceptance/aggregation.js +++ b/test/acceptance/aggregation.js @@ -357,6 +357,103 @@ describe('aggregation', function () { }); }); + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it('should provide all the requested columns in non-default aggregation ', + function (done) { + const response = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement, + columns: { + 'first_column': { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + dimensions: { + second_column: 'sqrt_value' + }, + threshold: 1 + }, + cartocss: '#layer { marker-width: [first_column]; line-width: [second_column]; }', + cartocss_version: '2.3.0' + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + this.testClient.getLayergroup({ response }, (err, body) => { + if (err) { + return done(err); + } + + + assert.equal(typeof body.metadata, 'object'); + assert.ok(Array.isArray(body.metadata.layers)); + + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.mvt)); + body.metadata.layers.forEach(layer => assert.ok(layer.meta.aggregation.png)); + done(); + }); + }); + + it('should provide only the requested columns in non-default aggregation ', + function (done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement, + columns: { + 'first_column': { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + dimensions: { + second_column: 'sqrt_value' + }, + threshold: 1 + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(0, 0, 0, { format: 'mvt' }, function (err, res, mvt) { + if (err) { + return done(err); + } + + const geojsonTile = JSON.parse(mvt.toGeoJSONSync(0)); + let columns = new Set(); + geojsonTile.features.forEach(f => { + Object.keys(f.properties).forEach(p => columns.add(p)); + }); + columns = Array.from(columns); + const expected_columns = [ + '_cdb_feature_count', 'cartodb_id', 'first_column', 'second_column' + ]; + assert.deepEqual(columns.sort(), expected_columns.sort()); + + done(); + }); + }); + }); + it('should skip aggregation to create a layergroup with aggregation defined already', function (done) { const mapConfig = createVectorMapConfig([ { @@ -689,6 +786,45 @@ describe('aggregation', function () { }); }); + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`dimensions with alias should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + dimensions: { + value2: "value" + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach( + feature => assert.equal(typeof feature.properties.value2, 'number') + ); + + done(); + }); + }); + }); + + it(`dimensions should trigger non-default aggregation`, function(done) { this.mapConfig = createVectorMapConfig([ { @@ -1475,6 +1611,569 @@ describe('aggregation', function () { done(); }); }); + + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than_or_equal_to: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value >= 0); + }); + + done(); + }); + }); + }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`multiple ORed filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: [ + { greater_than: 0 }, + { less_than: -2 } + ] + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > 0 || row.properties.value < -2); + }); + + done(); + }); + }); + }); + + ['centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`multiple ANDed filters should work for ${placement} placement`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2, + aggregation: { + placement: placement , + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + }, + value2: { + aggregate_function: 'sum', + aggregated_column: 'sqrt_value' + } + }, + filters: { + value: { greater_than: 0 }, + value2: { less_than: 9 } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > 0 && row.properties.value2 < 9); + }); + + done(); + }); + }); + }); + + it(`supports IN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { in: [1, 3] } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value === 1 || row.properties.value === 3); + }); + + done(); + }); + }); + + it(`supports NOT IN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { not_in: [1, 3] } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value !== 1 && row.properties.value !== 3); + }); + + done(); + }); + }); + + it(`supports EQUAL filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: [{ equal: 1}, { equal: 3}] + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value === 1 || row.properties.value === 3); + }); + + done(); + }); + }); + + it(`supports NOT EQUAL filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { not_equal: 1 } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value !== 1); + }); + + done(); + }); + }); + + it(`supports BETWEEN filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than_or_equal_to: -1, + less_than_or_equal_to: 2 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value >= -1 || row.properties.value <= 2); + }); + + done(); + }); + }); + + it(`supports RANGE filters`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + greater_than: -1, + less_than_or_equal_to: 2 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + const options = { + format: 'mvt' + }; + this.testClient.getTile(0, 0, 0, options, (err, res, tile) => { + if (err) { + return done(err); + } + + const tileJSON = tile.toJSON(); + + tileJSON[0].features.forEach(row => { + assert.ok(row.properties.value > -1 || row.properties.value <= 2); + }); + + done(); + }); + }); + + it(`invalid filters cause errors`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + not_a_valid_parameter: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ 'Invalid filter parameter name: not_a_valid_parameter'], + errors_with_context:[{ + type: 'layer', + message: 'Invalid filter parameter name: not_a_valid_parameter', + layer: { + id: "layer0", + index: 0, + type: "mapnik", + } + }] + }); + + done(); + }); + }); + + it(`filters on invalid columns cause errors`, function(done) { + this.mapConfig = createVectorMapConfig([ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1, + aggregation: { + threshold: 1, + columns: { + value_sum: { + aggregate_function: 'sum', + aggregated_column: 'value' + } + }, + filters: { + value: { + not_a_valid_parameter: 0 + } + } + } + } + } + ]); + + this.testClient = new TestClient(this.mapConfig); + + const options = { + response: { + status: 400 + } + }; + + this.testClient.getLayergroup(options, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + errors: [ 'Invalid filtered column: value'], + errors_with_context:[{ + type: 'layer', + message: 'Invalid filtered column: value', + layer: { + id: "layer0", + index: 0, + type: "mapnik", + } + }] + }); + + done(); + }); + }); + + + ['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => { + it(`aggregated ids are unique for ${placement} aggregation`, function (done) { + this.mapConfig = { + version: '1.6.0', + buffersize: { 'mvt': 0 }, + layers: [ + { + type: 'cartodb', + + options: { + sql: POINTS_SQL_1, + resolution: 1, + aggregation: { + threshold: 1 + } + } + } + ] + }; + if (placement !== 'default') { + this.mapConfig.layers[0].options.aggregation.placement = placement; + } + + this.testClient = new TestClient(this.mapConfig); + + this.testClient.getTile(1, 0, 1, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + + const tile1 = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(tile1.features)); + assert.ok(tile1.features.length > 0); + + this.testClient.getTile(1, 1, 0, { format: 'mvt' }, (err, res, mvt) => { + if (err) { + return done(err); + } + + const tile2 = JSON.parse(mvt.toGeoJSONSync(0)); + + assert.ok(Array.isArray(tile2.features)); + assert.ok(tile2.features.length > 0); + + const tile1Ids = tile1.features.map(f => f.properties.cartodb_id); + const tile2Ids = tile2.features.map(f => f.properties.cartodb_id); + const repeatedIds = tile1Ids.filter(id => tile2Ids.includes(id)); + assert.equal(repeatedIds.length, 0); + + done(); + }); + + }); + }); + }); + + }); }); }); diff --git a/test/acceptance/auth/authorization.js b/test/acceptance/auth/authorization.js index d7df8b32..02668251 100644 --- a/test/acceptance/auth/authorization.js +++ b/test/acceptance/auth/authorization.js @@ -37,7 +37,7 @@ describe('authorization', function() { }); }); - it('should create and get a named map tile using a regular apikey token', function (done) { + it.skip('should create and get a named map tile using a regular apikey token', function (done) { const apikeyToken = 'regular1'; const mapConfig = { version: '1.7.0', @@ -61,7 +61,7 @@ describe('authorization', function() { testClient.drain(done); }); - }); + }); it('should fail getting a named map tile with default apikey token', function (done) { const apikeyTokenCreate = 'regular1'; @@ -203,7 +203,7 @@ describe('authorization', function() { assert.ok(layergroupResult.hasOwnProperty('errors')); assert.equal(layergroupResult.errors.length, 1); assert.ok(layergroupResult.errors[0].match(/permission denied/), layergroupResult.errors[0]); - + testClient.drain(done); }); }); @@ -257,7 +257,7 @@ describe('authorization', function() { type: 'source', params: { query: 'select * from populated_places_simple_reduced' - } + } } ] }; @@ -354,7 +354,39 @@ describe('authorization', function() { }); }); - it('should create and get a named map tile using a regular apikey token', function (done) { + it('should fail while listing named maps with a regular apikey token', function (done) { + const apikeyToken = 'regular1'; + + const testClient = new TestClient({}, apikeyToken); + + testClient.getNamedMapList({ response: {status: 403 }}, function (err, res, body) { + assert.ifError(err); + + assert.equal(res.statusCode, 403); + + assert.equal(body.errors.length, 1); + assert.ok(body.errors[0].match(/Forbidden/), body.errors[0]); + + testClient.drain(done); + }); + }); + + it('should list named maps with master apikey token', function (done) { + const apikeyToken = 1234; + + const testClient = new TestClient({}, apikeyToken); + + testClient.getNamedMapList({}, function (err, res, body) { + assert.ifError(err); + + assert.equal(res.statusCode, 200); + assert.ok(Array.isArray(body.template_ids)); + + testClient.drain(done); + }); + }); + + it.skip('should create and get a named map tile using a regular apikey token', function (done) { const apikeyToken = 'regular1'; const template = { @@ -391,7 +423,7 @@ describe('authorization', function() { }); }); - it('should fail creating a named map using a regular apikey token and a private table', function (done) { + it.skip('should fail creating a named map using a regular apikey token and a private table', function (done) { const apikeyToken = 'regular1'; const template = { diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index dfb6174f..b4840879 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1272,6 +1272,8 @@ describe(suiteName, function() { it("cache control for layergroup default value", function(done) { global.environment.varnish.layergroupTtl = null; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate'); @@ -1287,6 +1289,8 @@ describe(suiteName, function() { var layergroupTtl = 300; global.environment.varnish.layergroupTtl = layergroupTtl; + var server = new CartodbWindshaft(serverOptions); + assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation, function(res) { assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate'); diff --git a/test/support/test-client.js b/test/support/test-client.js index 3e919175..7b068901 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1254,6 +1254,36 @@ TestClient.prototype.getAnalysesCatalog = function (params, callback) { ); }; +TestClient.prototype.getNamedMapList = function(params, callback) { + const request = { + url: `/api/v1/map/named?${qs.stringify({ api_key: this.apiKey })}`, + method: 'GET', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } + }; + + let expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + if (params.response) { + expectedResponse = Object.assign(expectedResponse, params.response); + } + + assert.response(this.server, request, expectedResponse, (res, err) => { + if (err) { + return callback(err); + } + const body = JSON.parse(res.body); + return callback(null, res, body); + }); +}; + TestClient.prototype.getNamedTile = function (name, z, x, y, format, options, callback) { const { params } = options; diff --git a/test/unit/cartodb/prepare-context.test.js b/test/unit/cartodb/prepare-context.test.js index 82526fe9..ff311cc6 100644 --- a/test/unit/cartodb/prepare-context.test.js +++ b/test/unit/cartodb/prepare-context.test.js @@ -11,7 +11,6 @@ const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/cl const authorizeMiddleware = require('../../../lib/cartodb/middleware/authorize'); const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/db-conn-setup'); const credentialsMiddleware = require('../../../lib/cartodb/middleware/credentials'); -const localsMiddleware = require('../../../lib/cartodb/middleware/locals'); var windshaft = require('windshaft'); @@ -66,18 +65,6 @@ describe('prepare-context', function() { return res; } - it('res.locals are created', function(done) { - const locals = localsMiddleware(); - let req = {}; - let res = {}; - - locals(prepareRequest(req), prepareResponse(res), function(err) { - if ( err ) { done(err); return; } - assert.ok(res.hasOwnProperty('locals'), 'response has locals'); - done(); - }); - }); - it('cleans up request', function(done){ var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}}; var res = {};