diff --git a/NEWS.md b/NEWS.md index 041491ef..62c2009e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ Released 2016-mm-dd Announcements: + - Upgrades camshaft to [0.10.0](https://github.com/CartoDB/camshaft/releases/tag/0.10.0) - Upgrades turbo-carto to [0.10.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.10.0) New features: diff --git a/docs/MapConfig-Analyses-extension.md b/docs/MapConfig-Analyses-extension.md new file mode 100644 index 00000000..53271eb3 --- /dev/null +++ b/docs/MapConfig-Analyses-extension.md @@ -0,0 +1,93 @@ +# 1. Purpose + +This specification describes an extension for +[MapConfig 1.4.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.4.0.md) version. + + +# 2. Changes over specification + +This extension targets layers with `sql` option, including layer types: `cartodb`, `mapnik`, and `torque`. + +It extends MapConfig with a new attribute: `analyses`. + +## 2.1 Analyses attribute + +The new analyses attribute must be an array of analyses as per [camshaft](https://github.com/CartoDB/camshaft). Each +analysis must adhere to the [camshaft-reference](https://github.com/CartoDB/camshaft/blob/0.8.0/reference/versions/0.7.0/reference.json) specification. + +Each node can have an id that can be later references to consume the query from MapConfig's layers. + +Basic analyses example: + +```javascript +[ + { + // REQUIRED + // string, `id` free identifier that can be reference from any layer + "id": "HEAD", + // REQUIRED + // string, `type` camshaft's analysis type + "type": "source", + // REQUIRED + // object, `params` will depend on `type`, check camshaft-reference for more information + "params": { + "query": "select * from your_table" + } + } +] +``` + +# 2.2. Integration with layers + +As pointed before an analysis node id can be referenced from layers to consume its output query. + +The layer consuming the output must reference it with the following option: + +``` +{ + "options": { + // REQUIRED + // object, `source` as in the future we might want to have other source options + "source": { + // REQUIRED + // string, `id` the analysis node identifier + "id": "HEAD" + } + } +} +``` + +## 2.3. Complete example + +``` +{ + "version": "1.4.0", + "layers": [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": "...", + "cartocss_version": "2.3.0" + } + } + ], + "analyses": [ + { + "id": "HEAD", + "type": "source", + "params": { + "query": "select * from your_table" + } + } + ] +} +``` + +# History + +## 1.0.0 + + - Initial version diff --git a/docs/MapConfig-Dataviews-extension.md b/docs/MapConfig-Dataviews-extension.md new file mode 100644 index 00000000..d8e23a82 --- /dev/null +++ b/docs/MapConfig-Dataviews-extension.md @@ -0,0 +1,314 @@ +# 1. Purpose + +This specification describes an extension for +[MapConfig 1.4.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.4.0.md) version. + + +# 2. Changes over specification + +This extension depends on Analyses extension. It extends MapConfig with a new attribute: `dataviews`. + +It makes possible to get tabular data from analysis nodes: lists, aggregated lists, aggregations, and histograms. + +## 2.1. Dataview types + +### List + +A list is a simple result set per row where is possible to retrieve several columns from the original layer query. + +Definition +``` +{ + // REQUIRED + // string, `type` the list type + “type”: “list”, + // REQUIRED + // object, `options` dataview params + “options”: { + // REQUIRED + // array, `columns` to select for the list + “columns”: [“name”, “description”] + } +} +``` + +Expected output +``` +{ + "type": "list", + "rows": [ + { + "{columnName1}": "val1", + "{columnName2}": 100 + }, + { + "{columnName1}": "val2", + "{columnName2}": 200 + } + ] +} +``` + +### Aggregation + +An aggregation is very similar to a list but results are aggregated by a column and a given aggregation function. + +Definition +``` +{ + // REQUIRED + // string, `type` the aggregation type + “type”: “aggregation”, + // REQUIRED + // object, `options` dataview params + “options”: { + // REQUIRED + // string, `column` column name to aggregate by + “column”: “country”, + // REQUIRED + // string, `aggregation` operation to perform + “aggregation”: “count” + // OPTIONAL + // string, `aggregationColumn` column value to aggregate + // This param is required when `aggregation` is different than "count" + “aggregationColumn”: “population” + } +} +``` + +Expected output +``` +{ + "type": "aggregation", + "categories": [ + { + "category": "foo", + "value": 100 + }, + { + "category": "bar", + "value": 200 + } + ] +} +``` + +### Histograms + +Histograms represent the data distribution for a column. + +Definition +``` +{ + // REQUIRED + // string, `type` the histogram type + “type”: “histogram”, + // REQUIRED + // object, `options` dataview params + “options”: { + // REQUIRED + // string, `column` column name to aggregate by + “column”: “name”, + // OPTIONAL + // number, `bins` how many buckets the histogram should use + “bins”: 10 + } +} +``` + +Expected output +``` +{ + "type": "histogram", + "bins": [{"bin": 0, "start": 2, "end": 2, "min": 2, "max": 2, "freq": 1}, null, null, {"bin": 3, "min": 40, "max": 44, "freq": 2}, null], + "width": 10 +} +``` + +### Formula + +Formulas given a final value representing the whole dataset. + +Definition +``` +{ + // REQUIRED + // string, `type` the formula type + “type”: “formula”, + // REQUIRED + // object, `options` dataview params + “options”: { + // REQUIRED + // string, `column` column name to aggregate by + “column”: “name”, + // REQUIRED + // string, `aggregation` operation to perform + “operation”: “count” + } +} +``` + +Operation must be: “min”, “max”, “count”, “avg”, or “sum”. + +Result +``` +{ + "type": "formula", + "operation": "count", + "result": 1000, + "nulls": 0 +} +``` + + +## 2.2 Dataviews attribute + +The new dataviews attribute must be a dictionary of dataviews. + +An analysis node id can be referenced from dataviews to consume its output query. + + +The layer consuming the output must reference it with the following option: + +``` +{ + // REQUIRED + // object, `source` as in the future we might want to have other source options + "source": { + // REQUIRED + // string, `id` the analysis node identifier + "id": "HEAD" + } +} +``` + +## 2.3. Complete example + +``` +{ + "version": "1.4.0", + "layers": [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": "...", + "cartocss_version": "2.3.0" + } + } + ], + "dataviews" { + "basic_histogram": { + "source": { + "id": "HEAD" + }, + "type": "histogram", + "options": { + "column": "pop_max" + } + } + }, + "analyses": [ + { + "id": "HEAD", + "type": "source", + "params": { + "query": "select * from your_table" + } + } + ] +} +``` + +## 3. Filters + +Camshaft's analyses expose a filtering capability and `aggregation` and `histogram` dataviews get them for free with + this extension. Filters are available with the very dataview id, so if you have a "basic_histogram" histogram dataview + you can filter with a range filter with "basic_histogram" name. + + +## 3.1 Filter types + +### Category + +Allows to remove results that are not contained within a set of elements. +Initially this filter can be applied to a `numeric` or `text` columns. + +Params + +``` +{ + “accept”: [“Spain”, “Germany”] + “reject”: [“Japan”] +} +``` + +### Range filter + +Allows to remove results that don’t satisfy numeric min and max values. +Filter is applied to a numeric column. + +Params + +``` +{ + “min”: 0, + “max”: 1000 +} +``` + +## 3.2. How to apply filters + +Filters must be applied at map instantiation time. + +With :mapconfig as a valid MapConfig and with :filters (a valid JSON) as: + +### Anonymous map + +`GET /api/v1/map?config=:mapconfig&filters=:filters` + +`POST /api/v1/map?filters=:filters` +with `BODY=:mapconfig` + +If in the future we need to support a bigger filters param and it doesn’t fit in the query string, + we might solve it by accepting: + +`POST /api/v1/map` +with `BODY={“config”: :mapconfig, “filters”: :filters}` + +### Named map + +Assume :params (a valid JSON) as named maps params, like in: `{“color”: “red”}` + +`GET /api/v1/named/:name/jsonp?config=:params&filters=:filters&callback=cb` + +`POST /api/v1/named/:name?filters=:filters` +with `BODY=:params` + +If, again, in the future we need to support a bigger filters param that doesn’t fit in the query string, + we might solve it by accepting: + +`POST /api/v1/named/:name` +with `BODY={“config”: :params, “filters”: :filters}` + + +## 3.3 Bounding box special filter + +A bounding box filter allows to remove results that don’t satisfy a geospatial range. + +The bounding box special filter is available per dataview and there is no need to create a bounding box definition as +it’s always possible to apply a bbox filter per dataview. + +A dataview can get its result filtered by bounding box by sending a bbox param in the query string, +param must be in the form `west,south,east,north`. + +So applying a bbox filter to a dataview looks like: +GET /api/v1/map/:layergroupid/dataview/:dataview_name?bbox=-90,-45,90,45 + +# History + +## 1.0.0-alpha + + - WIP document diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index 9adb29b9..ebafbbac 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -1,24 +1,17 @@ var _ = require('underscore'); var dot = require('dot'); -var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); -var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); -var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); +var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); var templateName = require('../backends/template_maps').templateName; var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, analysisBackend, userLimitsApi, - overviewsAdapter, turboCartoAdapter) { +function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; - - this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); - this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend); - this.overviewsAdapter = overviewsAdapter; - this.turboCartoAdapter = turboCartoAdapter; + this.mapConfigAdapter = mapConfigAdapter; this.providerCache = new LruCache({ max: 2000 }); } @@ -36,10 +29,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok this.pgConnection, this.metadataBackend, this.userLimitsApi, - this.namedLayersAdapter, - this.overviewsAdapter, - this.turboCartoAdapter, - this.analysisMapConfigAdapter, + this.mapConfigAdapter, user, templateId, config, diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 1c79e47a..8c426c7c 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -10,7 +10,7 @@ var userMiddleware = require('../middleware/user'); var DataviewBackend = require('../backends/dataview'); var AnalysisStatusBackend = require('../backends/analysis-status'); -var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); +var MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider'); var QueryTables = require('cartodb-query-tables'); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1f8f25bb..32459059 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -15,11 +15,8 @@ var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); -var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); -var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); -var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); -var SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); +var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); +var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); /** * @param {AuthApi} authApi @@ -30,14 +27,11 @@ var SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapc * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables - * @param {MapConfigOverviewsAdapter} overviewsAdapter - * @param {TurboCartoAdapter} turboCartoAdapter - * @param {AnalysisBackend} analysisBackend + * @param {MapConfigAdapter} mapConfigAdapter * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, - overviewsAdapter, turboCartoAdapter, analysisBackend) { + surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) { BaseController.call(this, authApi, pgConnection); @@ -48,12 +42,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; - this.turboCartoAdapter = turboCartoAdapter; - this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend); - this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); - this.overviewsAdapter = overviewsAdapter; - this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter(); + this.mapConfigAdapter = mapConfigAdapter; } util.inherits(MapController, BaseController); @@ -134,20 +124,17 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { var self = this; var mapConfig; - var analysesResults = []; + + var context = {}; step( function setupParams(){ self.req2params(req, this); }, prepareConfigFn, - function prepareSqlWrap(err, requestMapConfig) { + function prepareAdapterMapConfig(err, requestMapConfig) { assert.ifError(err); - self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this); - }, - function prepareAnalysisLayers(err, requestMapConfig) { - assert.ifError(err); - var analysisConfiguration = { + context.analysisConfiguration = { db: { host: req.params.dbhost, port: req.params.dbport, @@ -160,71 +147,12 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { apiKey: req.params.api_key } }; - - var filters = {}; - if (req.params.filters) { - try { - filters = JSON.parse(req.params.filters); - } catch (e) { - // ignore - } - } - self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this); + self.mapConfigAdapter.getMapConfig(req.context.user, requestMapConfig, req.params, context, this); }, - function beforeLayergroupCreate(err, requestMapConfig, _analysesResults) { + function createLayergroup(err, requestMapConfig) { assert.ifError(err); - var next = this; - analysesResults = _analysesResults; - self.namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, self.pgConnection, - function(err, layers, datasource) { - if (err) { - return next(err); - } - - if (layers) { - requestMapConfig.layers = layers; - } - return next(null, requestMapConfig, datasource); - } - ); - }, - function addOverviewsInformation(err, requestMapConfig, datasource) { - assert.ifError(err); - var next = this; - self.overviewsAdapter.getLayers( - req.context.user, requestMapConfig.layers, analysesResults, - function(err, layers) { - if (err) { - return next(err); - } - - if (layers) { - requestMapConfig.layers = layers; - } - - return next(null, requestMapConfig, datasource); - } - ); - }, - function parseTurboCarto(err, requestMapConfig, datasource) { - assert.ifError(err); - - var next = this; - self.turboCartoAdapter.getLayers(req.context.user, requestMapConfig.layers, function (err, layers) { - if (err) { - return next(err); - } - - if (layers) { - requestMapConfig.layers = layers; - } - - return next(null, requestMapConfig, datasource); - }); - }, - function createLayergroup(err, requestMapConfig, datasource) { - assert.ifError(err); - mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource()); + var datasource = context.datasource || Datasource.EmptyDatasource(); + mapConfig = new MapConfig(requestMapConfig, datasource); self.mapBackend.createLayergroup( mapConfig, req.params, new CreateLayergroupMapConfigProvider(mapConfig, req.context.user, self.userLimitsApi, req.params), @@ -233,6 +161,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); + var analysesResults = context.analysesResults || []; self.afterLayergroupCreate(req, res, mapConfig, analysesResults, layergroup, this); }, function finish(err, layergroup) { @@ -270,10 +199,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn self.pgConnection, self.metadataBackend, self.userLimitsApi, - self.namedLayersAdapter, - self.overviewsAdapter, - self.turboCartoAdapter, - self.analysisMapConfigAdapter, + self.mapConfigAdapter, cdbuser, req.params.template_id, templateParams, diff --git a/lib/cartodb/models/analysis-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js similarity index 93% rename from lib/cartodb/models/analysis-mapconfig-adapter.js rename to lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js index 6d54a3af..b704ed06 100644 --- a/lib/cartodb/models/analysis-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js @@ -11,73 +11,25 @@ function AnalysisMapConfigAdapter(analysisBackend) { module.exports = AnalysisMapConfigAdapter; -var SKIP_COLUMNS = { - 'the_geom': true, - 'the_geom_webmercator': true -}; - -function skipColumns(columnNames) { - return columnNames - .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); -} - -var layerQueryTemplate = dot.template([ - 'SELECT {{=it._columns}}', - 'FROM ({{=it._query}}) _cdb_analysis_query' -].join('\n')); - -function layerQuery(node) { - if (node.type === 'source') { - return node.getQuery(); - } - var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); - return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') }); -} - -function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { - var analyses = requestMapConfig.analyses || []; - - requestMapConfig.analyses = analyses.map(function(analysisDefinition) { - var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition); - var definition = analysisDefinition; - Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) { - definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] }); - }); - - return definition; - }); - - return requestMapConfig; -} - -function shouldAdaptLayers(requestMapConfig) { - return Array.isArray(requestMapConfig.layers) && - Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; -} - -var DATAVIEW_TYPE_2_FILTER_TYPE = { - aggregation: 'category', - histogram: 'range' -}; -function getFilter(dataview, params) { - var type = dataview.type; - - return { - type: DATAVIEW_TYPE_2_FILTER_TYPE[type], - column: dataview.options.column, - params: params - }; -} - -AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration, requestMapConfig, filters, callback) { +AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) { // jshint maxcomplexity:7 var self = this; - filters = filters || {}; if (!shouldAdaptLayers(requestMapConfig)) { return callback(null, requestMapConfig); } + var analysisConfiguration = context.analysisConfiguration; + + var filters = {}; + if (params.filters) { + try { + filters = JSON.parse(params.filters); + } catch (e) { + // ignore + } + } + var dataviewsFilters = filters.dataviews || {}; debug(dataviewsFilters); var dataviews = requestMapConfig.dataviews || {}; @@ -145,6 +97,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration var analysisSql = layerQuery(layerNode); var sqlQueryWrap = layer.options.sql_wrap; if (sqlQueryWrap) { + layer.options.sql_raw = analysisSql; analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql); } layer.options.sql = analysisSql; @@ -167,10 +120,70 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(analysisConfiguration return callback(missingNodesErrors); } - return callback(null, requestMapConfig, analysesResults); + context.analysesResults = analysesResults; + + return callback(null, requestMapConfig); }); }; +var SKIP_COLUMNS = { + 'the_geom': true, + 'the_geom_webmercator': true +}; + +function skipColumns(columnNames) { + return columnNames + .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); +} + +var layerQueryTemplate = dot.template([ + 'SELECT {{=it._columns}}', + 'FROM ({{=it._query}}) _cdb_analysis_query' +].join('\n')); + +function layerQuery(node) { + if (node.type === 'source') { + return node.getQuery(); + } + var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); + return layerQueryTemplate({ _query: node.getQuery(), _columns: _columns.join(', ') }); +} + +function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { + var analyses = requestMapConfig.analyses || []; + + requestMapConfig.analyses = analyses.map(function(analysisDefinition) { + var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition); + var definition = analysisDefinition; + Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) { + definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] }); + }); + + return definition; + }); + + return requestMapConfig; +} + +function shouldAdaptLayers(requestMapConfig) { + return Array.isArray(requestMapConfig.layers) && + Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; +} + +var DATAVIEW_TYPE_2_FILTER_TYPE = { + aggregation: 'category', + histogram: 'range' +}; +function getFilter(dataview, params) { + var type = dataview.type; + + return { + type: DATAVIEW_TYPE_2_FILTER_TYPE[type], + column: dataview.options.column, + params: params + }; +} + function getLayerSourceId(layer) { return layer.options.source && layer.options.source.id; } diff --git a/lib/cartodb/models/mapconfig/adapter/index.js b/lib/cartodb/models/mapconfig/adapter/index.js new file mode 100644 index 00000000..70704bea --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/index.js @@ -0,0 +1,26 @@ +'use strict'; + +function MapConfigAdapter(adapters) { + this.adapters = Array.isArray(adapters) ? adapters : Array.apply(null, arguments); +} + +module.exports = MapConfigAdapter; + +MapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) { + var self = this; + var i = 0; + var tasksLeft = this.adapters.length; + + function next(err, _requestMapConfig) { + if (err) { + return callback(err); + } + if (tasksLeft-- === 0) { + return callback(null, _requestMapConfig); + } + var nextAdapter = self.adapters[i++]; + nextAdapter.getMapConfig(user, _requestMapConfig, params, context, next); + } + + next(null, requestMapConfig); +}; diff --git a/lib/cartodb/models/mapconfig_named_layers_adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js similarity index 82% rename from lib/cartodb/models/mapconfig_named_layers_adapter.js rename to lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js index 26b6c3a1..fd00e344 100644 --- a/lib/cartodb/models/mapconfig_named_layers_adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter.js @@ -2,17 +2,20 @@ var queue = require('queue-async'); var _ = require('underscore'); var Datasource = require('windshaft').model.Datasource; -function MapConfigNamedLayersAdapter(templateMaps) { +function MapConfigNamedLayersAdapter(templateMaps, pgConnection) { this.templateMaps = templateMaps; + this.pgConnection = pgConnection; } module.exports = MapConfigNamedLayersAdapter; -MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbMetadata, callback) { +MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { var self = this; + var layers = requestMapConfig.layers; + if (!layers) { - return callback(null); + return callback(null, requestMapConfig); } var adaptLayersQueue = queue(layers.length); @@ -28,9 +31,9 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM var templateConfigParams = layer.options.config || {}; var templateAuthTokens = layer.options.auth_tokens; - self.templateMaps.getTemplate(username, templateName, function(err, template) { + self.templateMaps.getTemplate(user, templateName, function(err, template) { if (err || !template) { - return done(new Error("Template '" + templateName + "' of user '" + username + "' not found")); + return done(new Error("Template '" + templateName + "' of user '" + user + "' not found")); } if (self.templateMaps.isAuthorized(template, templateAuthTokens)) { @@ -96,7 +99,10 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM }); - return callback(null, layers, datasourceBuilder.build()); + requestMapConfig.layers = layers; + context.datasource = datasourceBuilder.build(); + + return callback(null, requestMapConfig); } @@ -104,7 +110,7 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM if (_.some(layers, isNamedTypeLayer)) { // Lazy load dbAuth - dbMetadata.setDBAuth(username, dbAuth, function(err) { + this.pgConnection.setDBAuth(user, dbAuth, function(err) { if (err) { return callback(err); } @@ -114,7 +120,8 @@ MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbM adaptLayersQueue.awaitAll(layersAdaptQueueFinish); }); } else { - return callback(null, layers, datasourceBuilder.build()); + context.datasource = datasourceBuilder.build(); + return callback(null, requestMapConfig); } }; diff --git a/lib/cartodb/models/mapconfig_overviews_adapter.js b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js similarity index 87% rename from lib/cartodb/models/mapconfig_overviews_adapter.js rename to lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js index 3b8d54a2..b8a473b5 100644 --- a/lib/cartodb/models/mapconfig_overviews_adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js @@ -9,11 +9,14 @@ function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) { module.exports = MapConfigOverviewsAdapter; -MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analysesResults, callback) { +MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) { var self = this; + var layers = requestMapConfig.layers; + var analysesResults = context.analysesResults; + if (!layers || layers.length === 0) { - return callback(null, layers); + return callback(null, requestMapConfig); } var augmentLayersQueue = queue(layers.length); @@ -22,7 +25,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) { return done(null, layer); } - self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){ + self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){ if (err) { done(err, layer); } else { @@ -51,7 +54,7 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy var next_step = this; if ( filters ) { self.filterStatsApi.getFilterStats( - username, + user, unfiltered_query, filters, function(err, stats) { if ( !err ) { @@ -85,7 +88,9 @@ MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, analy return callback(new Error('Missing layers array from layergroup config')); } - return callback(null, layers); + requestMapConfig.layers = layers; + + return callback(null, requestMapConfig); } layers.forEach(function(layer) { diff --git a/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js b/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js index 6b9a5130..8bb3d201 100644 --- a/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js +++ b/lib/cartodb/models/mapconfig/adapter/sql-wrap-mapconfig-adapter.js @@ -4,7 +4,7 @@ function SqlWrapMapConfigAdapter() { module.exports = SqlWrapMapConfigAdapter; -SqlWrapMapConfigAdapter.prototype.getMapConfig = function(requestMapConfig, callback) { +SqlWrapMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfig, params, context, callback) { if (requestMapConfig && Array.isArray(requestMapConfig.layers)) { requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { if (layer.options) { diff --git a/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js new file mode 100644 index 00000000..3eb29197 --- /dev/null +++ b/lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js @@ -0,0 +1,102 @@ +'use strict'; + +var dot = require('dot'); +dot.templateSettings.strip = false; +var queue = require('queue-async'); +var SubstitutionTokens = require('../../../utils/substitution-tokens'); + +function TurboCartoAdapter(turboCartoParser) { + this.turboCartoParser = turboCartoParser; +} + +module.exports = TurboCartoAdapter; + +TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { + var self = this; + + var layers = requestMapConfig.layers; + + if (!layers || layers.length === 0) { + return callback(null, requestMapConfig); + } + + var parseCartoQueue = queue(layers.length); + + layers.forEach(function(layer) { + parseCartoQueue.defer(self._parseCartoCss.bind(self), user, layer); + }); + + parseCartoQueue.awaitAll(function (err, layers) { + if (err) { + return callback(err); + } + + requestMapConfig.layers = layers; + + return callback(null, requestMapConfig); + }); +}; + +var bboxTemplate = dot.template('(select ST_SetSRID(st_extent(the_geom_webmercator), 3857) from ({{=it._sql}}) __c)'); +var zoomTemplate = dot.template([ + 'GREATEST(', + 'ceil(log(40075017000 / 256 / GREATEST(', + ' st_xmax({{=it._bbox}}) - st_xmin({{=it._bbox}}),', + ' st_ymax({{=it._bbox}}) - st_ymin({{=it._bbox}})', + '))/log(2)),', + '0', + ')' +].join('\n')); +var pixelSizeTemplate = dot.template('40075017 * cos(ST_Y(ST_Centroid({{=it._bbox}}))) / 2 ^ (({{=it._zoom}}) + 8)'); +var scaleDenominatorTemplate = dot.template('({{=it._pixelSize}} / 0.00028)::numeric'); + + +TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) { + if (!shouldParseLayerCartocss(layer)) { + return callback(null, layer); + } + + var tokens = { + bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)', + scale_denominator: '500000001', + pixel_width: '156412', + pixel_height: '156412' + }; + + var layerSql = layer.options.sql; + var layerRawSql = layer.options.sql_raw; + if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) { + var bbox = bboxTemplate({ _sql: layerRawSql }); + var zoom = zoomTemplate({ _bbox: bbox }); + var pixelSize = pixelSizeTemplate({ _bbox: bbox, _zoom: zoom }); + var scaleDenominator = scaleDenominatorTemplate({ _pixelSize: pixelSize }); + + tokens = { + bbox: bbox, + scale_denominator: scaleDenominator, + pixel_width: pixelSize, + pixel_height: pixelSize + }; + } + + var sql = SubstitutionTokens.replace(layerSql, tokens); + + this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) { + // Only return turbo-carto errors + if (err && err.name === 'TurboCartoError') { + err = new Error('turbo-carto: ' + err.message); + err.http_status = 400; + return callback(err); + } + + // Try to continue in the rest of the cases + if (cartocss) { + layer.options.cartocss = cartocss; + } + return callback(null, layer); + }); +}; + +function shouldParseLayerCartocss(layer) { + return layer && layer.options && layer.options.cartocss && layer.options.sql; +} diff --git a/lib/cartodb/models/mapconfig/create_layergroup_provider.js b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js similarity index 95% rename from lib/cartodb/models/mapconfig/create_layergroup_provider.js rename to lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js index 409d9bda..af5a775b 100644 --- a/lib/cartodb/models/mapconfig/create_layergroup_provider.js +++ b/lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js @@ -1,7 +1,7 @@ var assert = require('assert'); var step = require('step'); -var MapStoreMapConfigProvider = require('./map_store_provider'); +var MapStoreMapConfigProvider = require('./map-store-provider'); /** * @param {MapConfig} mapConfig diff --git a/lib/cartodb/models/mapconfig/map_store_provider.js b/lib/cartodb/models/mapconfig/provider/map-store-provider.js similarity index 100% rename from lib/cartodb/models/mapconfig/map_store_provider.js rename to lib/cartodb/models/mapconfig/provider/map-store-provider.js diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/provider/named-map-provider.js similarity index 73% rename from lib/cartodb/models/mapconfig/named_map_provider.js rename to lib/cartodb/models/mapconfig/provider/named-map-provider.js index e1ad6fea..7c9c0dc5 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/provider/named-map-provider.js @@ -4,26 +4,20 @@ var crypto = require('crypto'); var dot = require('dot'); var step = require('step'); var MapConfig = require('windshaft').model.MapConfig; -var templateName = require('../../backends/template_maps').templateName; +var templateName = require('../../../backends/template_maps').templateName; var QueryTables = require('cartodb-query-tables'); -var SqlWrapMapConfigAdapter = require('./adapter/sql-wrap-mapconfig-adapter'); /** * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, - namedLayersAdapter, overviewsAdapter, turboCartoAdapter, analysisMapConfigAdapter, +function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter, owner, templateId, config, authToken, params) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; - this.namedLayersAdapter = namedLayersAdapter; - this.turboCartoAdapter = turboCartoAdapter; - this.analysisMapConfigAdapter = analysisMapConfigAdapter; - this.overviewsAdapter = overviewsAdapter; - this.sqlWrapMapConfigAdapter = new SqlWrapMapConfigAdapter(); + this.mapConfigAdapter = mapConfigAdapter; this.owner = owner; this.templateName = templateName(templateId); @@ -56,10 +50,11 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { var self = this; var mapConfig = null; - var datasource = null; var rendererParams; var apiKey; + var context = {}; + step( function getTemplate() { self.getTemplate(this); @@ -97,13 +92,9 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { assert.ifError(err); return self.templateMaps.instance(self.template, templateParams); }, - function prepareSqlWrap(err, requestMapConfig) { + function prepareAdapterMapConfig(err, requestMapConfig) { assert.ifError(err); - self.sqlWrapMapConfigAdapter.getMapConfig(requestMapConfig, this); - }, - function prepareAnalysisLayers(err, requestMapConfig) { - assert.ifError(err); - var analysisConfiguration = { + context.analysisConfiguration = { db: { host: rendererParams.dbhost, port: rendererParams.dbport, @@ -116,75 +107,17 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { apiKey: apiKey } }; - - var filters = {}; - if (self.params.filters) { - try { - filters = JSON.parse(self.params.filters); - } catch (e) { - // ignore - } - } - self.analysisMapConfigAdapter.getMapConfig(analysisConfiguration, requestMapConfig, filters, this); + self.mapConfigAdapter.getMapConfig(self.owner, requestMapConfig, rendererParams, context, this); }, - function prepareLayergroup(err, _mapConfig, analysesResults) { - assert.ifError(err); - var next = this; - self.analysesResults = analysesResults || []; - self.namedLayersAdapter.getLayers(self.owner, _mapConfig.layers, self.pgConnection, - function(err, layers, datasource) { - if (err) { - return next(err); - } - - if (layers) { - _mapConfig.layers = layers; - } - return next(null, _mapConfig, datasource); - } - ); - }, - function addOverviewsInformation(err, _mapConfig, datasource) { - assert.ifError(err); - var next = this; - - self.overviewsAdapter.getLayers(self.owner, _mapConfig.layers, self.analysesResults, function(err, layers) { - if (err) { - return next(err); - } - - if (layers) { - _mapConfig.layers = layers; - } - - return next(null, _mapConfig, datasource); - }); - }, - function parseTurboCarto(err, _mapConfig, datasource) { - assert.ifError(err); - var next = this; - - self.turboCartoAdapter.getLayers(self.owner, _mapConfig.layers, function (err, layers) { - if (err) { - return next(err); - } - - if (layers) { - _mapConfig.layers = layers; - } - - return next(null, _mapConfig, datasource); - }); - }, - function prepareContextLimits(err, _mapConfig, _datasource) { + function prepareContextLimits(err, _mapConfig) { assert.ifError(err); mapConfig = _mapConfig; - datasource = _datasource; self.userLimitsApi.getRenderLimits(self.owner, this); }, function cacheAndReturnMapConfig(err, renderLimits) { self.err = err; - self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, datasource); + self.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource); + self.analysesResults = context.analysesResults || []; self.rendererParams = rendererParams; self.context.limits = renderLimits || {}; return callback(self.err, self.mapConfig, self.rendererParams, self.context); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 8117d208..825ae2b7 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -34,10 +34,14 @@ var AnalysisBackend = require('./backends/analysis'); var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); -var MapConfigOverviewsAdapter = require('./models/mapconfig_overviews_adapter'); - var TurboCartoParser = require('./utils/style/turbo-carto-parser'); -var TurboCartoAdapter = require('./utils/style/turbo-carto-adapter'); + +var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); +var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter'); +var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter'); +var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter'); +var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'); +var MapConfigAdapter = require('./models/mapconfig/adapter'); module.exports = function(serverOptions) { // Make stats client globally accessible @@ -150,19 +154,22 @@ module.exports = function(serverOptions) { var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; - var overviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi); - var turboCartoParser = new TurboCartoParser(pgQueryRunner); - var turboCartoAdapter = new TurboCartoAdapter(turboCartoParser); + + var mapConfigAdapter = new MapConfigAdapter( + new MapConfigNamedLayersAdapter(templateMaps, pgConnection), + new SqlWrapMapConfigAdapter(), + new AnalysisMapConfigAdapter(analysisBackend), + new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi), + new TurboCartoAdapter(turboCartoParser) + ); var namedMapProviderCache = new NamedMapProviderCache( templateMaps, pgConnection, metadataBackend, - analysisBackend, userLimitsApi, - overviewsAdapter, - turboCartoAdapter + mapConfigAdapter ); ['update', 'delete'].forEach(function(eventType) { @@ -201,9 +208,7 @@ module.exports = function(serverOptions) { surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - overviewsAdapter, - turboCartoAdapter, - analysisBackend + mapConfigAdapter ).register(app); new controller.NamedMaps( diff --git a/lib/cartodb/utils/style/turbo-carto-adapter.js b/lib/cartodb/utils/style/turbo-carto-adapter.js deleted file mode 100644 index d1c0c81f..00000000 --- a/lib/cartodb/utils/style/turbo-carto-adapter.js +++ /dev/null @@ -1,70 +0,0 @@ -'use strict'; - -var queue = require('queue-async'); -var SubstitutionTokens = require('../substitution-tokens'); - -function TurboCartoAdapter(turboCartoParser) { - this.turboCartoParser = turboCartoParser; -} - -module.exports = TurboCartoAdapter; - -TurboCartoAdapter.prototype.getLayers = function (username, layers, callback) { - var self = this; - - if (!layers || layers.length === 0) { - return callback(null, layers); - } - - var parseCartoQueue = queue(layers.length); - - layers.forEach(function(layer) { - parseCartoQueue.defer(self._parseCartoCss.bind(self), username, layer); - }); - - parseCartoQueue.awaitAll(function (err, layers) { - if (err) { - return callback(err); - } - - return callback(null, layers); - }); -}; - -TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, callback) { - if (isNotLayerToParseCartocss(layer)) { - return process.nextTick(function () { - callback(null, layer); - }); - } - - var sql = SubstitutionTokens.replace(layer.options.sql, { - bbox: 'ST_MakeEnvelope(-20037508.34,-20037508.34,20037508.34,20037508.34,3857)', - scale_denominator: '500000001', - pixel_width: '156412', - pixel_height: '156412' - }); - - this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) { - // Only return turbo-carto errors - if (err && err.name === 'TurboCartoError') { - err = new Error('turbo-carto: ' + err.message); - err.http_status = 400; - return callback(err); - } - - // Try to continue in the rest of the cases - if (cartocss) { - layer.options.cartocss = cartocss; - } - return callback(null, layer); - }); -}; - -function isNotLayerToParseCartocss(layer) { - if (!layer || !layer.options || !layer.options.cartocss || !layer.options.sql) { - return true; - } - - return false; -} diff --git a/lib/cartodb/utils/substitution-tokens.js b/lib/cartodb/utils/substitution-tokens.js index af09f838..5905fa4d 100644 --- a/lib/cartodb/utils/substitution-tokens.js +++ b/lib/cartodb/utils/substitution-tokens.js @@ -6,6 +6,16 @@ var SUBSTITUTION_TOKENS = { }; var SubstitutionTokens = { + tokens: function(sql) { + return Object.keys(SUBSTITUTION_TOKENS).filter(function(tokenName) { + return !!sql.match(SUBSTITUTION_TOKENS[tokenName]); + }); + }, + + hasTokens: function(sql) { + return this.tokens(sql).length > 0; + }, + replace: function(sql, replaceValues) { Object.keys(replaceValues).forEach(function(token) { if (SUBSTITUTION_TOKENS[token]) { diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 512abfc7..c37e5370 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -80,7 +80,7 @@ }, "type-is": { "version": "1.6.13", - "from": "type-is@>=1.6.10 <1.7.0", + "from": "type-is@>=1.6.6 <1.7.0", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz", "dependencies": { "media-typer": { @@ -90,7 +90,7 @@ }, "mime-types": { "version": "2.1.11", - "from": "mime-types@>=2.1.2 <2.2.0", + "from": "mime-types@>=2.1.11 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.11.tgz", "dependencies": { "mime-db": { @@ -105,9 +105,9 @@ } }, "camshaft": { - "version": "0.8.0", - "from": "camshaft@0.8.0", - "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.8.0.tgz", + "version": "0.10.0", + "from": "camshaft@0.10.0", + "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.10.0.tgz", "dependencies": { "async": { "version": "1.5.2", @@ -1212,7 +1212,8 @@ }, "step-profiler": { "version": "0.3.0", - "from": "step-profiler@>=0.3.0 <0.4.0" + "from": "step-profiler@>=0.3.0 <0.4.0", + "resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz" }, "turbo-carto": { "version": "0.10.0", @@ -3009,16 +3010,16 @@ "from": "combined-stream@>=1.0.5 <1.1.0", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz" }, - "commander": { - "version": "2.9.0", - "from": "commander@>=2.9.0 <3.0.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz" - }, "concat-map": { "version": "0.0.1", "from": "concat-map@0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz" }, + "commander": { + "version": "2.9.0", + "from": "commander@>=2.9.0 <3.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz" + }, "core-util-is": { "version": "1.0.2", "from": "core-util-is@>=1.0.0 <1.1.0", @@ -3199,16 +3200,16 @@ "from": "jsbn@>=0.1.0 <0.2.0", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" }, - "json-stringify-safe": { - "version": "5.0.1", - "from": "json-stringify-safe@>=5.0.1 <5.1.0", - "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" - }, "json-schema": { "version": "0.2.2", "from": "json-schema@0.2.2", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz" }, + "json-stringify-safe": { + "version": "5.0.1", + "from": "json-stringify-safe@>=5.0.1 <5.1.0", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + }, "jsonpointer": { "version": "2.0.0", "from": "jsonpointer@2.0.0", @@ -3279,16 +3280,16 @@ "from": "ms@0.7.1", "resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz" }, - "node-uuid": { - "version": "1.4.7", - "from": "node-uuid@>=1.4.7 <1.5.0", - "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" - }, "nopt": { "version": "3.0.6", "from": "nopt@>=3.0.1 <3.1.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-3.0.6.tgz" }, + "node-uuid": { + "version": "1.4.7", + "from": "node-uuid@>=1.4.7 <1.5.0", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" + }, "npmlog": { "version": "2.0.3", "from": "npmlog@>=2.0.0 <2.1.0", @@ -3299,16 +3300,16 @@ "from": "oauth-sign@>=0.8.1 <0.9.0", "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz" }, - "path-is-absolute": { - "version": "1.0.0", - "from": "path-is-absolute@>=1.0.0 <2.0.0", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.0.tgz" - }, "once": { "version": "1.3.3", "from": "once@>=1.3.0 <2.0.0", "resolved": "https://registry.npmjs.org/once/-/once-1.3.3.tgz" }, + "path-is-absolute": { + "version": "1.0.0", + "from": "path-is-absolute@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.0.tgz" + }, "pinkie": { "version": "2.0.4", "from": "pinkie@>=2.0.0 <3.0.0", @@ -3339,16 +3340,16 @@ "from": "request@>=2.0.0 <3.0.0", "resolved": "https://registry.npmjs.org/request/-/request-2.72.0.tgz" }, - "semver": { - "version": "5.1.0", - "from": "semver@>=5.1.0 <5.2.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz" - }, "rimraf": { "version": "2.5.2", "from": "rimraf@>=2.5.0 <2.6.0", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.5.2.tgz" }, + "semver": { + "version": "5.1.0", + "from": "semver@>=5.1.0 <5.2.0", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.1.0.tgz" + }, "sntp": { "version": "1.0.9", "from": "sntp@>=1.0.0 <2.0.0", diff --git a/package.json b/package.json index 57f219c5..baf6d069 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "dependencies": { "body-parser": "~1.14.0", - "camshaft": "0.8.0", + "camshaft": "0.10.0", "cartodb-psql": "~0.6.1", "cartodb-query-tables": "~0.1.0", "cartodb-redis": "~0.13.0", diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index 26341a0a..bab87ee4 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -7,7 +7,7 @@ var serverOptions = require('../../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); var TestClient = require('../../support/test-client'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('named-maps analysis', function() { diff --git a/test/acceptance/dynamic-styling-named-maps.js b/test/acceptance/dynamic-styling-named-maps.js index 934ddbcb..5fe2db3c 100644 --- a/test/acceptance/dynamic-styling-named-maps.js +++ b/test/acceptance/dynamic-styling-named-maps.js @@ -1,6 +1,6 @@ var assert = require('../support/assert'); var step = require('step'); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); var testHelper = require(__dirname + '/../support/test_helper'); var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); diff --git a/test/acceptance/limits.js b/test/acceptance/limits.js index ef91dac4..30a4e739 100644 --- a/test/acceptance/limits.js +++ b/test/acceptance/limits.js @@ -7,7 +7,7 @@ var redis = require('redis'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); describe('render limits', function() { diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 2dbfbbbd..7095f4c5 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -6,7 +6,7 @@ var strftime = require('strftime'); var redis_stats_db = 5; var helper = require(__dirname + '/../support/test_helper'); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures'; diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index e443f3aa..251ecc38 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -4,7 +4,7 @@ var assert = require('../support/assert'); var _ = require('underscore'); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); var QueryTables = require('cartodb-query-tables'); diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js index eeb74bc2..8788e5a6 100644 --- a/test/acceptance/named_layers.js +++ b/test/acceptance/named_layers.js @@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); var RedisPool = require('redis-mpool'); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); diff --git a/test/acceptance/overviews_metadata.js b/test/acceptance/overviews_metadata.js index 0822cfe8..ad0f23e5 100644 --- a/test/acceptance/overviews_metadata.js +++ b/test/acceptance/overviews_metadata.js @@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); var RedisPool = require('redis-mpool'); diff --git a/test/acceptance/overviews_metadata_named_maps.js b/test/acceptance/overviews_metadata_named_maps.js index 4f0ff6e7..8e9720ec 100644 --- a/test/acceptance/overviews_metadata_named_maps.js +++ b/test/acceptance/overviews_metadata_named_maps.js @@ -5,7 +5,7 @@ var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); var RedisPool = require('redis-mpool'); diff --git a/test/acceptance/ported/attributes.js b/test/acceptance/ported/attributes.js index 1c8f3983..5151cc09 100644 --- a/test/acceptance/ported/attributes.js +++ b/test/acceptance/ported/attributes.js @@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server'); var PortedServerOptions = require('./support/ported_server_options'); var BaseController = require('../../../lib/cartodb/controllers/base'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('attributes', function() { diff --git a/test/acceptance/ported/multilayer.js b/test/acceptance/ported/multilayer.js index eb337070..fa6648f4 100644 --- a/test/acceptance/ported/multilayer.js +++ b/test/acceptance/ported/multilayer.js @@ -7,7 +7,7 @@ var step = require('step'); var mapnik = require('windshaft').mapnik; var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); var BaseController = require('../../../lib/cartodb/controllers/base'); describe('multilayer', function() { diff --git a/test/acceptance/ported/multilayer_interactivity.js b/test/acceptance/ported/multilayer_interactivity.js index 44b54d4f..3f3f12b1 100644 --- a/test/acceptance/ported/multilayer_interactivity.js +++ b/test/acceptance/ported/multilayer_interactivity.js @@ -5,7 +5,7 @@ var _ = require('underscore'); var cartodbServer = require('../../../lib/cartodb/server'); var getLayerTypeFn = require('windshaft').model.MapConfig.prototype.getType; var PortedServerOptions = require('./support/ported_server_options'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); var BaseController = require('../../../lib/cartodb/controllers/base'); diff --git a/test/acceptance/ported/raster.js b/test/acceptance/ported/raster.js index cd3d43b9..2cea4bf4 100644 --- a/test/acceptance/ported/raster.js +++ b/test/acceptance/ported/raster.js @@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var BaseController = require('../../../lib/cartodb/controllers/base'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('raster', function() { diff --git a/test/acceptance/ported/retina.js b/test/acceptance/ported/retina.js index b13d0fe4..1c4b16f1 100644 --- a/test/acceptance/ported/retina.js +++ b/test/acceptance/ported/retina.js @@ -6,7 +6,7 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var BaseController = require('../../../lib/cartodb/controllers/base'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('retina support', function() { diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 39c89c66..30b5f2bc 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -7,7 +7,7 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var BaseController = require('../../../lib/cartodb/controllers/base'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 85; diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 1cc5331f..875d42dc 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -1,6 +1,6 @@ var _ = require('underscore'); var serverOptions = require('../../../../lib/cartodb/server_options'); -var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../../support/layergroup-token'); var mapnik = require('windshaft').mapnik; var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter'); var overviewsQueryRewriter = new OverviewsQueryRewriter({ diff --git a/test/acceptance/ported/support/test_client.js b/test/acceptance/ported/support/test_client.js index 54893755..dad3ff3e 100644 --- a/test/acceptance/ported/support/test_client.js +++ b/test/acceptance/ported/support/test_client.js @@ -1,5 +1,5 @@ var testHelper = require('../../../support/test_helper'); -var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../../support/layergroup-token'); var step = require('step'); var assert = require('../../../support/assert'); diff --git a/test/acceptance/ported/torque.js b/test/acceptance/ported/torque.js index 84846ae1..b7a8c36f 100644 --- a/test/acceptance/ported/torque.js +++ b/test/acceptance/ported/torque.js @@ -7,7 +7,7 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var BaseController = require('../../../lib/cartodb/controllers/base'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('torque', function() { diff --git a/test/acceptance/ported/torque_boundaries.js b/test/acceptance/ported/torque_boundaries.js index f605570b..fb88be52 100644 --- a/test/acceptance/ported/torque_boundaries.js +++ b/test/acceptance/ported/torque_boundaries.js @@ -5,7 +5,7 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var BaseController = require('../../../lib/cartodb/controllers/base'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('torque boundary points', function() { diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index ae997ca8..e8e0a2ca 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -23,7 +23,7 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); describe('template_api', function() { server.layergroupAffectedTablesCache.cache.reset(); diff --git a/test/acceptance/turbo-cartocss/named-maps.js b/test/acceptance/turbo-cartocss/named-maps.js index 9c88b6db..35dc8cd8 100644 --- a/test/acceptance/turbo-cartocss/named-maps.js +++ b/test/acceptance/turbo-cartocss/named-maps.js @@ -1,6 +1,6 @@ var assert = require('../../support/assert'); var step = require('step'); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); var testHelper = require('../../support/test_helper'); var CartodbWindshaft = require('../../../lib/cartodb/server'); var serverOptions = require('../../../lib/cartodb/server_options'); diff --git a/test/acceptance/turbo-cartocss/regressions.js b/test/acceptance/turbo-cartocss/regressions.js index 4f152da3..e2dd38cc 100644 --- a/test/acceptance/turbo-cartocss/regressions.js +++ b/test/acceptance/turbo-cartocss/regressions.js @@ -102,4 +102,69 @@ describe('turbo-carto regressions', function() { done(); }); }); + + it('should work with mapnik substitution tokens and analyses', function(done) { + var cartocss = [ + "#layer {", + " line-width: 2;", + " line-color: #3B3B58;", + " line-opacity: 1;", + " polygon-opacity: 0.7;", + " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", + "}" + ].join('\n'); + + var sqlWrap = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (<%= sql %>) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + var mapConfig = { + "version": "1.5.0", + "layers": [ + { + "type": 'mapnik', + "options": { + "cartocss_version": '2.3.0', + "source": { + "id": "head" + }, + sql_wrap: sqlWrap, + "cartocss": cartocss + } + } + ], + "analyses": [ + { + "id": "head", + "type": "source", + "params": { + "query": "SELECT * FROM populated_places_simple_reduced" + } + } + ] + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err, err); + + assert.ok(layergroup.hasOwnProperty('layergroupid')); + assert.ok(!layergroup.hasOwnProperty('errors')); + + done(); + }); + }); }); diff --git a/test/acceptance/widgets/named-maps.js b/test/acceptance/widgets/named-maps.js index cc603d72..eb5a0d29 100644 --- a/test/acceptance/widgets/named-maps.js +++ b/test/acceptance/widgets/named-maps.js @@ -10,7 +10,7 @@ var CartodbWindshaft = require('../../../lib/cartodb/server'); var serverOptions = require('../../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); -var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../../support/layergroup-token'); describe('named-maps widgets', function() { diff --git a/test/acceptance/x_cache_channel.js b/test/acceptance/x_cache_channel.js index 8f536e1f..4ebb45e8 100644 --- a/test/acceptance/x_cache_channel.js +++ b/test/acceptance/x_cache_channel.js @@ -8,7 +8,7 @@ var serverOptions = require('../../lib/cartodb/server_options'); var server = new CartodbWindshaft(serverOptions); server.setMaxListeners(0); -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('../support/layergroup-token'); describe('get requests x-cache-channel', function() { diff --git a/test/integration/mapconfig_named_layers_datasource.js b/test/integration/mapconfig_named_layers_datasource.js index e312b7b3..aac8d335 100644 --- a/test/integration/mapconfig_named_layers_datasource.js +++ b/test/integration/mapconfig_named_layers_datasource.js @@ -4,7 +4,7 @@ var assert = require('assert'); var RedisPool = require('redis-mpool'); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); -var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter'); +var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter'); // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); @@ -14,7 +14,7 @@ var templateMaps = new TemplateMaps(redisPool, { max_user_templates: global.environment.maxUserTemplates }); -var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); +var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps, pgConnection); var wadusSql = 'select 1 wadusLayer, null::geometry the_geom_webmercator'; var wadusLayer = { @@ -294,9 +294,11 @@ describe('named_layers datasources', function() { testScenarios.forEach(function(testScenario) { it('should return a list of layers ' + testScenario.desc, function(done) { - mapConfigNamedLayersAdapter.getLayers(username, testScenario.config.layers, pgConnection, - function(err, layers, datasource) { - testScenario.test(err, layers, datasource, done); + var params = {}; + var context = {}; + mapConfigNamedLayersAdapter.getMapConfig(username, testScenario.config, params, context, + function(err, mapConfig) { + testScenario.test(err, mapConfig.layers, context.datasource, done); } ); }); diff --git a/test/integration/mapconfig_named_layers_expanded.js b/test/integration/mapconfig_named_layers_expanded.js index d78726c9..fd542514 100644 --- a/test/integration/mapconfig_named_layers_expanded.js +++ b/test/integration/mapconfig_named_layers_expanded.js @@ -4,9 +4,9 @@ var assert = require('assert'); var RedisPool = require('redis-mpool'); var TemplateMaps = require('../../lib/cartodb/backends/template_maps.js'); var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); -var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter'); +var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-named-layers-adapter'); -describe('mapconfig_named_layers_adapter', function() { +describe('mapconfig-named-layers-adapter', function() { // configure redis pool instance to use in tests var redisPool = new RedisPool(global.environment.redis); @@ -16,7 +16,7 @@ describe('mapconfig_named_layers_adapter', function() { max_user_templates: global.environment.maxUserTemplates }); - var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps, pgConnection); var wadusLayer = { type: 'cartodb', @@ -134,6 +134,8 @@ describe('mapconfig_named_layers_adapter', function() { }; } + var params = {}; + var context = {}; beforeEach(function(done) { templateMaps.addTemplate(username, template, done); @@ -147,11 +149,11 @@ describe('mapconfig_named_layers_adapter', function() { var missingNamedMapLayerConfig = makeNamedMapLayerConfig({ config: {} }); - mapConfigNamedLayersAdapter.getLayers(username, missingNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, missingNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(err); - assert.ok(!layers); - assert.ok(!datasource); + assert.ok(!mapConfig); + assert.ok(!context.datasource); assert.equal(err.message, 'Missing Named Map `name` in layer options'); done(); @@ -164,11 +166,11 @@ describe('mapconfig_named_layers_adapter', function() { var nonExistentNamedMapLayerConfig = makeNamedMapLayerConfig({ name: missingTemplateName }); - mapConfigNamedLayersAdapter.getLayers(username, nonExistentNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, nonExistentNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(err); - assert.ok(!layers); - assert.ok(!datasource); + assert.ok(!mapConfig); + assert.ok(!context.datasource); assert.equal( err.message, "Template '" + missingTemplateName + "' of user '" + username + "' not found" ); @@ -187,11 +189,11 @@ describe('mapconfig_named_layers_adapter', function() { var nonAuthTokensNamedMapLayerConfig = makeNamedMapLayerConfig({ name: tokenAuthTemplateName }); - mapConfigNamedLayersAdapter.getLayers(username, nonAuthTokensNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, nonAuthTokensNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(err); - assert.ok(!layers); - assert.ok(!datasource); + assert.ok(!mapConfig); + assert.ok(!context.datasource); assert.equal(err.message, "Unauthorized '" + tokenAuthTemplateName + "' template instantiation"); templateMaps.delTemplate(username, tokenAuthTemplateName, done); @@ -209,11 +211,11 @@ describe('mapconfig_named_layers_adapter', function() { var nestedNamedMapLayerConfig = makeNamedMapLayerConfig({ name: nestedNamedMapTemplateName }); - mapConfigNamedLayersAdapter.getLayers(username, nestedNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, nestedNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(err); - assert.ok(!layers); - assert.ok(!datasource); + assert.ok(!mapConfig); + assert.ok(!context.datasource); assert.equal(err.message, 'Nested named layers are not allowed'); templateMaps.delTemplate(username, nestedNamedMapTemplateName, done); @@ -226,12 +228,13 @@ describe('mapconfig_named_layers_adapter', function() { var validNamedMapMapLayerConfig = makeNamedMapLayerConfig({ name: templateName }); - mapConfigNamedLayersAdapter.getLayers(username, validNamedMapMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, validNamedMapMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(!err); + var layers = mapConfig.layers; assert.ok(layers.length, 1); assert.ok(layers[0].type, 'cartodb'); - assert.notEqual(datasource.getLayerDatasource(0), undefined); + assert.notEqual(context.datasource.getLayerDatasource(0), undefined); done(); } @@ -248,11 +251,12 @@ describe('mapconfig_named_layers_adapter', function() { name: tokenAuthTemplateName, auth_tokens: ['valid1'] }); - mapConfigNamedLayersAdapter.getLayers(username, validAuthTokensNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, validAuthTokensNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(!err); + var layers = mapConfig.layers; assert.equal(layers.length, 1); - assert.notEqual(datasource.getLayerDatasource(0), undefined); + assert.notEqual(context.datasource.getLayerDatasource(0), undefined); templateMaps.delTemplate(username, tokenAuthTemplateName, done); } @@ -270,18 +274,19 @@ describe('mapconfig_named_layers_adapter', function() { name: multipleLayersTemplateName, auth_tokens: ['valid2'] }); - mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, multipleLayersNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(!err); + var layers = mapConfig.layers; assert.equal(layers.length, 2); assert.equal(layers[0].type, 'mapnik'); assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: green; }'); - assert.notEqual(datasource.getLayerDatasource(0), undefined); + assert.notEqual(context.datasource.getLayerDatasource(0), undefined); assert.equal(layers[1].type, 'cartodb'); assert.equal(layers[1].options.cartocss, '#layer { marker-fill: red; }'); - assert.notEqual(datasource.getLayerDatasource(1), undefined); + assert.notEqual(context.datasource.getLayerDatasource(1), undefined); templateMaps.delTemplate(username, multipleLayersTemplateName, done); } @@ -306,18 +311,19 @@ describe('mapconfig_named_layers_adapter', function() { }, auth_tokens: ['valid2'] }); - mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection, - function(err, layers, datasource) { + mapConfigNamedLayersAdapter.getMapConfig(username, multipleLayersNamedMapLayerConfig, params, context, + function(err, mapConfig) { assert.ok(!err); + var layers = mapConfig.layers; assert.equal(layers.length, 2); assert.equal(layers[0].type, 'mapnik'); assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: ' + polygonColor + '; }'); - assert.notEqual(datasource.getLayerDatasource(0), undefined); + assert.notEqual(context.datasource.getLayerDatasource(0), undefined); assert.equal(layers[1].type, 'cartodb'); assert.equal(layers[1].options.cartocss, '#layer { marker-fill: ' + color + '; }'); - assert.notEqual(datasource.getLayerDatasource(1), undefined); + assert.notEqual(context.datasource.getLayerDatasource(1), undefined); templateMaps.delTemplate(username, multipleLayersTemplateName, done); } diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js index d168c02f..87a87935 100644 --- a/test/integration/mapconfig_overviews_adapter.js +++ b/test/integration/mapconfig_overviews_adapter.js @@ -7,11 +7,7 @@ var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connectio var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api'); -var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter'); - -// configure redis pool instance to use in tests -var redisPool = new RedisPool(global.environment.redis); -var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool })); +var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter'); var redisPool = new RedisPool(global.environment.redis); var metadataBackend = cartodbRedis({pool: redisPool}); @@ -37,8 +33,16 @@ describe('MapConfigOverviewsAdapter', function() { } }; - mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) { + var _mapConfig = { + layers: [layer_without_overviews] + }; + + var params = {}; + var context = {}; + + mapConfigOverviewsAdapter.getMapConfig('localhost', _mapConfig, params, context, function(err, mapConfig) { assert.ok(!err); + var layers = mapConfig.layers; assert.equal(layers.length, 1); assert.equal(layers[0].type, 'cartodb'); assert.equal(layers[0].options.sql, sql); @@ -56,7 +60,7 @@ describe('MapConfigOverviewsAdapter', function() { var sql = 'SELECT * FROM test_table_overviews'; var cartocss = '#layer { marker-fill: black; }'; var cartocss_version = '2.3.0'; - var layer_without_overviews = { + var layer_with_overviews = { type: 'cartodb', options: { sql: sql, @@ -65,8 +69,16 @@ describe('MapConfigOverviewsAdapter', function() { } }; - mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], [], function(err, layers) { + var _mapConfig = { + layers: [layer_with_overviews] + }; + + var params = {}; + var context = {}; + + mapConfigOverviewsAdapter.getMapConfig('localhost', _mapConfig, params, context, function(err, mapConfig) { assert.ok(!err); + var layers = mapConfig.layers; assert.equal(layers.length, 1); assert.equal(layers[0].type, 'cartodb'); assert.equal(layers[0].options.sql, sql); diff --git a/lib/cartodb/models/layergroup_token.js b/test/support/layergroup-token.js similarity index 100% rename from lib/cartodb/models/layergroup_token.js rename to test/support/layergroup-token.js diff --git a/test/support/test-client.js b/test/support/test-client.js index a9ec5bee..e0fadeee 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -6,7 +6,7 @@ var urlParser = require('url'); var mapnik = require('windshaft').mapnik; -var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); +var LayergroupToken = require('./layergroup-token'); var assert = require('./assert'); var helper = require('./test_helper'); diff --git a/test/unit/cartodb/mapconfig/adapter.test.js b/test/unit/cartodb/mapconfig/adapter.test.js new file mode 100644 index 00000000..eeb71896 --- /dev/null +++ b/test/unit/cartodb/mapconfig/adapter.test.js @@ -0,0 +1,93 @@ +//require('../../../support/test_helper'); +var assert = require('assert'); + +var MapConfigAdapter = require('../../../../lib/cartodb/models/mapconfig/adapter'); + +describe('MapConfigAdapter', function() { + var user = 'wadus'; + function requestMapConfig() { + return { + val: 0 + }; + } + function params() { + return {}; + } + function context() { + return {}; + } + + function createAdapter(valOperatorFn) { + return function ValMapConfigAdapter() { + this.getMapConfig = function(user, requestMapConfig, params, context, callback) { + requestMapConfig.val = valOperatorFn(requestMapConfig.val); + return callback(null, requestMapConfig); + }; + }; + } + var IncValMapConfigAdapter = createAdapter(function(val) { return val + 1; }); + var Mul2ValMapConfigAdapter = createAdapter(function(val) { return val * 2; }); + + function validateMapConfig(adapter, expectedNumAdapters, expectedVal, callback) { + assert.equal(adapter.adapters.length, expectedNumAdapters); + adapter.getMapConfig(user, requestMapConfig(), params(), context(), function(err, mapConfig) { + assert.equal(mapConfig.val, expectedVal); + return callback(err); + }); + } + + it('works with no adapters', function(done) { + var adapter = new MapConfigAdapter(); + validateMapConfig(adapter, 0, 0, done); + }); + + it('works with no adapters as empty array', function(done) { + var adapter = new MapConfigAdapter([]); + validateMapConfig(adapter, 0, 0, done); + }); + + it('works with basic adapter', function(done) { + var adapter = new MapConfigAdapter(new IncValMapConfigAdapter()); + validateMapConfig(adapter, 1, 1, done); + }); + + it('works with basic adapter as array', function(done) { + var adapter = new MapConfigAdapter([new IncValMapConfigAdapter()]); + validateMapConfig(adapter, 1, 1, done); + }); + + it('works with several adapters', function(done) { + var adapter = new MapConfigAdapter(new IncValMapConfigAdapter(), new IncValMapConfigAdapter()); + validateMapConfig(adapter, 2, 2, done); + }); + + it('works with several adapters as array', function(done) { + var adapter = new MapConfigAdapter([new IncValMapConfigAdapter(), new IncValMapConfigAdapter()]); + validateMapConfig(adapter, 2, 2, done); + }); + + it('should execute in order 1', function(done) { + var adapter = new MapConfigAdapter([new Mul2ValMapConfigAdapter(), new IncValMapConfigAdapter()]); + validateMapConfig(adapter, 2, 1, done); + }); + + it('should execute in order 2', function(done) { + var adapter = new MapConfigAdapter([new IncValMapConfigAdapter(), new Mul2ValMapConfigAdapter()]); + validateMapConfig(adapter, 2, 2, done); + }); + + it('should execute in order 3', function(done) { + var adapter = new MapConfigAdapter([new Mul2ValMapConfigAdapter(), new Mul2ValMapConfigAdapter()]); + validateMapConfig(adapter, 2, 0, done); + }); + + it('should execute in order 4', function(done) { + var Mul5ValMapConfigAdapter = createAdapter(function(val) { return val * 5; }); + var adapter = new MapConfigAdapter( + new IncValMapConfigAdapter(), + new Mul2ValMapConfigAdapter(), + new Mul5ValMapConfigAdapter() + ); + validateMapConfig(adapter, 3, 10, done); + }); +}); diff --git a/test/unit/cartodb/substitution-tokens.test.js b/test/unit/cartodb/substitution-tokens.test.js new file mode 100644 index 00000000..70c3a0d0 --- /dev/null +++ b/test/unit/cartodb/substitution-tokens.test.js @@ -0,0 +1,40 @@ +var assert = require('assert'); +var SubstitutionTokens = require('../../../lib/cartodb/utils/substitution-tokens'); + +describe('SubstitutionTokens', function() { + + var sql = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (select * from table) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + it('should return tokens present in sql', function() { + assert.deepEqual(SubstitutionTokens.tokens(sql), ['bbox', 'scale_denominator', 'pixel_width', 'pixel_height']); + }); + + it('should return just one token', function() { + assert.deepEqual(SubstitutionTokens.tokens('select !bbox! from wadus'), ['bbox']); + }); + + it('should not return other tokens', function() { + assert.deepEqual(SubstitutionTokens.tokens('select !wadus! from wadus'), []); + }); + + it('should report sql has tokens', function() { + assert.equal(SubstitutionTokens.hasTokens(sql), true); + assert.equal(SubstitutionTokens.hasTokens('select !bbox! from wadus'), true); + assert.equal(SubstitutionTokens.hasTokens('select !wadus! from wadus'), false); + }); +});