diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 8a1cc7ad..480f15b8 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -191,6 +191,22 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/tmp/cdb-tiler-dev/millstone-dev' diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 73c68f1d..2a457768 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -185,6 +185,22 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/home/ubuntu/tile_assets/' diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index f81b88f7..72195fef 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -185,6 +185,22 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: false, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/home/ubuntu/tile_assets/' diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 0400be6e..00c19628 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -186,6 +186,22 @@ var config = { } } } + // anything analyses related + ,analysis: { + // batch configuration + batch: { + // Inline execution avoid the use of SQL API as batch endpoint + // When set to true it will run all analysis queries in series, with a direct connection to the DB + // This might be useful for: + // - testing + // - running an standalone server without any dependency on external services + inlineExecution: true, + // where the SQL API is running, it will use a custom Host header to specify the username. + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + // the template to use for adding the host header in the batch api requests + hostHeaderTemplate: '{{=it.username}}.localhost.lan' + } + } ,millstone: { // Needs to be writable by server user cache_basedir: '/tmp/cdb-tiler-test/millstone' diff --git a/lib/cartodb/backends/analysis-status.js b/lib/cartodb/backends/analysis-status.js new file mode 100644 index 00000000..211157d8 --- /dev/null +++ b/lib/cartodb/backends/analysis-status.js @@ -0,0 +1,49 @@ +var PSQL = require('cartodb-psql'); + +function AnalysisStatusBackend() { +} + +module.exports = AnalysisStatusBackend; + + +AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) { + var nodeId = params.nodeId; + + var statusQuery = 'SELECT node_id, status, updated_at FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\''; + + var pg = new PSQL(dbParamsFromReqParams(params)); + pg.query(statusQuery, function(err, result) { + if (err) { + return callback(err, result); + } + + result = result || {}; + + var rows = result.rows || []; + + return callback(null, rows[0] || { + node_id: nodeId, + status: 'unknown' + }); + }, true); // use read-only transaction +}; + +function dbParamsFromReqParams(params) { + var dbParams = {}; + if ( params.dbuser ) { + dbParams.user = params.dbuser; + } + if ( params.dbpassword ) { + dbParams.pass = params.dbpassword; + } + if ( params.dbhost ) { + dbParams.host = params.dbhost; + } + if ( params.dbport ) { + dbParams.port = params.dbport; + } + if ( params.dbname ) { + dbParams.dbname = params.dbname; + } + return dbParams; +} diff --git a/lib/cartodb/backends/analysis.js b/lib/cartodb/backends/analysis.js new file mode 100644 index 00000000..5855da46 --- /dev/null +++ b/lib/cartodb/backends/analysis.js @@ -0,0 +1,19 @@ +var camshaft = require('camshaft'); + +function AnalysisBackend(options) { + var batchConfig = options.batch || {}; + batchConfig.endpoint = batchConfig.endpoint || 'http://127.0.0.1:8080/api/v1/sql/job'; + batchConfig.inlineExecution = batchConfig.inlineExecution || false; + batchConfig.hostHeaderTemplate = batchConfig.hostHeaderTemplate || '{{=it.username}}.localhost.lan'; + this.batchConfig = batchConfig; +} + +module.exports = AnalysisBackend; + +AnalysisBackend.prototype.create = function(analysisConfiguration, analysisDefinition, callback) { + analysisConfiguration.batch.endpoint = this.batchConfig.endpoint; + analysisConfiguration.batch.inlineExecution = this.batchConfig.inlineExecution; + analysisConfiguration.batch.hostHeaderTemplate = this.batchConfig.hostHeaderTemplate; + + camshaft.create(analysisConfiguration, analysisDefinition, callback); +}; diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js new file mode 100644 index 00000000..510709c4 --- /dev/null +++ b/lib/cartodb/backends/dataview.js @@ -0,0 +1,280 @@ +var assert = require('assert'); + +var _ = require('underscore'); +var PSQL = require('cartodb-psql'); +var camshaft = require('camshaft'); +var step = require('step'); + +var Timer = require('../stats/timer'); + +var BBoxFilter = require('../models/filter/bbox'); +var DataviewFactory = require('../models/dataview/factory'); + +function DataviewBackend(analysisBackend) { + this.analysisBackend = analysisBackend; +} + +module.exports = DataviewBackend; + + +DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { + var self = this; + + var timer = new Timer(); + + var dataviewName = params.dataviewName; + + var mapConfig; + var dataviewDefinition; + step( + function getMapConfig() { + mapConfigProvider.getMapConfig(this); + }, + function _getDataviewDefinition(err, _mapConfig) { + assert.ifError(err); + + mapConfig = _mapConfig; + + var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!_dataviewDefinition) { + throw new Error("Dataview '" + dataviewName + "' does not exists"); + } + + dataviewDefinition = _dataviewDefinition; + + return dataviewDefinition; + }, + function loadAnalysis(err) { + assert.ifError(err); + + var analysisConfiguration = { + db: { + host: params.dbhost, + port: params.dbport, + dbname: params.dbname, + user: params.dbuser, + pass: params.dbpassword + }, + batch: { + username: user, + apiKey: params.api_key + } + }; + + var sourceId = dataviewDefinition.source.id; + var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId); + + var next = this; + + self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) { + if (err) { + return next(err); + } + + var sourceId2Node = {}; + var rootNode = analysis.getRoot(); + if (rootNode.params && rootNode.params.id) { + sourceId2Node[rootNode.params.id] = rootNode; + } + + analysis.getSortedNodes().forEach(function(node) { + if (node.params && node.params.id) { + sourceId2Node[node.params.id] = node; + } + }); + + var node = sourceId2Node[sourceId]; + + if (!node) { + return next(new Error('Analysis node not found for dataview')); + } + + return next(null, node); + }); + }, + function runDataviewQuery(err, node) { + assert.ifError(err); + + var pg = new PSQL(dbParamsFromReqParams(params)); + + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query; + if (ownFilter) { + query = node.getQuery(); + } else { + var applyFilters = {}; + applyFilters[dataviewName] = false; + query = node.getQuery(applyFilters); + } + + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'), + function castNumbers(overrides, val, k) { + overrides[k] = Number.isFinite(+val) ? +val : val; + return overrides; + }, + {ownFilter: ownFilter} + ); + + var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + dataview.getResult(pg, overrideParams, this); + }, + function returnCallback(err, result) { + return callback(err, result, timer.getTimes()); + } + ); +}; + +DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) { + var self = this; + + var timer = new Timer(); + + var dataviewName = params.dataviewName; + + var mapConfig; + var dataviewDefinition; + step( + function getMapConfig() { + mapConfigProvider.getMapConfig(this); + }, + function _getDataviewDefinition(err, _mapConfig) { + assert.ifError(err); + + mapConfig = _mapConfig; + + var _dataviewDefinition = getDataviewDefinition(mapConfig.obj(), dataviewName); + if (!_dataviewDefinition) { + throw new Error("Dataview '" + dataviewName + "' does not exists"); + } + + dataviewDefinition = _dataviewDefinition; + + return dataviewDefinition; + }, + function loadAnalysis(err) { + assert.ifError(err); + + var analysisConfiguration = { + db: { + host: params.dbhost, + port: params.dbport, + dbname: params.dbname, + user: params.dbuser, + pass: params.dbpassword + }, + batch: { + // TODO load this from configuration + endpoint: 'http://127.0.0.1:8080/api/v1/sql/job', + username: user, + apiKey: params.api_key + } + }; + + var sourceId = dataviewDefinition.source.id; + var analysisDefinition = getAnalysisDefinition(mapConfig.obj().analyses, sourceId); + + var next = this; + + self.analysisBackend.create(analysisConfiguration, analysisDefinition, function(err, analysis) { + if (err) { + return next(err); + } + + var sourceId2Node = {}; + var rootNode = analysis.getRoot(); + if (rootNode.params && rootNode.params.id) { + sourceId2Node[rootNode.params.id] = rootNode; + } + + analysis.getSortedNodes().forEach(function(node) { + if (node.params && node.params.id) { + sourceId2Node[node.params.id] = node; + } + }); + + var node = sourceId2Node[sourceId]; + + if (!node) { + return next(new Error('Analysis node not found for dataview')); + } + + return next(null, node); + }); + }, + function runDataviewQuery(err, node) { + assert.ifError(err); + + var pg = new PSQL(dbParamsFromReqParams(params)); + + var ownFilter = +params.own_filter; + ownFilter = !!ownFilter; + + var query; + if (ownFilter) { + query = node.getQuery(); + } else { + var applyFilters = {}; + applyFilters[dataviewName] = false; + query = node.getQuery(applyFilters); + } + + if (params.bbox) { + var bboxFilter = new BBoxFilter({column: 'the_geom', srid: 4326}, {bbox: params.bbox}); + query = bboxFilter.sql(query); + } + + var userQuery = params.q; + + var dataview = DataviewFactory.getDataview(query, dataviewDefinition); + dataview.search(pg, userQuery, this); + }, + function returnCallback(err, result) { + return callback(err, result, timer.getTimes()); + } + ); +}; + +function getAnalysisDefinition(mapConfigAnalyses, sourceId) { + mapConfigAnalyses = mapConfigAnalyses || []; + for (var i = 0; i < mapConfigAnalyses.length; i++) { + var analysisGraph = new camshaft.reference.AnalysisGraph(mapConfigAnalyses[i]); + var nodes = analysisGraph.getNodesWithId(); + if (nodes.hasOwnProperty(sourceId)) { + return mapConfigAnalyses[i]; + } + } + throw new Error('There is no associated analysis for the dataview source id'); +} + +function getDataviewDefinition(mapConfig, dataviewName) { + var dataviews = mapConfig.dataviews || {}; + return dataviews[dataviewName]; +} + +function dbParamsFromReqParams(params) { + var dbParams = {}; + if ( params.dbuser ) { + dbParams.user = params.dbuser; + } + if ( params.dbpassword ) { + dbParams.pass = params.dbpassword; + } + if ( params.dbhost ) { + dbParams.host = params.dbhost; + } + if ( params.dbport ) { + dbParams.port = params.dbport; + } + if ( params.dbname ) { + dbParams.dbname = params.dbname; + } + return dbParams; +} \ No newline at end of file diff --git a/lib/cartodb/cache/named_map_provider_cache.js b/lib/cartodb/cache/named_map_provider_cache.js index df536af5..301f21ce 100644 --- a/lib/cartodb/cache/named_map_provider_cache.js +++ b/lib/cartodb/cache/named_map_provider_cache.js @@ -2,17 +2,21 @@ var _ = require('underscore'); var dot = require('dot'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); +var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); var templateName = require('../backends/template_maps').templateName; var queue = require('queue-async'); var LruCache = require("lru-cache"); -function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, overviewsAdapter, turboCartocssAdapter) { +function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, overviewsAdapter, + turboCartocssAdapter) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(); this.overviewsAdapter = overviewsAdapter; this.turboCartocssAdapter = turboCartocssAdapter; @@ -30,10 +34,12 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok namedMapProviders[providerKey] = new NamedMapMapConfigProvider( this.templateMaps, this.pgConnection, + this.metadataBackend, this.userLimitsApi, this.namedLayersAdapter, this.overviewsAdapter, this.turboCartocssAdapter, + this.analysisMapConfigAdapter, user, templateId, config, diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index b6b4d640..99eca0c5 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -7,6 +7,9 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); +var DataviewBackend = require('../backends/dataview'); +var AnalysisStatusBackend = require('../backends/analysis-status'); + var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider'); var QueryTables = require('cartodb-query-tables'); @@ -22,10 +25,11 @@ var QueryTables = require('cartodb-query-tables'); * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {AnalysisBackend} analysisBackend * @constructor */ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { + widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) { BaseController.call(this, authApi, pgConnection); this.pgConnection = pgConnection; @@ -37,6 +41,9 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; + + this.dataviewBackend = new DataviewBackend(analysisBackend); + this.analysisStatusBackend = new AnalysisStatusBackend(); } util.inherits(LayergroupController, BaseController); @@ -78,6 +85,97 @@ LayergroupController.prototype.register = function(app) { app.get(app.base_url_mapconfig + '/:token/:layer/widget/:widgetName/search', cors(), userMiddleware, this.widgetSearch.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/dataview/:dataviewName', cors(), userMiddleware, + this.dataview.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/dataview/:dataviewName/search', cors(), userMiddleware, + this.dataviewSearch.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/analysis/node/:nodeId', cors(), userMiddleware, + this.analysisNodeStatus.bind(this)); +}; + +LayergroupController.prototype.analysisNodeStatus = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function retrieveNodeStatus(err) { + assert.ifError(err); + self.analysisStatusBackend.getNodeStatus(req.params, this); + }, + function finish(err, nodeStatus, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET NODE STATUS'); + } else { + self.sendResponse(req, res, nodeStatus, 200); + } + } + ); +}; + +LayergroupController.prototype.dataview = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function retrieveDataview(err) { + assert.ifError(err); + + var mapConfigProvider = new MapStoreMapConfigProvider( + self.mapStore, req.context.user, self.userLimitsApi, req.params + ); + self.dataviewBackend.getDataview(mapConfigProvider, req.context.user, req.params, this); + }, + function finish(err, dataview, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET DATAVIEW'); + } else { + self.sendResponse(req, res, dataview, 200); + } + } + ); + +}; + +LayergroupController.prototype.dataviewSearch = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function searchDataview(err) { + assert.ifError(err); + + var mapConfigProvider = new MapStoreMapConfigProvider( + self.mapStore, req.context.user, self.userLimitsApi, req.params + ); + self.dataviewBackend.search(mapConfigProvider, req.context.user, req.params, this); + }, + function finish(err, searchResult, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET DATAVIEW SEARCH'); + } else { + self.sendResponse(req, res, searchResult, 200); + } + } + ); + }; LayergroupController.prototype.widget = function(req, res) { @@ -95,13 +193,13 @@ LayergroupController.prototype.widget = function(req, res) { ); self.widgetBackend.getWidget(mapConfigProvider, req.params, this); }, - function finish(err, tile, stats) { + function finish(err, widget, stats) { req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'GET WIDGET'); } else { - self.sendResponse(req, res, tile, 200); + self.sendResponse(req, res, widget, 200); } } ); @@ -123,13 +221,13 @@ LayergroupController.prototype.widgetSearch = function(req, res) { ); self.widgetBackend.search(mapConfigProvider, req.params, this); }, - function finish(err, tile, stats) { + function finish(err, searchResult, stats) { req.profiler.add(stats || {}); if (err) { self.sendError(req, res, err, 'GET WIDGET'); } else { - self.sendResponse(req, res, tile, 200); + self.sendResponse(req, res, searchResult, 200); } } ); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 07e0eea5..aed91641 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -16,6 +16,7 @@ var Datasource = windshaft.model.Datasource; var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); +var AnalysisMapConfigAdapter = require('../models/analysis-mapconfig-adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); @@ -25,15 +26,17 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye * @param {TemplateMaps} templateMaps * @param {MapBackend} mapBackend * @param metadataBackend - * @param {OverviewsMetadataApi} overviewsMetadataApi * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables + * @param {MapConfigOverviewsAdapter} overviewsAdapter + * @param {TurboCartocssAdapter} turboCartoCssAdapter + * @param {AnalysisBackend} analysisBackend * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, - overviewsAdapter, turboCartoCssAdapter) { + overviewsAdapter, turboCartoCssAdapter, analysisBackend) { BaseController.call(this, authApi, pgConnection); @@ -46,6 +49,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.layergroupAffectedTables = layergroupAffectedTables; this.turboCartoCssAdapter = turboCartoCssAdapter; + this.analysisMapConfigAdapter = new AnalysisMapConfigAdapter(analysisBackend); this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); this.overviewsAdapter = overviewsAdapter; } @@ -128,15 +132,43 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { var self = this; var mapConfig; + var analysesResults = []; step( function setupParams(){ self.req2params(req, this); }, prepareConfigFn, - function beforeLayergroupCreate(err, requestMapConfig) { + function prepareAnalysisLayers(err, requestMapConfig) { + assert.ifError(err); + var analysisConfiguration = { + db: { + host: req.params.dbhost, + port: req.params.dbport, + dbname: req.params.dbname, + user: req.params.dbuser, + pass: req.params.dbpassword + }, + batch: { + username: req.context.user, + apiKey: req.params.api_key + } + }; + + var filters = {}; + if (req.params.filters) { + try { + filters = JSON.parse(req.params.filters); + } catch (e) { + // ignore + } + } + self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, filters, this); + }, + function beforeLayergroupCreate(err, requestMapConfig, _analysesResults) { assert.ifError(err); var next = this; + analysesResults = _analysesResults; self.namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, self.pgConnection, function(err, layers, datasource) { if (err) { @@ -192,7 +224,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, analysesResults, layergroup, this); }, function finish(err, layergroup) { if (err) { @@ -227,10 +259,12 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn mapConfigProvider = new NamedMapMapConfigProvider( self.templateMaps, self.pgConnection, + self.metadataBackend, self.userLimitsApi, self.namedLayersAdapter, self.overviewsAdapter, self.turboCartoCssAdapter, + self.analysisMapConfigAdapter, cdbuser, req.params.template_id, templateParams, @@ -250,7 +284,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, this); + self.afterLayergroupCreate(req, res, mapConfig, [], layergroup, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -271,7 +305,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }; -MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, callback) { +MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, analysesResults, layergroup, callback) { var self = this; var username = req.context.user; @@ -336,6 +370,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la // TODO this should take into account several URL patterns addWidgetsUrl(username, layergroup); + addDataviewsUrls(username, layergroup, mapconfig.obj()); + addAnalysesMetadata(username, layergroup, analysesResults); if (req.method === 'GET') { var ttl = global.environment.varnish.layergroupTtl || 86400; res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); @@ -354,6 +390,41 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la ); }; +function addAnalysesMetadata(username, layergroup, analysesResults) { + analysesResults = analysesResults || []; + layergroup.metadata.analyses = layergroup.metadata.analyses || []; + + analysesResults.forEach(function(analysis) { + var nodes = analysis.getSortedNodes(); + layergroup.metadata.analyses.push({ + nodes: nodes.reduce(function(nodesIdMap, node) { + if (node.params.id) { + var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); + nodesIdMap[node.params.id] = { + status: node.getStatus(), + query: node.getQuery(), + url: getUrls(username, nodeResource) + }; + } + + return nodesIdMap; + }, {}) + }); + }); +} + +function addDataviewsUrls(username, layergroup, mapConfig) { + layergroup.metadata.dataviews = layergroup.metadata.dataviews || {}; + var dataviews = mapConfig.dataviews || {}; + + Object.keys(dataviews).forEach(function(dataviewName) { + var resource = layergroup.layergroupid + '/dataview/' + dataviewName; + layergroup.metadata.dataviews[dataviewName] = { + url: getUrls(username, resource) + }; + }); +} + function addWidgetsUrl(username, layergroup) { if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) { diff --git a/lib/cartodb/models/analysis-mapconfig-adapter.js b/lib/cartodb/models/analysis-mapconfig-adapter.js new file mode 100644 index 00000000..3c5761c6 --- /dev/null +++ b/lib/cartodb/models/analysis-mapconfig-adapter.js @@ -0,0 +1,142 @@ +var queue = require('queue-async'); +var debug = require('debug')('windshaft:analysis'); + +var camshaft = require('camshaft'); +var dot = require('dot'); +dot.templateSettings.strip = false; + +function AnalysisMapConfigAdapter(analysisBackend) { + this.analysisBackend = analysisBackend; +} + +module.exports = AnalysisMapConfigAdapter; + +var SKIP_COLUMNS = { + 'the_geom': true, + 'the_geom_webmercator': true +}; + +function skipColumns(columnNames) { + return columnNames + .filter(function(columnName) { return !SKIP_COLUMNS[columnName]; }); +} + +var layerQueryTemplate = dot.template([ + 'SELECT {{=it._columns}}', + 'FROM ({{=it._query}}) _cdb_analysis_query' +].join('\n')); + +function layerQuery(query, columnNames) { + var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(columnNames)); + return layerQueryTemplate({ _query: query, _columns: _columns.join(', ') }); +} + +function appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId) { + var analyses = requestMapConfig.analyses || []; + + requestMapConfig.analyses = analyses.map(function(analysisDefinition) { + var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition); + var definition = analysisDefinition; + Object.keys(dataviewsFiltersBySourceId).forEach(function(sourceId) { + definition = analysisGraph.getDefinitionWith(sourceId, {filters: dataviewsFiltersBySourceId[sourceId] }); + }); + + return definition; + }); + + return requestMapConfig; +} + +function shouldAdaptLayers(requestMapConfig) { + return Array.isArray(requestMapConfig.layers) && + Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0; +} + +var DATAVIEW_TYPE_2_FILTER_TYPE = { + aggregation: 'category', + histogram: 'range' +}; +function getFilter(dataview, params) { + var type = dataview.type; + + return { + type: DATAVIEW_TYPE_2_FILTER_TYPE[type], + column: dataview.options.column, + params: params + }; +} + +AnalysisMapConfigAdapter.prototype.getLayers = function(analysisConfiguration, requestMapConfig, filters, callback) { + // jshint maxcomplexity:7 + var self = this; + filters = filters || {}; + + if (!shouldAdaptLayers(requestMapConfig)) { + return callback(null, requestMapConfig); + } + + var dataviewsFilters = filters.dataviews || {}; + debug(dataviewsFilters); + var dataviews = requestMapConfig.dataviews || {}; + + var dataviewsFiltersBySourceId = Object.keys(dataviewsFilters).reduce(function(bySourceId, dataviewName) { + var dataview = dataviews[dataviewName]; + if (dataview) { + var sourceId = dataview.source.id; + if (!bySourceId.hasOwnProperty(sourceId)) { + bySourceId[sourceId] = {}; + } + + bySourceId[sourceId][dataviewName] = getFilter(dataview, dataviewsFilters[dataviewName]); + } + return bySourceId; + }, {}); + + debug(dataviewsFiltersBySourceId); + + debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4)); + + requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); + + function createAnalysis(analysisDefinition, done) { + self.analysisBackend.create(analysisConfiguration, analysisDefinition, done); + } + + var analysesQueue = queue(requestMapConfig.analyses.length); + requestMapConfig.analyses.forEach(function(analysis) { + analysesQueue.defer(createAnalysis, analysis); + }); + + analysesQueue.awaitAll(function(err, analysesResults) { + if (err) { + return callback(err); + } + + var sourceId2Node = analysesResults.reduce(function(sourceId2Query, analysis) { + var rootNode = analysis.getRoot(); + if (rootNode.params && rootNode.params.id) { + sourceId2Query[rootNode.params.id] = rootNode; + } + + analysis.getSortedNodes().forEach(function(node) { + if (node.params && node.params.id) { + sourceId2Query[node.params.id] = node; + } + }); + + return sourceId2Query; + }, {}); + + requestMapConfig.layers = requestMapConfig.layers.map(function(layer) { + if (layer.options.source && layer.options.source.id) { + var layerNode = sourceId2Node[layer.options.source.id]; + layer.options.sql = layerQuery(layerNode.getQuery(), layerNode.getColumns()); + } + return layer; + }); + + debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); + + return callback(null, requestMapConfig, analysesResults); + }); +}; diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js new file mode 100644 index 00000000..f7a49210 --- /dev/null +++ b/lib/cartodb/models/dataview/aggregation.js @@ -0,0 +1,278 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:widget:aggregation'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var summaryQueryTpl = dot.template([ + 'summary AS (', + ' SELECT', + ' count(1) AS count,', + ' sum(CASE WHEN {{=it._column}} IS NULL THEN 1 ELSE 0 END) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_aggregation_nulls', + ')' +].join('\n')); + +var rankedCategoriesQueryTpl = dot.template([ + 'categories AS(', + ' SELECT {{=it._column}} AS category, {{=it._aggregation}} AS value,', + ' row_number() OVER (ORDER BY {{=it._aggregation}} desc) as rank', + ' FROM ({{=it._query}}) _cdb_aggregation_all', + ' GROUP BY {{=it._column}}', + ' ORDER BY 2 DESC', + ')' +].join('\n')); + +var categoriesSummaryQueryTpl = dot.template([ + 'categories_summary AS(', + ' SELECT count(1) categories_count, max(value) max_val, min(value) min_val', + ' FROM categories', + ')' +].join('\n')); + +var rankedAggregationQueryTpl = dot.template([ + 'SELECT CAST(category AS text), value, false as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank < {{=it._limit}}', + 'UNION ALL', + 'SELECT \'Other\' category, sum(value), true as agg, nulls_count, min_val, max_val, count, categories_count', + ' FROM categories, summary, categories_summary', + ' WHERE rank >= {{=it._limit}}', + 'GROUP BY nulls_count, min_val, max_val, count, categories_count' +].join('\n')); + +var aggregationQueryTpl = dot.template([ + 'SELECT CAST({{=it._column}} AS text) AS category, {{=it._aggregation}} AS value, false as agg,', + ' nulls_count, min_val, max_val, count, categories_count', + 'FROM ({{=it._query}}) _cdb_aggregation_all, summary, categories_summary', + 'GROUP BY category, nulls_count, min_val, max_val, count, categories_count', + 'ORDER BY value DESC' +].join('\n')); + +var CATEGORIES_LIMIT = 6; + +var VALID_OPERATIONS = { + count: [], + sum: ['aggregationColumn'] +}; + +var TYPE = 'aggregation'; + +/** + { + type: 'aggregation', + options: { + column: 'name', + aggregation: 'count' // it could be, e.g., sum if column is numeric + } + } + */ +function Aggregation(query, options) { + if (!_.isString(options.column)) { + throw new Error('Aggregation expects `column` in widget options'); + } + + if (!_.isString(options.aggregation)) { + throw new Error('Aggregation expects `aggregation` operation in widget options'); + } + + if (!VALID_OPERATIONS[options.aggregation]) { + throw new Error("Aggregation does not support '" + options.aggregation + "' operation"); + } + + var requiredOptions = VALID_OPERATIONS[options.aggregation]; + var missingOptions = _.difference(requiredOptions, Object.keys(options)); + if (missingOptions.length > 0) { + throw new Error( + "Aggregation '" + options.aggregation + "' is missing some options: " + missingOptions.join(',') + ); + } + + BaseWidget.apply(this); + + this.query = query; + this.column = options.column; + this.aggregation = options.aggregation; + this.aggregationColumn = options.aggregationColumn; +} + +Aggregation.prototype = new BaseWidget(); +Aggregation.prototype.constructor = Aggregation; + +module.exports = Aggregation; + +Aggregation.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.query; + + var aggregationSql; + if (!!override.ownFilter) { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + aggregationQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql(), + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } else { + aggregationSql = [ + "WITH", + [ + summaryQueryTpl({ + _query: _query, + _column: this.column + }), + rankedCategoriesQueryTpl({ + _query: _query, + _column: this.column, + _aggregation: this.getAggregationSql() + }), + categoriesSummaryQueryTpl({ + _query: _query, + _column: this.column + }) + ].join(',\n'), + rankedAggregationQueryTpl({ + _query: _query, + _column: this.column, + _limit: CATEGORIES_LIMIT + }) + ].join('\n'); + } + + debug(aggregationSql); + + return callback(null, aggregationSql); +}; + +var aggregationFnQueryTpl = dot.template('{{=it._aggregationFn}}({{=it._aggregationColumn}})'); +Aggregation.prototype.getAggregationSql = function() { + return aggregationFnQueryTpl({ + _aggregationFn: this.aggregation, + _aggregationColumn: this.aggregationColumn || 1 + }); +}; + +Aggregation.prototype.format = function(result) { + var categories = []; + var count = 0; + var nulls = 0; + var minValue = 0; + var maxValue = 0; + var categoriesCount = 0; + + + if (result.rows.length) { + var firstRow = result.rows[0]; + count = firstRow.count; + nulls = firstRow.nulls_count; + minValue = firstRow.min_val; + maxValue = firstRow.max_val; + categoriesCount = firstRow.categories_count; + + result.rows.forEach(function(row) { + categories.push(_.omit(row, 'count', 'nulls_count', 'min_val', 'max_val', 'categories_count')); + }); + } + + return { + count: count, + nulls: nulls, + min: minValue, + max: maxValue, + categoriesCount: categoriesCount, + categories: categories + }; +}; + +var filterCategoriesQueryTpl = dot.template([ + 'SELECT {{=it._column}} AS category, {{=it._value}} AS value', + 'FROM ({{=it._query}}) _cdb_aggregation_search', + 'WHERE CAST({{=it._column}} as text) ILIKE {{=it._userQuery}}', + 'GROUP BY {{=it._column}}' +].join('\n')); + +var searchQueryTpl = dot.template([ + 'WITH', + 'search_unfiltered AS (', + ' {{=it._searchUnfiltered}}', + '),', + 'search_filtered AS (', + ' {{=it._searchFiltered}}', + '),', + 'search_union AS (', + ' SELECT * FROM search_unfiltered', + ' UNION ALL', + ' SELECT * FROM search_filtered', + ')', + 'SELECT category, sum(value) AS value', + 'FROM search_union', + 'GROUP BY category', + 'ORDER BY value desc' +].join('\n')); + + +Aggregation.prototype.search = function(psql, userQuery, callback) { + var self = this; + + var _userQuery = psql.escapeLiteral('%' + userQuery + '%'); + + // TODO unfiltered will be wrong as filters are already applied at this point + var query = searchQueryTpl({ + _searchUnfiltered: filterCategoriesQueryTpl({ + _query: this.query, + _column: this.column, + _value: '0', + _userQuery: _userQuery + }), + _searchFiltered: filterCategoriesQueryTpl({ + _query: this.query, + _column: this.column, + _value: 'count(1)', + _userQuery: _userQuery + }) + }); + + psql.query(query, function(err, result) { + if (err) { + return callback(err, result); + } + + return callback(null, {type: self.getType(), categories: result.rows }); + }, true); // use read-only transaction +}; + +Aggregation.prototype.getType = function() { + return TYPE; +}; + +Aggregation.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _aggregation: this.aggregation + }); +}; diff --git a/lib/cartodb/models/dataview/base.js b/lib/cartodb/models/dataview/base.js new file mode 100644 index 00000000..b2e2f188 --- /dev/null +++ b/lib/cartodb/models/dataview/base.js @@ -0,0 +1,26 @@ +function BaseDataview() {} + +module.exports = BaseDataview; + +BaseDataview.prototype.getResult = function(psql, override, callback) { + var self = this; + this.sql(psql, override, function(err, query) { + psql.query(query, function(err, result) { + + if (err) { + return callback(err, result); + } + + result = self.format(result, override); + result.type = self.getType(); + + return callback(null, result); + + }, true); // use read-only transaction + }); + +}; + +BaseDataview.prototype.search = function(psql, userQuery, callback) { + return callback(null, this.format({ rows: [] })); +}; diff --git a/lib/cartodb/models/dataview/factory.js b/lib/cartodb/models/dataview/factory.js new file mode 100644 index 00000000..464a0d44 --- /dev/null +++ b/lib/cartodb/models/dataview/factory.js @@ -0,0 +1,18 @@ +var dataviews = require('./'); + +var DataviewFactory = { + dataviews: Object.keys(dataviews).reduce(function(allDataviews, dataviewClassName) { + allDataviews[dataviewClassName.toLowerCase()] = dataviews[dataviewClassName]; + return allDataviews; + }, {}), + + getDataview: function(query, dataviewDefinition) { + var type = dataviewDefinition.type; + if (!this.dataviews[type]) { + throw new Error('Invalid dataview type: "' + type + '"'); + } + return new this.dataviews[type](query, dataviewDefinition.options); + } +}; + +module.exports = DataviewFactory; diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js new file mode 100644 index 00000000..c3c176dd --- /dev/null +++ b/lib/cartodb/models/dataview/formula.js @@ -0,0 +1,104 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:widget:formula'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var formulaQueryTpl = dot.template([ + 'SELECT', + '{{=it._operation}}({{=it._column}}) AS result,', + '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', + 'FROM ({{=it._query}}) _cdb_formula' +].join('\n')); + +var VALID_OPERATIONS = { + count: true, + avg: true, + sum: true, + min: true, + max: true +}; + +var TYPE = 'formula'; + +/** + { + type: 'formula', + options: { + column: 'name', + operation: 'count' // count, sum, avg + } + } + */ +function Formula(query, options) { + if (!_.isString(options.operation)) { + throw new Error('Formula expects `operation` in widget options'); + } + + if (!VALID_OPERATIONS[options.operation]) { + throw new Error("Formula does not support '" + options.operation + "' operation"); + } + + if (options.operation !== 'count' && !_.isString(options.column)) { + throw new Error('Formula expects `column` in widget options'); + } + + BaseWidget.apply(this); + + this.query = query; + this.column = options.column || '1'; + this.operation = options.operation; +} + +Formula.prototype = new BaseWidget(); +Formula.prototype.constructor = Formula; + +module.exports = Formula; + +Formula.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var _query = this.query; + var formulaSql = formulaQueryTpl({ + _query: _query, + _operation: this.operation, + _column: this.column + }); + + debug(formulaSql); + + return callback(null, formulaSql); +}; + +Formula.prototype.format = function(result) { + var formattedResult = { + operation: this.operation, + result: 0, + nulls: 0 + }; + + if (result.rows.length) { + formattedResult.operation = this.operation; + formattedResult.result = result.rows[0].result; + formattedResult.nulls = result.rows[0].nulls_count; + } + + return formattedResult; +}; + +Formula.prototype.getType = function() { + return TYPE; +}; + +Formula.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _column: this.column, + _operation: this.operation + }); +}; diff --git a/lib/cartodb/models/dataview/histogram.js b/lib/cartodb/models/dataview/histogram.js new file mode 100644 index 00000000..d5d611b4 --- /dev/null +++ b/lib/cartodb/models/dataview/histogram.js @@ -0,0 +1,297 @@ +var _ = require('underscore'); +var BaseWidget = require('./base'); +var debug = require('debug')('windshaft:dataview:histogram'); + +var dot = require('dot'); +dot.templateSettings.strip = false; + +var columnTypeQueryTpl = dot.template( + 'SELECT pg_typeof({{=it.column}})::oid FROM ({{=it.query}}) _cdb_histogram_column_type limit 1' +); +var columnCastTpl = dot.template("date_part('epoch', {{=it.column}})"); + +var BIN_MIN_NUMBER = 6; +var BIN_MAX_NUMBER = 48; + +var basicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._column}}) AS max_val, min({{=it._column}}) AS min_val,', + ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join(' \n')); + +var overrideBasicsQueryTpl = dot.template([ + 'basics AS (', + ' SELECT', + ' max({{=it._end}}) AS max_val, min({{=it._start}}) AS min_val,', + ' avg({{=it._column}}) AS avg_val, count(1) AS total_rows', + ' FROM ({{=it._query}}) _cdb_basics', + ')' +].join('\n')); + +var iqrQueryTpl = dot.template([ + 'iqrange AS (', + ' SELECT max(quartile_max) - min(quartile_max) AS iqr', + ' FROM (', + ' SELECT quartile, max(_cdb_iqr_column) AS quartile_max from (', + ' SELECT {{=it._column}} AS _cdb_iqr_column, ntile(4) over (order by {{=it._column}}', + ' ) AS quartile', + ' FROM ({{=it._query}}) _cdb_rank) _cdb_quartiles', + ' WHERE quartile = 1 or quartile = 3', + ' GROUP BY quartile', + ' ) _cdb_iqr', + ')' +].join('\n')); + +var binsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT CASE WHEN total_rows = 0 OR iqr = 0', + ' THEN 1', + ' ELSE GREATEST(', + ' LEAST({{=it._minBins}}, CAST(total_rows AS INT)),', + ' LEAST(', + ' CAST(((max_val - min_val) / (2 * iqr * power(total_rows, 1/3))) AS INT),', + ' {{=it._maxBins}}', + ' )', + ' )', + ' END AS bins_number', + ' FROM basics, iqrange, ({{=it._query}}) _cdb_bins', + ' LIMIT 1', + ')' +].join('\n')); + +var overrideBinsQueryTpl = dot.template([ + 'bins AS (', + ' SELECT {{=it._bins}} AS bins_number', + ')' +].join('\n')); + +var nullsQueryTpl = dot.template([ + 'nulls AS (', + ' SELECT', + ' count(*) AS nulls_count', + ' FROM ({{=it._query}}) _cdb_histogram_nulls', + ' WHERE {{=it._column}} IS NULL', + ')' +].join('\n')); + +var histogramQueryTpl = dot.template([ + 'SELECT', + ' (max_val - min_val) / cast(bins_number as float) AS bin_width,', + ' bins_number,', + ' nulls_count,', + ' avg_val,', + ' CASE WHEN min_val = max_val', + ' THEN 0', + ' ELSE GREATEST(1, LEAST(WIDTH_BUCKET({{=it._column}}, min_val, max_val, bins_number), bins_number)) - 1', + ' END AS bin,', + ' min({{=it._column}})::numeric AS min,', + ' max({{=it._column}})::numeric AS max,', + ' avg({{=it._column}})::numeric AS avg,', + ' count(*) AS freq', + 'FROM ({{=it._query}}) _cdb_histogram, basics, nulls, bins', + 'WHERE {{=it._column}} IS NOT NULL', + 'GROUP BY bin, bins_number, bin_width, nulls_count, avg_val', + 'ORDER BY bin' +].join('\n')); + + +var TYPE = 'histogram'; + +/** + { + type: 'histogram', + options: { + column: 'name', + bins: 10 // OPTIONAL + } + } + */ +function Histogram(query, options) { + if (!_.isString(options.column)) { + throw new Error('Histogram expects `column` in widget options'); + } + + this.query = query; + this.column = options.column; + this.bins = options.bins; + + this._columnType = null; +} + +Histogram.prototype = new BaseWidget(); +Histogram.prototype.constructor = Histogram; + +module.exports = Histogram; + +var DATE_OIDS = { + 1082: true, + 1114: true, + 1184: true +}; + +Histogram.prototype.sql = function(psql, override, callback) { + if (!callback) { + callback = override; + override = {}; + } + + var self = this; + + var _column = this.column; + + var columnTypeQuery = columnTypeQueryTpl({ + column: _column, query: this.query + }); + + if (this._columnType === null) { + psql.query(columnTypeQuery, function(err, result) { + // assume numeric, will fail later + self._columnType = 'numeric'; + if (!err && !!result.rows[0]) { + var pgType = result.rows[0].pg_typeof; + if (DATE_OIDS.hasOwnProperty(pgType)) { + self._columnType = 'date'; + } + } + self.sql(psql, override, callback); + }, true); // use read-only transaction + return null; + } + + if (this._columnType === 'date') { + _column = columnCastTpl({column: _column}); + } + + var _query = this.query; + + var basicsQuery, binsQuery; + + if (override && _.has(override, 'start') && _.has(override, 'end') && _.has(override, 'bins')) { + debug('overriding with %j', override); + basicsQuery = overrideBasicsQueryTpl({ + _query: _query, + _column: _column, + _start: override.start, + _end: override.end + }); + + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + basicsQuery = basicsQueryTpl({ + _query: _query, + _column: _column + }); + + if (override && _.has(override, 'bins')) { + binsQuery = [ + overrideBinsQueryTpl({ + _bins: override.bins + }) + ].join(',\n'); + } else { + binsQuery = [ + iqrQueryTpl({ + _query: _query, + _column: _column + }), + binsQueryTpl({ + _query: _query, + _minBins: BIN_MIN_NUMBER, + _maxBins: BIN_MAX_NUMBER + }) + ].join(',\n'); + } + } + + + var histogramSql = [ + "WITH", + [ + basicsQuery, + binsQuery, + nullsQueryTpl({ + _query: _query, + _column: _column + }) + ].join(',\n'), + histogramQueryTpl({ + _query: _query, + _column: _column + }) + ].join('\n'); + + debug(histogramSql); + + return callback(null, histogramSql); +}; + +Histogram.prototype.format = function(result, override) { + override = override || {}; + var buckets = []; + + var binsCount = getBinsCount(override); + var width = getWidth(override); + var binsStart = getBinStart(override); + var nulls = 0; + var avg; + + if (result.rows.length) { + var firstRow = result.rows[0]; + binsCount = firstRow.bins_number; + width = firstRow.bin_width || width; + avg = firstRow.avg_val; + nulls = firstRow.nulls_count; + binsStart = override.hasOwnProperty('start') ? override.start : firstRow.min; + + buckets = result.rows.map(function(row) { + return _.omit(row, 'bins_number', 'bin_width', 'nulls_count', 'avg_val'); + }); + } + + return { + bin_width: width, + bins_count: binsCount, + bins_start: binsStart, + nulls: nulls, + avg: avg, + bins: buckets + }; +}; + +function getBinStart(override) { + return override.start || 0; +} + +function getBinsCount(override) { + return override.bins || 0; +} + +function getWidth(override) { + var width = 0; + var binsCount = override.bins; + + if (binsCount && Number.isFinite(override.start) && Number.isFinite(override.end)) { + width = (override.end - override.start) / binsCount; + } + + return width; +} + +Histogram.prototype.getType = function() { + return TYPE; +}; + +Histogram.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _column: this.column, + _query: this.query + }); +}; diff --git a/lib/cartodb/models/dataview/index.js b/lib/cartodb/models/dataview/index.js new file mode 100644 index 00000000..d508f1bb --- /dev/null +++ b/lib/cartodb/models/dataview/index.js @@ -0,0 +1,6 @@ +module.exports = { + Aggregation: require('./aggregation'), + Formula: require('./formula'), + Histogram: require('./histogram'), + List: require('./list') +}; diff --git a/lib/cartodb/models/dataview/list.js b/lib/cartodb/models/dataview/list.js new file mode 100644 index 00000000..4127103a --- /dev/null +++ b/lib/cartodb/models/dataview/list.js @@ -0,0 +1,66 @@ +var dot = require('dot'); +dot.templateSettings.strip = false; + +var BaseWidget = require('./base'); + +var TYPE = 'list'; + +var listSqlTpl = dot.template('select {{=it._columns}} from ({{=it._query}}) as _cdb_list'); + +/** +{ + type: 'list', + options: { + columns: ['name', 'description'] + } +} +*/ + +function List(query, options) { + options = options || {}; + + if (!Array.isArray(options.columns)) { + throw new Error('List expects `columns` array in widget options'); + } + + BaseWidget.apply(this); + + this.query = query; + this.columns = options.columns; +} + +List.prototype = new BaseWidget(); +List.prototype.constructor = List; + +module.exports = List; + +List.prototype.sql = function(psql, filters, override, callback) { + if (!callback) { + callback = override; + } + + var listSql = listSqlTpl({ + _query: this.query, + _columns: this.columns.join(', ') + }); + + return callback(null, listSql); +}; + +List.prototype.format = function(result) { + return { + rows: result.rows + }; +}; + +List.prototype.getType = function() { + return TYPE; +}; + +List.prototype.toString = function() { + return JSON.stringify({ + _type: TYPE, + _query: this.query, + _columns: this.columns.join(', ') + }); +}; diff --git a/lib/cartodb/models/filter/bbox.js b/lib/cartodb/models/filter/bbox.js new file mode 100644 index 00000000..8afdf905 --- /dev/null +++ b/lib/cartodb/models/filter/bbox.js @@ -0,0 +1,122 @@ +var debug = require('debug')('windshaft:filter:bbox'); +var dot = require('dot'); +dot.templateSettings.strip = false; + +var filterQueryTpl = dot.template([ + 'SELECT * FROM ({{=it._sql}}) _cdb_bbox_filter', + 'WHERE {{=it._filters}}' +].join('\n')); + +var bboxFilterTpl = dot.template( + '{{=it._column}} && ST_Transform(ST_MakeEnvelope({{=it._bbox}}, 4326), {{=it._srid}})' +); + +var LATITUDE_MAX_VALUE = 85.0511287798066; +var LONGITUDE_LOWER_BOUND = -180; +var LONGITUDE_UPPER_BOUND = 180; +var LONGITUDE_RANGE = LONGITUDE_UPPER_BOUND - LONGITUDE_LOWER_BOUND; + +/** + Definition + { + "type”: "bbox", + "options": { + "column": "the_geom_webmercator", + "srid": 3857 + } + } + + Params + { + “bbox”: "west,south,east,north" + } + */ +function BBox(filterDefinition, filterParams) { + var bbox = filterParams.bbox; + + if (!bbox) { + throw new Error('BBox filter expects to have a bbox param'); + } + + var bboxElements = bbox.split(',').map(function(e) { return +e; }); + + validateBboxElements(bboxElements); + + this.column = filterDefinition.column || 'the_geom_webmercator'; + this.srid = filterDefinition.srid || 3857; + + // Latitudes must be within max extent + var south = Math.max(bboxElements[1], -LATITUDE_MAX_VALUE); + var north = Math.min(bboxElements[3], LATITUDE_MAX_VALUE); + + // Longitudes crossing 180º need another approach + var adjustedLongitudeRange = adjustLongitudeRange([bboxElements[0], bboxElements[2]]); + var west = adjustedLongitudeRange[0]; + var east = adjustedLongitudeRange[1]; + + this.bboxes = getBoundingBoxes(west, south, east, north); +} + +function getBoundingBoxes(west, south, east, north) { + var bboxes = []; + + if (east - west >= 360) { + bboxes.push([-180, south, 180, north]); + } else if (west >= -180 && east <= 180) { + bboxes.push([west, south, east, north]); + } else { + bboxes.push([west, south, 180, north]); + bboxes.push([-180, south, east % 180, north]); + } + + return bboxes; +} + +function validateBboxElements(bboxElements) { + var isNumericBbox = bboxElements + .map(function(n) { return Number.isFinite(n); }) + .reduce(function(allFinite, isFinite) { + if (!allFinite) { + return false; + } + return isFinite; + }, true); + + if (bboxElements.length !== 4 || !isNumericBbox) { + throw new Error('Invalid bbox filter, expected format="west,south,east,north"'); + } +} + +function adjustLongitudeRange(we) { + var west = we[0]; + west -= LONGITUDE_LOWER_BOUND; + west = west - (LONGITUDE_RANGE * Math.floor(west / LONGITUDE_RANGE)) + LONGITUDE_LOWER_BOUND; + + var longitudeRange = Math.min(we[1] - we[0], 360); + + return [west, west + longitudeRange]; +} + +module.exports = BBox; + +module.exports.adjustLongitudeRange = adjustLongitudeRange; +module.exports.LATITUDE_MAX_VALUE = LATITUDE_MAX_VALUE; +module.exports.LONGITUDE_MAX_VALUE = LONGITUDE_UPPER_BOUND; + + +BBox.prototype.sql = function(rawSql) { + var bboxSql = filterQueryTpl({ + _sql: rawSql, + _filters: this.bboxes.map(function(bbox) { + return bboxFilterTpl({ + _column: this.column, + _bbox: bbox.join(','), + _srid: this.srid + }); + }.bind(this)).join(' OR ') + }); + + debug(bboxSql); + + return bboxSql; +}; diff --git a/lib/cartodb/models/mapconfig/named_map_provider.js b/lib/cartodb/models/mapconfig/named_map_provider.js index deeb39a2..b1eaba8b 100644 --- a/lib/cartodb/models/mapconfig/named_map_provider.js +++ b/lib/cartodb/models/mapconfig/named_map_provider.js @@ -11,14 +11,16 @@ var QueryTables = require('cartodb-query-tables'); * @constructor * @type {NamedMapMapConfigProvider} */ -function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, - namedLayersAdapter, overviewsAdapter, turboCartoCssAdapter, +function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, + namedLayersAdapter, overviewsAdapter, turboCartoCssAdapter, analysisMapConfigAdapter, owner, templateId, config, authToken, params) { this.templateMaps = templateMaps; this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; this.userLimitsApi = userLimitsApi; this.namedLayersAdapter = namedLayersAdapter; this.turboCartoCssAdapter = turboCartoCssAdapter; + this.analysisMapConfigAdapter = analysisMapConfigAdapter; this.overviewsAdapter = overviewsAdapter; this.owner = owner; @@ -53,15 +55,29 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { var mapConfig = null; var datasource = null; var rendererParams; + var apiKey; step( function getTemplate() { self.getTemplate(this); }, - function prepareParams(err, tpl) { + function prepareDbParams(err, tpl) { + assert.ifError(err); + self.template = tpl; + + rendererParams = _.extend({}, self.params, { + user: self.owner + }); + self.setDBParams(self.owner, rendererParams, this); + }, + function getUserApiKey(err) { + assert.ifError(err); + self.metadataBackend.getUserMapKey(self.owner, this); + }, + function prepareParams(err, _apiKey) { assert.ifError(err); - self.template = tpl; + apiKey = _apiKey; var templateParams = {}; if (self.config) { @@ -78,6 +94,32 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { assert.ifError(err); return self.templateMaps.instance(self.template, templateParams); }, + function prepareAnalysisLayers(err, requestMapConfig) { + assert.ifError(err); + var analysisConfiguration = { + db: { + host: rendererParams.dbhost, + port: rendererParams.dbport, + dbname: rendererParams.dbname, + user: rendererParams.dbuser, + pass: rendererParams.dbpassword + }, + batch: { + username: self.owner, + apiKey: apiKey + } + }; + + var filters = {}; + if (self.params.filters) { + try { + filters = JSON.parse(self.params.filters); + } catch (e) { + // ignore + } + } + self.analysisMapConfigAdapter.getLayers(analysisConfiguration, requestMapConfig, filters, this); + }, function prepareLayergroup(err, _mapConfig) { assert.ifError(err); var next = this; @@ -126,17 +168,10 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) { return next(null, _mapConfig, datasource); }); }, - function beforeLayergroupCreate(err, _mapConfig, _datasource) { + function prepareContextLimits(err, _mapConfig, _datasource) { assert.ifError(err); mapConfig = _mapConfig; datasource = _datasource; - rendererParams = _.extend({}, self.params, { - user: self.owner - }); - self.setDBParams(self.owner, rendererParams, this); - }, - function prepareContextLimits(err) { - assert.ifError(err); self.userLimitsApi.getRenderLimits(self.owner, this); }, function cacheAndReturnMapConfig(err, renderLimits) { diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 5f628501..e727f4b2 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -27,6 +27,8 @@ var NamedMapProviderCache = require('./cache/named_map_provider_cache'); var PgQueryRunner = require('./backends/pg_query_runner'); var PgConnection = require('./backends/pg_connection'); +var AnalysisBackend = require('./backends/analysis'); + var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png'; var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null}); @@ -140,6 +142,7 @@ module.exports = function(serverOptions) { var tileBackend = new windshaft.backend.Tile(rendererCache); var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + var analysisBackend = new AnalysisBackend(serverOptions.analysis); var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; @@ -152,6 +155,7 @@ module.exports = function(serverOptions) { var namedMapProviderCache = new NamedMapProviderCache( templateMaps, pgConnection, + metadataBackend, userLimitsApi, overviewsAdapter, turboCartocssAdapter @@ -180,7 +184,8 @@ module.exports = function(serverOptions) { new windshaft.backend.Widget(), surrogateKeysCache, userLimitsApi, - layergroupAffectedTablesCache + layergroupAffectedTablesCache, + analysisBackend ).register(app); new controller.Map( @@ -193,7 +198,8 @@ module.exports = function(serverOptions) { userLimitsApi, layergroupAffectedTablesCache, overviewsAdapter, - turboCartocssAdapter + turboCartocssAdapter, + analysisBackend ).register(app); new controller.NamedMaps( diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 00011356..22039b8c 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -31,6 +31,14 @@ if (global.environment.statsd) { } } +var analysisConfig = _.defaults(global.environment.analysis || {}, { + batch: { + inlineExecution: false, + endpoint: 'http://127.0.0.1:8080/api/v2/sql/job', + hostHeaderTemplate: '{{=it.username}}.localhost.lan' + } +}); + module.exports = { bind: { port: global.environment.port, @@ -81,6 +89,14 @@ module.exports = { torque: rendererConfig.torque, http: rendererConfig.http }, + + analysis: { + batch: { + inlineExecution: analysisConfig.batch.inlineExecution, + endpoint: analysisConfig.batch.endpoint, + hostHeaderTemplate: analysisConfig.batch.hostHeaderTemplate + } + }, // Do not send unwatch on release. See http://github.com/CartoDB/Windshaft-cartodb/issues/161 redis: _.extend(global.environment.redis, {unwatchOnRelease: false}), enable_cors: global.environment.enable_cors, diff --git a/lib/cartodb/stats/timer.js b/lib/cartodb/stats/timer.js new file mode 100644 index 00000000..0ef725ab --- /dev/null +++ b/lib/cartodb/stats/timer.js @@ -0,0 +1,32 @@ +function Timer() { + this.times = {}; +} + +module.exports = Timer; + +Timer.prototype.start = function(label) { + this.timeIt(label, 'start'); +}; + +Timer.prototype.end = function(label) { + this.timeIt(label, 'end'); +}; + +Timer.prototype.timeIt = function(label, what) { + this.times[label] = this.times[label] || {}; + this.times[label][what] = Date.now(); +}; + +Timer.prototype.getTimes = function() { + var self = this; + var times = {}; + + Object.keys(this.times).forEach(function(label) { + var stat = self.times[label]; + if (stat.start && stat.end) { + times[label] = Math.max(0, stat.end - stat.start); + } + }); + + return times; +}; diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 4c0100dd..25c12d5b 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -90,7 +90,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.2 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -104,6 +104,403 @@ } } }, + "camshaft": { + "version": "0.5.0", + "from": "camshaft@0.5.0", + "resolved": "https://registry.npmjs.org/camshaft/-/camshaft-0.5.0.tgz", + "dependencies": { + "async": { + "version": "1.5.2", + "from": "async@>=1.5.2 <2.0.0", + "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" + }, + "request": { + "version": "2.72.0", + "from": "request@>=2.69.0 <3.0.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.72.0.tgz", + "dependencies": { + "aws-sign2": { + "version": "0.6.0", + "from": "aws-sign2@>=0.6.0 <0.7.0", + "resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz" + }, + "aws4": { + "version": "1.3.2", + "from": "aws4@>=1.2.1 <2.0.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz", + "dependencies": { + "lru-cache": { + "version": "4.0.1", + "from": "lru-cache@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.1.tgz", + "dependencies": { + "pseudomap": { + "version": "1.0.2", + "from": "pseudomap@>=1.0.1 <2.0.0", + "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz" + }, + "yallist": { + "version": "2.0.0", + "from": "yallist@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz" + } + } + } + } + }, + "bl": { + "version": "1.1.2", + "from": "bl@>=1.1.2 <1.2.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-1.1.2.tgz", + "dependencies": { + "readable-stream": { + "version": "2.0.6", + "from": "readable-stream@>=2.0.5 <2.1.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz", + "dependencies": { + "core-util-is": { + "version": "1.0.2", + "from": "core-util-is@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz" + }, + "inherits": { + "version": "2.0.1", + "from": "inherits@>=2.0.1 <2.1.0", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" + }, + "isarray": { + "version": "1.0.0", + "from": "isarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" + }, + "process-nextick-args": { + "version": "1.0.6", + "from": "process-nextick-args@>=1.0.6 <1.1.0", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.6.tgz" + }, + "string_decoder": { + "version": "0.10.31", + "from": "string_decoder@>=0.10.0 <0.11.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz" + }, + "util-deprecate": { + "version": "1.0.2", + "from": "util-deprecate@>=1.0.1 <1.1.0", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz" + } + } + } + } + }, + "caseless": { + "version": "0.11.0", + "from": "caseless@>=0.11.0 <0.12.0", + "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.11.0.tgz" + }, + "combined-stream": { + "version": "1.0.5", + "from": "combined-stream@>=1.0.5 <1.1.0", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", + "dependencies": { + "delayed-stream": { + "version": "1.0.0", + "from": "delayed-stream@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz" + } + } + }, + "extend": { + "version": "3.0.0", + "from": "extend@>=3.0.0 <3.1.0", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.0.tgz" + }, + "forever-agent": { + "version": "0.6.1", + "from": "forever-agent@>=0.6.1 <0.7.0", + "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz" + }, + "form-data": { + "version": "1.0.0-rc4", + "from": "form-data@>=1.0.0-rc3 <1.1.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-1.0.0-rc4.tgz" + }, + "har-validator": { + "version": "2.0.6", + "from": "har-validator@>=2.0.6 <2.1.0", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-2.0.6.tgz", + "dependencies": { + "chalk": { + "version": "1.1.3", + "from": "chalk@>=1.1.1 <2.0.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "dependencies": { + "ansi-styles": { + "version": "2.2.1", + "from": "ansi-styles@>=2.2.1 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz" + }, + "escape-string-regexp": { + "version": "1.0.5", + "from": "escape-string-regexp@>=1.0.2 <2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz" + }, + "has-ansi": { + "version": "2.0.0", + "from": "has-ansi@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "ansi-regex@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "strip-ansi": { + "version": "3.0.1", + "from": "strip-ansi@>=3.0.0 <4.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "dependencies": { + "ansi-regex": { + "version": "2.0.0", + "from": "ansi-regex@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz" + } + } + }, + "supports-color": { + "version": "2.0.0", + "from": "supports-color@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz" + } + } + }, + "commander": { + "version": "2.9.0", + "from": "commander@>=2.9.0 <3.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.9.0.tgz", + "dependencies": { + "graceful-readlink": { + "version": "1.0.1", + "from": "graceful-readlink@>=1.0.0", + "resolved": "https://registry.npmjs.org/graceful-readlink/-/graceful-readlink-1.0.1.tgz" + } + } + }, + "is-my-json-valid": { + "version": "2.13.1", + "from": "is-my-json-valid@>=2.12.4 <3.0.0", + "resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz", + "dependencies": { + "generate-function": { + "version": "2.0.0", + "from": "generate-function@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/generate-function/-/generate-function-2.0.0.tgz" + }, + "generate-object-property": { + "version": "1.2.0", + "from": "generate-object-property@>=1.1.0 <2.0.0", + "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", + "dependencies": { + "is-property": { + "version": "1.0.2", + "from": "is-property@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz" + } + } + }, + "jsonpointer": { + "version": "2.0.0", + "from": "jsonpointer@2.0.0", + "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-2.0.0.tgz" + }, + "xtend": { + "version": "4.0.1", + "from": "xtend@>=4.0.0 <5.0.0", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz" + } + } + }, + "pinkie-promise": { + "version": "2.0.1", + "from": "pinkie-promise@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz", + "dependencies": { + "pinkie": { + "version": "2.0.4", + "from": "pinkie@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz" + } + } + } + } + }, + "hawk": { + "version": "3.1.3", + "from": "hawk@>=3.1.3 <3.2.0", + "resolved": "https://registry.npmjs.org/hawk/-/hawk-3.1.3.tgz", + "dependencies": { + "hoek": { + "version": "2.16.3", + "from": "hoek@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/hoek/-/hoek-2.16.3.tgz" + }, + "boom": { + "version": "2.10.1", + "from": "boom@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/boom/-/boom-2.10.1.tgz" + }, + "cryptiles": { + "version": "2.0.5", + "from": "cryptiles@>=2.0.0 <3.0.0", + "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-2.0.5.tgz" + }, + "sntp": { + "version": "1.0.9", + "from": "sntp@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/sntp/-/sntp-1.0.9.tgz" + } + } + }, + "http-signature": { + "version": "1.1.1", + "from": "http-signature@>=1.1.0 <1.2.0", + "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.1.1.tgz", + "dependencies": { + "assert-plus": { + "version": "0.2.0", + "from": "assert-plus@>=0.2.0 <0.3.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-0.2.0.tgz" + }, + "jsprim": { + "version": "1.2.2", + "from": "jsprim@>=1.2.2 <2.0.0", + "resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.2.2.tgz", + "dependencies": { + "extsprintf": { + "version": "1.0.2", + "from": "extsprintf@1.0.2", + "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.0.2.tgz" + }, + "json-schema": { + "version": "0.2.2", + "from": "json-schema@0.2.2", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.2.tgz" + }, + "verror": { + "version": "1.3.6", + "from": "verror@1.3.6", + "resolved": "https://registry.npmjs.org/verror/-/verror-1.3.6.tgz" + } + } + }, + "sshpk": { + "version": "1.7.4", + "from": "sshpk@>=1.7.0 <2.0.0", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.7.4.tgz", + "dependencies": { + "asn1": { + "version": "0.2.3", + "from": "asn1@>=0.2.3 <0.3.0", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz" + }, + "dashdash": { + "version": "1.13.0", + "from": "dashdash@>=1.10.1 <2.0.0", + "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.13.0.tgz", + "dependencies": { + "assert-plus": { + "version": "1.0.0", + "from": "assert-plus@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz" + } + } + }, + "jsbn": { + "version": "0.1.0", + "from": "jsbn@>=0.1.0 <0.2.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz" + }, + "tweetnacl": { + "version": "0.14.3", + "from": "tweetnacl@>=0.13.0 <1.0.0", + "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.3.tgz" + }, + "jodid25519": { + "version": "1.0.2", + "from": "jodid25519@>=1.0.0 <2.0.0", + "resolved": "https://registry.npmjs.org/jodid25519/-/jodid25519-1.0.2.tgz" + }, + "ecc-jsbn": { + "version": "0.1.1", + "from": "ecc-jsbn@>=0.0.1 <1.0.0", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz" + } + } + } + } + }, + "is-typedarray": { + "version": "1.0.0", + "from": "is-typedarray@>=1.0.0 <1.1.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz" + }, + "isstream": { + "version": "0.1.2", + "from": "isstream@>=0.1.2 <0.2.0", + "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz" + }, + "json-stringify-safe": { + "version": "5.0.1", + "from": "json-stringify-safe@>=5.0.1 <5.1.0", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz" + }, + "mime-types": { + "version": "2.1.10", + "from": "mime-types@>=2.1.7 <2.2.0", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", + "dependencies": { + "mime-db": { + "version": "1.22.0", + "from": "mime-db@>=1.22.0 <1.23.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.22.0.tgz" + } + } + }, + "node-uuid": { + "version": "1.4.7", + "from": "node-uuid@>=1.4.7 <1.5.0", + "resolved": "https://registry.npmjs.org/node-uuid/-/node-uuid-1.4.7.tgz" + }, + "oauth-sign": { + "version": "0.8.1", + "from": "oauth-sign@>=0.8.1 <0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.1.tgz" + }, + "qs": { + "version": "6.1.0", + "from": "qs@>=6.1.0 <6.2.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.1.0.tgz" + }, + "stringstream": { + "version": "0.0.5", + "from": "stringstream@>=0.0.4 <0.1.0", + "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz" + }, + "tough-cookie": { + "version": "2.2.2", + "from": "tough-cookie@>=2.2.0 <2.3.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.2.2.tgz" + }, + "tunnel-agent": { + "version": "0.4.2", + "from": "tunnel-agent@>=0.4.1 <0.5.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.4.2.tgz" + } + } + } + } + }, "cartodb-psql": { "version": "0.6.1", "from": "cartodb-psql@>=0.6.1 <0.7.0", @@ -203,7 +600,7 @@ "dependencies": { "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { @@ -267,7 +664,7 @@ "dependencies": { "unpipe": { "version": "1.0.0", - "from": "unpipe@>=1.0.0 <1.1.0", + "from": "unpipe@1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" } } @@ -392,7 +789,7 @@ }, "mime-types": { "version": "2.1.10", - "from": "mime-types@>=2.1.10 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz", "dependencies": { "mime-db": { diff --git a/package.json b/package.json index 9ef9d4f3..3cd3ad82 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "request": "~2.62.0", "cartodb-redis": "~0.13.0", "cartodb-psql": "~0.6.1", + "camshaft": "0.5.0", "fastly-purge": "~1.0.1", "redis-mpool": "~0.4.0", "lru-cache": "2.6.5", diff --git a/test/acceptance/analysis/analysis-layers-dataviews.js b/test/acceptance/analysis/analysis-layers-dataviews.js new file mode 100644 index 00000000..a328ed09 --- /dev/null +++ b/test/acceptance/analysis/analysis-layers-dataviews.js @@ -0,0 +1,148 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); +var dot = require('dot'); + +describe('analysis-layers-dataviews', function() { + + var multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: {{=it._opacity}};", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" + ].join('\n')); + + + function cartocss(color, opacity) { + return multitypeStyleTemplate({ + _color: color || '#F11810', + _opacity: Number.isFinite(opacity) ? opacity : 1 + }); + } + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var DEFAULT_MULTITYPE_STYLE = cartocss(); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + { + pop_max_histogram: { + source: { + id: '2570e105-7b37-40d2-bdf4-1af889598745' + }, + type: 'histogram', + options: { + column: 'pop_max' + } + } + }, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ); + + it('should get histogram dataview', function(done) { + var testClient = new TestClient(mapConfig, 1234); + + testClient.getDataview('pop_max_histogram', function(err, dataview) { + assert.ok(!err, err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins_start, 0); + + testClient.drain(done); + }); + }); + + it('should get a filtered histogram dataview', function(done) { + var testClient = new TestClient(mapConfig, 1234); + + var params = { + filters: { + dataviews: { + pop_max_histogram: { + min: 2e6 + } + } + } + }; + + testClient.getDataview('pop_max_histogram', params, function(err, dataview) { + assert.ok(!err, err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins_start, 2008000); + + testClient.drain(done); + }); + }); + + it('should skip the filter when sending own_filter=0 for histogram dataview', function(done) { + var testClient = new TestClient(mapConfig, 1234); + + var params = { + filters: { + dataviews: { + pop_max_histogram: { + min: 2e6 + } + } + }, + own_filter: 0 + }; + + testClient.getDataview('pop_max_histogram', params, function(err, dataview) { + assert.ok(!err, err); + + assert.equal(dataview.type, 'histogram'); + assert.equal(dataview.bins_start, 0); + + testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/analysis/analysis-layers-use-cases.js b/test/acceptance/analysis/analysis-layers-use-cases.js new file mode 100644 index 00000000..e60c8e79 --- /dev/null +++ b/test/acceptance/analysis/analysis-layers-use-cases.js @@ -0,0 +1,671 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); +var dot = require('dot'); + +describe('analysis-layers use cases', function() { + + + var multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: {{=it._opacity}};", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" + ].join('\n')); + + + function cartocss(color, opacity) { + return multitypeStyleTemplate({ + _color: color || '#F11810', + _opacity: Number.isFinite(opacity) ? opacity : 1 + }); + } + + function mapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analysis: analysis || [] + }; + } + + function analysisDef(analysis) { + return JSON.stringify(analysis); + } + + var DEFAULT_MULTITYPE_STYLE = cartocss(); + + var TILE_ANALYSIS_TABLES = { z: 14, x: 8023, y: 6177 }; + + var useCases = [ + { + desc: '1 mapnik layer', + mapConfig: { + version: '1.5.0', + layers: [ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ] + } + }, + + { + desc: '2 mapnik layers', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_banks", + cartocss: cartocss('#2167AB'), + cartocss_version: '2.3.0' + } + }, + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ]) + }, + + { + desc: 'rent listings + buffer over atm-machines', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radius": 250 + } + }), + cartocss: cartocss('black', 0.5) + } + } + ]) + }, + + { + desc: 'rent listings + point-in-polygon from buffer atm-machines and rent listings', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "point-in-polygon", + "params": { + "pointsSource": { + "type": "source", + "params": { + "query": "select * from analysis_rent_listings" + } + }, + "polygonsSource": { + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radius": 250 + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + } + ]) + }, + + { + desc: 'point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig([ + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "point-in-polygon", + "params": { + "pointsSource": { + "type": "source", + "params": { + "query": "select * from analysis_rent_listings" + } + }, + "polygonsSource": { + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radius": 250 + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + }, + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + } + ]) + }, + + { + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + sql: "select * from analysis_rent_listings", + cartocss: DEFAULT_MULTITYPE_STYLE, + cartocss_version: '2.3.0' + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radius": 300 + } + }), + cartocss: cartocss('magenta', 0.5) + } + }, + { + type: 'analysis', + options: { + def: analysisDef({ + "type": "point-in-polygon", + "params": { + "pointsSource": { + "type": "source", + "params": { + "query": "select * from analysis_rent_listings" + } + }, + "polygonsSource": { + "type": "buffer", + "params": { + "source": { + "type": "source", + "params": { + "query": "select * from analysis_banks" + } + }, + "radius": 300 + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + } + ]) + }, + + { + skip: true, + desc: 'buffer + point-in-polygon from buffer atm-machines and rent listings + rent listings', + mapConfig: mapConfig([ + { + type: 'cartodb', + options: { + "source": { id: "a" }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "b1" }, + "cartocss": cartocss('green', 1.0), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "b2" }, + "cartocss": cartocss('magenta', 0.5), + "cartocss_version": "2.3.0" + } + } + ], + [ + { + id: "b2", + options: { + def: analysisDef({ + "type": "count-in-polygon", + "id": "a0", + "params": { + "columnName": 'count_airbnb', + "pointsSource": { + "type": "source", + "params": { + query: "select * from analysis_rent_listings" + }, + dataviews: { + price_histogram: { + type: 'histogram', + options: { + column: 'price' + } + } + } + }, + "polygonsSource": { + "id": "b1", + "type": "buffer", + "params": { + "source": { + "id": "b0", + "type": "source", + "params": { + query: "select * from analysis_banks" + } + }, + "radius": 250 + }, + dataviews: { + bank_category: { + type: 'aggregation', + options: { + column: 'bank' + } + } + } + } + }, + dataviews: { + count_histogram: { + type: 'histogram', + options: { + column: 'count_airbnb' + } + } + } + }), + cartocss: cartocss('green', 1.0) + } + } + ]) + }, + + { + skip: true, + desc: 'I. Distribution centers', + mapConfig: mapConfig( + // layers + [ + { + type: 'cartodb', + options: { + "source": { id: "b0" }, + "cartocss": [ + "#distribution_centers {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 0.7;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: blue;", + " marker-allow-overlap: true;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "a0" }, + "cartocss": [ + "#shops {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 0.7;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "a1" }, + "cartocss": [ + "#routing {", + " line-color: ramp([routing_time], colorbrewer(Reds));", + " line-width: ramp([routing_time], 2, 8);", + " line-opacity: 1.0;", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + } + ], + // dataviews + { + distribution_center_name_category: { + source: { id: 'b0' }, + type: 'aggregation', + options: { + column: 'name' + } + }, + time_histogram: { + source: { id: 'a1' }, + type: 'histogram', + options: { + column: 'routing_time' + } + }, + distance_histogram: { + source: { id: 'a1' }, + type: 'histogram', + options: { + column: 'routing_distance' + } + } + }, + // analysis + [ + { + id: 'a1', + type: 'routing-n-to-n', + params: { + // distanceColumn: 'routing_distance', + // timeColumn: 'routing_time', + originSource: { + id: 'b0', + type: 'source', + params: { + query: 'select * from distribution_centers' + } + }, + destinationSource: { + id: 'a0', + type: 'source', + params: { + query: 'select * from shops' + } + } + } + } + ] + ) + }, + + { + skip: true, + desc: 'II. Population analysis', + mapConfig: mapConfig( + // layers + [ + { + type: 'cartodb', + options: { + "source": { id: "a2" }, + "cartocss": [ + "#count_in_polygon {", + " polygon-opacity: 1.0", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: 0.7", + " polygon-fill: ramp([estimated_people], colorbrewer(Reds));", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + }, + { + type: 'cartodb', + options: { + "source": { id: "a0" }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + // dataviews + { + total_population_formula: { + "source": { id: "a3" }, + type: 'formula', + options: { + column: 'total_population', + operation: 'sum' + } + }, + people_histogram: { // this injects a range filter at `a2` node output + "source": { id: "a2" }, + type: 'histogram', + options: { + column: 'estimated_people' + } + }, + subway_line_category: { // this injects a category filter at `a0` node output + "source": { id: "a0" }, + type: 'aggregation', + options: { + column: 'subway_line' + } + } + }, + // analysis + [ + { + id: 'a3', + // this will union the polygons, produce just one polygon, and calculate the total population for it + type: 'total-population', + params: { + columnName: 'total_population', + source: { + id: 'a2', + type: 'estimated-population', + params: { + columnName: 'estimated_people', + source: { + id: 'a1', + type: 'trade-area', + params: { + source: { + "id": "a0", + "type": "source", + "params": { + query: "select * from subway_stops" + } + }, + kind: 'walk', + time: 300 + } + } + } + } + } + } + ]) + }, + + { + skip: true, + desc: 'III. Point in polygon', + mapConfig: mapConfig( + // layers + [ + { + type: 'cartodb', + options: { + "source": { id: "a1" }, + "cartocss": [ + "#count_in_polygon {", + " polygon-opacity: 1.0", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: 0.7", + " polygon-fill: ramp([count_people], colorbrewer(Reds));", + "}" + ].join('\n'), + "cartocss_version": "2.3.0" + } + } + ], + // dataviews + { + age_histogram: { + "source": { id: "a0" }, + type: 'histogram', + options: { + column: 'age' + } + }, + income_histogram: { + "source": { id: "a0" }, + type: 'histogram', + options: { + column: 'income' + } + }, + gender_category: { + "source": { id: "a0" }, + type: 'aggregation', + options: { + column: 'gender' + } + } + }, + // analysis + [ + { + "id": "a1", + "type": "count-in-polygon", + "params": { + "columnName": 'count_people', + "pointsSource": { + "id": 'a0', + "type": "source", + "params": { + query: "select the_geom, age, gender, income from people" + } + }, + "polygonsSource": { + "id": "b0", + "type": "source", + "params": { + query: "select * from postal_codes" + } + } + } + } + ] + ) + } + + ]; + + useCases.forEach(function(useCase, imageIdx) { + if (!!useCase.skip) { + console.log(JSON.stringify(useCase.mapConfig, null, 4)); + } + it.skip('should implement use case: "' + useCase.desc + '"', function(done) { + + var testClient = new TestClient(useCase.mapConfig, 1234); + + var tile = useCase.tile || TILE_ANALYSIS_TABLES; + + testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + assert.ok(!err, err); + + image.save('/tmp/tests/' + imageIdx + '---' + useCase.desc.replace(/\s/g, '-') + '.png'); + + assert.equal(image.width(), 256); + + testClient.drain(done); + }); + }); + }); +}); diff --git a/test/acceptance/analysis/analysis-layers.js b/test/acceptance/analysis/analysis-layers.js new file mode 100644 index 00000000..f2a3569f --- /dev/null +++ b/test/acceptance/analysis/analysis-layers.js @@ -0,0 +1,187 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); +var dot = require('dot'); + +describe('analysis-layers', function() { + + var IMAGE_TOLERANCE_PER_MIL = 20; + + var multitypeStyleTemplate = dot.template([ + "#points['mapnik::geometry_type'=1] {", + " marker-fill-opacity: {{=it._opacity}};", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: {{=it._opacity}};", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: {{=it._color}};", + " marker-allow-overlap: true;", + "}", + "#lines['mapnik::geometry_type'=2] {", + " line-color: {{=it._color}};", + " line-width: 2;", + " line-opacity: {{=it._opacity}};", + "}", + "#polygons['mapnik::geometry_type'=3] {", + " polygon-fill: {{=it._color}};", + " polygon-opacity: {{=it._opacity}};", + " line-color: #FFF;", + " line-width: 0.5;", + " line-opacity: {{=it._opacity}};", + "}" + ].join('\n')); + + + function cartocss(color, opacity) { + return multitypeStyleTemplate({ + _color: color || '#F11810', + _opacity: Number.isFinite(opacity) ? opacity : 1 + }); + } + + function mapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var DEFAULT_MULTITYPE_STYLE = cartocss(); + + var TILE_ANALYSIS_TABLES = { z: 0, x: 0, y: 0 }; + + var useCases = [ + { + desc: 'basic source-id mapnik layer', + fixture: 'basic-source-id-mapnik-layer.png', + mapConfig: mapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + {}, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ) + }, + + { + desc: 'buffer over source', + fixture: 'buffer-over-source.png', + tile: { z: 7, x: 61, y: 47 }, + mapConfig: mapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": DEFAULT_MULTITYPE_STYLE, + "cartocss_version": "2.3.0" + } + } + ], + {}, + [ + { + "id": "HEAD", + "type": "buffer", + "params": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + }, + "radius": 50000 + } + } + ] + ) + } + ]; + + useCases.forEach(function(useCase) { + it('should implement use case: "' + useCase.desc + '"', function(done) { + + var testClient = new TestClient(useCase.mapConfig, 1234); + + var tile = useCase.tile || TILE_ANALYSIS_TABLES; + + testClient.getTile(tile.z, tile.x, tile.y, function(err, res, image) { + assert.ok(!err, err); + + // To generate images use: + // image.save('/tmp/' + useCase.desc.replace(/\s/g, '-') + '.png'); + + var fixturePath = './test/fixtures/analysis/' + useCase.fixture; + assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { + assert.ok(!err, err); + + testClient.drain(done); + }); + }); + }); + }); + + it('should NOT fail for non-authenticated requests when it is just source', function(done) { + var useCase = useCases[0]; + + // No API key here + var testClient = new TestClient(useCase.mapConfig); + + testClient.getLayergroup(function(err, layergroupResult) { + assert.ok(!err, err); + + assert.equal(layergroupResult.metadata.layers.length, 1); + + testClient.drain(done); + }); + }); + + it('should fail for non-authenticated requests that has a node other than "source"', function(done) { + var useCase = useCases[1]; + + // No API key here + var testClient = new TestClient(useCase.mapConfig); + + var PERMISSION_DENIED_RESPONSE = { + status: 403, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + testClient.getLayergroup(PERMISSION_DENIED_RESPONSE, function(err, layergroupResult) { + assert.ok(!err, err); + assert.deepEqual( + layergroupResult.errors, + ["Analysis requires authentication with API key: permission denied."] + ); + + testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js new file mode 100644 index 00000000..197c281f --- /dev/null +++ b/test/acceptance/analysis/named-maps.js @@ -0,0 +1,188 @@ +var assert = require('../../support/assert'); +var step = require('step'); + +var helper = require('../../support/test_helper'); + +var CartodbWindshaft = require('../../../lib/cartodb/server'); +var serverOptions = require('../../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); + +var LayergroupToken = require('../../../lib/cartodb/models/layergroup_token'); + +describe('named-maps analysis', function() { + + var IMAGE_TOLERANCE_PER_MIL = 20; + + var username = 'localhost'; + var widgetsTemplateName = 'widgets-template'; + + var layergroupid; + var layergroup; + var keysToDelete; + + beforeEach(function(done) { + keysToDelete = {}; + + var widgetsTemplate = { + version: '0.0.1', + name: widgetsTemplateName, + layergroup: { + version: '1.5.0', + layers: [ + { + "type": "cartodb", + "options": { + "source": { + "id": "HEAD" + }, + "cartocss": '#buffer { polygon-fill: red; }', + "cartocss_version": "2.3.0" + } + } + ], + analyses: [ + { + "id": "HEAD", + "type": "buffer", + "params": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + }, + "radius": 50000 + } + } + ] + } + }; + + var template_params = {}; + + step( + function createTemplate() + { + var next = this; + assert.response( + server, + { + url: '/api/v1/map/named?api_key=1234', + method: 'POST', + headers: { + host: username, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(widgetsTemplate) + }, + { + status: 200 + }, + function(res, err) { + next(err, res); + } + ); + }, + function instantiateTemplate(err, res) { + assert.ifError(err); + + assert.deepEqual(JSON.parse(res.body), { template_id: widgetsTemplateName }); + var next = this; + assert.response( + server, + { + url: '/api/v1/map/named/' + widgetsTemplateName, + method: 'POST', + headers: { + host: username, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(template_params) + }, + { + status: 200 + }, + function(res) { + next(null, res); + } + ); + }, + function finish(err, res) { + assert.ifError(err); + + layergroup = JSON.parse(res.body); + assert.ok(layergroup.hasOwnProperty('layergroupid'), "Missing 'layergroupid' from: " + res.body); + layergroupid = layergroup.layergroupid; + + keysToDelete['map_cfg|' + LayergroupToken.parse(layergroup.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + return done(); + } + ); + + }); + + afterEach(function(done) { + step( + function deleteTemplate(err) { + assert.ifError(err); + var next = this; + assert.response( + server, + { + url: '/api/v1/map/named/' + widgetsTemplateName + '?api_key=1234', + method: 'DELETE', + headers: { + host: username + } + }, + { + status: 204 + }, + function(res, err) { + next(err, res); + } + ); + }, + function deleteRedisKeys(err) { + assert.ifError(err); + helper.deleteRedisKeys(keysToDelete, done); + } + ); + }); + + it('should be able to retrieve images from analysis', function(done) { + assert.response( + server, + { + url: '/api/v1/map/' + layergroupid + '/6/31/24.png', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 200, + headers: { + 'Content-Type': 'image/png' + } + }, + function(res, err) { + if (err) { + return done(err); + } + + var fixturePath = './test/fixtures/analysis/named-map-buffer.png'; + assert.imageBufferIsSimilarToFile(res.body, fixturePath, IMAGE_TOLERANCE_PER_MIL, function(err) { + assert.ok(!err, err); + done(); + }); + + } + ); + }); + +}); diff --git a/test/fixtures/analysis/basic-source-id-mapnik-layer.png b/test/fixtures/analysis/basic-source-id-mapnik-layer.png new file mode 100644 index 00000000..bdb315a9 Binary files /dev/null and b/test/fixtures/analysis/basic-source-id-mapnik-layer.png differ diff --git a/test/fixtures/analysis/buffer-over-source.png b/test/fixtures/analysis/buffer-over-source.png new file mode 100644 index 00000000..06aeb50a Binary files /dev/null and b/test/fixtures/analysis/buffer-over-source.png differ diff --git a/test/fixtures/analysis/named-map-buffer.png b/test/fixtures/analysis/named-map-buffer.png new file mode 100644 index 00000000..6aee72e9 Binary files /dev/null and b/test/fixtures/analysis/named-map-buffer.png differ diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index f03d52de..928272fd 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -71,6 +71,9 @@ if test x"$PREPARE_PGSQL" = xyes; then dropdb "${TEST_DB}" createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" + curl -L -s https://raw.githubusercontent.com/CartoDB/camshaft/master/test/fixtures/cdb_analysis_catalog.sql -o sql/cdb_analysis_catalog.sql + cat sql/cdb_analysis_catalog.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + cat sql/windshaft.test.sql sql/gadm4.sql | sed "s/:PUBLICUSER/${PUBLICUSER}/" | sed "s/:PUBLICPASS/${PUBLICPASS}/" | diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 8a01903b..4690f081 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -332,3 +332,298 @@ INSERT INTO _vovw_2_test_table_overviews VALUES INSERT INTO _vovw_1_test_table_overviews VALUES ('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'); + + +-- analysis tables ----------------------------------------------- + +ALTER TABLE cdb_analysis_catalog OWNER TO :TESTUSER; + + +-- +-- TOC entry 804 (class 1259 OID 13870252) +-- Name: analysis_banks; Type: TABLE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE TABLE analysis_banks ( + cartodb_id bigint NOT NULL, + the_geom geometry(Geometry,4326), + the_geom_webmercator geometry(Geometry,3857), + bank text +); + + +ALTER TABLE analysis_banks OWNER TO :TESTUSER; + +-- +-- TOC entry 803 (class 1259 OID 13870250) +-- Name: analysis_banks_cartodb_id_seq; Type: SEQUENCE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE SEQUENCE analysis_banks_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE analysis_banks_cartodb_id_seq OWNER TO :TESTUSER; + +-- +-- TOC entry 5784 (class 0 OID 0) +-- Dependencies: 803 +-- Name: analysis_banks_cartodb_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER SEQUENCE analysis_banks_cartodb_id_seq OWNED BY analysis_banks.cartodb_id; + + +-- +-- TOC entry 802 (class 1259 OID 13870235) +-- Name: analysis_rent_listings; Type: TABLE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE TABLE analysis_rent_listings ( + cartodb_id bigint NOT NULL, + the_geom geometry(Geometry,4326), + the_geom_webmercator geometry(Geometry,3857), + price double precision +); + + +ALTER TABLE analysis_rent_listings OWNER TO :TESTUSER; + +-- +-- TOC entry 801 (class 1259 OID 13870233) +-- Name: analysis_rent_listings_cartodb_id_seq; Type: SEQUENCE; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +CREATE SEQUENCE analysis_rent_listings_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +ALTER TABLE analysis_rent_listings_cartodb_id_seq OWNER TO :TESTUSER; + +-- +-- TOC entry 5786 (class 0 OID 0) +-- Dependencies: 801 +-- Name: analysis_rent_listings_cartodb_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER SEQUENCE analysis_rent_listings_cartodb_id_seq OWNED BY analysis_rent_listings.cartodb_id; + + +-- +-- TOC entry 5612 (class 2604 OID 13870258) +-- Name: cartodb_id; Type: DEFAULT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER TABLE ONLY analysis_banks ALTER COLUMN cartodb_id SET DEFAULT nextval('analysis_banks_cartodb_id_seq'::regclass); + + +-- +-- TOC entry 5611 (class 2604 OID 13870241) +-- Name: cartodb_id; Type: DEFAULT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +ALTER TABLE ONLY analysis_rent_listings ALTER COLUMN cartodb_id SET DEFAULT nextval('analysis_rent_listings_cartodb_id_seq'::regclass); + + +-- +-- TOC entry 5778 (class 0 OID 13870252) +-- Dependencies: 804 +-- Data for Name: analysis_banks; Type: TABLE DATA; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +COPY analysis_banks (cartodb_id, the_geom, the_geom_webmercator, bank) FROM stdin; +1 0101000020E61000000AEC3F6BE5A80DC09EF022E20C364440 0101000020110F0000AD24852BA63019C13215440E02CC5241 BBVA +2 0101000020E61000005AB3C72EE6A40DC02A499181E3364440 0101000020110F0000DE3E9A22412D19C1B059CF80F1CC5241 Santander +3 0101000020E6100000FD52FCA1E1960DC0DD48F7ADB9354440 0101000020110F00008C1CE860592119C12B37FC3BA5CB5241 BBVA +4 0101000020E61000008A52C823B69F0DC02579E6E0B4354440 0101000020110F0000DC41553AD92819C170BBE0E09FCB5241 Santander +5 0101000020E6100000A6C9EAA399B00DC00A31DF64BD354440 0101000020110F000041C09F2D313719C1EDC9C960A9CB5241 Santander +6 0101000020E6100000D2AD3EB570970DC01F85791E1B354440 0101000020110F00001D0E73E4D22119C14730F65AF4CA5241 BBVA +7 0101000020E6100000B2F9AF3A6FA40DC0BA91F3D44C364440 0101000020110F0000B071A11BDC2C19C16DD5026649CC5241 BBVA +8 0101000020E61000000F1B8134EE8E0DC067D8E65807374440 0101000020110F000033B46EB0981A19C10322017E19CD5241 BBVA +9 0101000020E6100000F12A179193880DC07EBCEDABC2354440 0101000020110F0000C11C4B2F331519C1B2B8FD43AFCB5241 BBVA +10 0101000020E61000004B602775E0A50DC0037374A875364440 0101000020110F00004B5097B1152E19C1481948F276CC5241 BBVA +11 0101000020E6100000F796A5DDEE880DC05337CD960A364440 0101000020110F0000ACE39CB9801519C1F843077FFFCB5241 Santander +12 0101000020E610000041BC5F214A920DC0AFCE2B5507354440 0101000020110F000027682406731D19C15145B148DECA5241 BBVA +13 0101000020E6100000CC2EFD14A0AF0DC00550C5AE47354440 0101000020110F000095956F3A5D3619C11CACED1026CB5241 BBVA +14 0101000020E6100000B113F5FFF28E0DC01075C9419A354440 0101000020110F00005E9EE8C29C1A19C1DB38342E82CB5241 BBVA +15 0101000020E6100000997EFF432F990DC0E6D8677BF1364440 0101000020110F00007E0667274E2319C1ACE7B01801CD5241 Santander +\. + + +-- +-- TOC entry 5787 (class 0 OID 0) +-- Dependencies: 803 +-- Name: analysis_banks_cartodb_id_seq; Type: SEQUENCE SET; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +SELECT pg_catalog.setval('analysis_banks_cartodb_id_seq', 15, true); + + +-- +-- TOC entry 5776 (class 0 OID 13870235) +-- Dependencies: 802 +-- Data for Name: analysis_rent_listings; Type: TABLE DATA; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +COPY analysis_rent_listings (cartodb_id, the_geom, the_geom_webmercator, price) FROM stdin; +1 0101000020E61000006300CEA9C4920DC0FCB8FEFB04354440 0101000020110F0000EA3B5C17DB1D19C135CF17AADBCA5241 81 +2 0101000020E6100000F49AD93201910DC0E37AB970F8354440 0101000020110F00009BD483A95B1C19C11FC4D53FEBCB5241 169 +3 0101000020E610000096F9FBAA0F880DC03974BF03B8354440 0101000020110F00006891BA29C31419C1055F8260A3CB5241 163 +4 0101000020E61000009F08C1D17C930DC0651427FA68364440 0101000020110F00006C35BB7E771E19C1837381CC68CC5241 127 +5 0101000020E61000000A7B7928EAB40DC0425EE239C4364440 0101000020110F00000C6ADB3EDB3A19C14FDB889ACECC5241 58 +6 0101000020E610000095B232CF20A00DC0584C660044354440 0101000020110F0000979587D2332919C177DEB3F521CB5241 184 +7 0101000020E61000001E8C81FF328F0DC031AE47DFB2364440 0101000020110F00006634761DD31A19C13723EB3DBBCC5241 109 +8 0101000020E61000008A014F8FD79D0DC0B1E8EA3376364440 0101000020110F000021A20DC5422719C15A18E08D77CC5241 87 +9 0101000020E61000006ECAA9C393970DC0F71C39950A364440 0101000020110F0000A3016DAAF02119C19C71447DFFCB5241 132 +10 0101000020E6100000FB434A9D57A60DC0F9FD273C3F354440 0101000020110F00008E7BC3E47A2E19C1798082A41CCB5241 134 +11 0101000020E6100000C597512EFE8C0DC032DE0DCDFD344440 0101000020110F0000F51B6C6AF31819C15512CCA6D3CA5241 167 +12 0101000020E610000025E84904589F0DC00BC88CB0BA364440 0101000020110F000024C7054A892819C1C84EBBF6C3CC5241 135 +13 0101000020E61000002917E524E69E0DC0EE4915018C354440 0101000020110F00008011BC93282819C153F8E34772CB5241 160 +14 0101000020E6100000617E8939B1A40DC05EF963BEF0344440 0101000020110F000071896E28142D19C160872616C5CA5241 185 +15 0101000020E6100000B20B809602A30DC0514274FE37354440 0101000020110F00007D63FC6AA62B19C1DC88B19014CB5241 94 +16 0101000020E6100000191978861B9C0DC05C902E6C45354440 0101000020110F0000EBC9ACA6C92519C10A76818B23CB5241 172 +17 0101000020E610000098D6FCD1F2AC0DC01ED4EEFCAE354440 0101000020110F0000FA7E3A3C173419C1DFFAA34E99CB5241 154 +18 0101000020E6100000F9C7CB37DC970DC0B7E914C94E354440 0101000020110F0000795D5C332E2219C1FAAE47FD2DCB5241 183 +19 0101000020E61000002D0263B27B940DC09D26F27AC4354440 0101000020110F0000C4616AF64F1F19C1D2868548B1CB5241 77 +20 0101000020E6100000BD682E39DA8D0DC0DA3ADC937A364440 0101000020110F000038D83D4CAE1919C143A35B6F7CCC5241 196 +21 0101000020E6100000165B41DEB2A90DC0D0D1F568A4354440 0101000020110F00008B9179A8543119C1EAADBA818DCB5241 74 +22 0101000020E6100000DECD65EC4A8D0DC08D309F6B1F354440 0101000020110F000063E4D797341919C1A1034827F9CA5241 164 +23 0101000020E6100000D14DF59EC1AD0DC086C17713C7354440 0101000020110F0000E8ED02DFC63419C15FACD82DB4CB5241 180 +24 0101000020E61000002A41F69FA7900DC063FF9C14AC364440 0101000020110F0000DDF34D960F1C19C1070119AAB3CC5241 62 +25 0101000020E61000001E390026BA8B0DC0B506A19336354440 0101000020110F00005620FE36E01719C109E9F5FB12CB5241 103 +26 0101000020E61000001856B89F9E890DC0AE3A805FA7364440 0101000020110F0000F4E717FF151619C102619069AECC5241 132 +27 0101000020E61000006244348FA7AD0DC093992086E1354440 0101000020110F0000204AB0BCB03419C10E2515AFD1CB5241 64 +28 0101000020E6100000C249D99ADA970DC0D7CF322454364440 0101000020110F00005303A5D42C2219C1463AA98D51CC5241 142 +29 0101000020E61000001E36C089F2930DC0098310FAF2344440 0101000020110F0000E9293E79DB1E19C134C7D593C7CA5241 149 +30 0101000020E610000084DC236A48970DC08AA0D30E37354440 0101000020110F0000A4E5D3ABB02119C12A57638513CB5241 190 +31 0101000020E610000068C42C3D6DAE0DC0E316EF8262354440 0101000020110F000007FF5AA0583519C1CEF97EFE43CB5241 132 +32 0101000020E6100000F0FCDE54BCA70DC03311579A9F354440 0101000020110F0000CE2C83DAA92F19C120A8E62488CB5241 180 +33 0101000020E6100000B5A43D9E809B0DC059B84FF6F1364440 0101000020110F000035B5A016462519C15C07D2A101CD5241 160 +34 0101000020E6100000D4D36C80629E0DC0249F8DFC8B354440 0101000020110F0000F241EAC5B82719C1904ED64272CB5241 95 +35 0101000020E61000008BEA878AF1A50DC05DA77AA5B9354440 0101000020110F0000BBD0E633242E19C15D7E8432A5CB5241 108 +36 0101000020E610000022D3CD1F62B40DC08AC22B852E364440 0101000020110F0000C8E240B6673A19C1DCF9E99427CC5241 168 +37 0101000020E6100000EBAC4CB637950DC0E989CFA0D8364440 0101000020110F0000883BDDA4EF1F19C15478CE5DE5CC5241 80 +38 0101000020E61000007CEFD63A8A960DC006B8BDD1C8364440 0101000020110F00003728B0250F2119C169FD71BAD3CC5241 168 +39 0101000020E6100000756B387CCFA70DC084C36711F7354440 0101000020110F0000CEB4ED1EBA2F19C13F1EE7B7E9CB5241 56 +40 0101000020E6100000A47B2BAC4E960DC0820DC4B760354440 0101000020110F00008BCCAF90DC2019C1AE4248FE41CB5241 162 +41 0101000020E6100000D5645E0C31990DC0F535DB974F364440 0101000020110F00007E8BFFAA4F2319C1B884AA7A4CCC5241 102 +42 0101000020E6100000E3B6E0EA41B40DC02ADE56A899364440 0101000020110F0000FFC4D55B4C3A19C1A602351C9FCC5241 174 +43 0101000020E61000006C513CE4E3B00DC010B3C38659364440 0101000020110F00002817653D703719C14258A88F57CC5241 97 +44 0101000020E61000006B6DF64366A10DC0C980E41EB2364440 0101000020110F0000CFE47B3B482A19C1EFCB4567BACC5241 63 +45 0101000020E61000007031674EA8960DC073B9E40B22354440 0101000020110F0000A608EEB0282119C159033215FCCA5241 159 +46 0101000020E610000002328782CB890DC0A6462A1EAF364440 0101000020110F000008823D1E3C1619C14C749B0DB7CC5241 141 +47 0101000020E61000009694069312A70DC09438822A79354440 0101000020110F0000922DC0AD192F19C113F51A445DCB5241 185 +48 0101000020E610000081F9078F75AE0DC069F4C3ADCD364440 0101000020110F00003F393EB15F3519C1D4F16926D9CC5241 113 +49 0101000020E6100000E348615820AC0DC0D165849A50364440 0101000020110F0000ACEC8F7A643319C14D783D9B4DCC5241 124 +50 0101000020E6100000378F35768FAD0DC085A048F310364440 0101000020110F0000BFF95B459C3419C12A94C89706CC5241 62 +51 0101000020E6100000F25E60BD14B40DC0FA9EFA964F354440 0101000020110F0000EB4840FD253A19C1FF36F6E22ECB5241 109 +52 0101000020E6100000B73BFD3FB6AF0DC0A3D844C97B364440 0101000020110F0000AD1F370E703619C1B9278EC87DCC5241 149 +53 0101000020E6100000FF1181969EB10DC0136E39E5A0364440 0101000020110F0000D10A15CD0E3819C1828B712FA7CC5241 176 +54 0101000020E6100000426F39BD459A0DC0A5505E4CB7354440 0101000020110F000074615DA93A2419C106D4EF93A2CB5241 90 +55 0101000020E6100000E2B0F14972A80DC0AFA4D0BF84354440 0101000020110F00008817D563443019C1202D06306ACB5241 162 +56 0101000020E6100000ADE43CA9F79A0DC0C24A1417D9364440 0101000020110F0000C4944EC5D12419C1D0B5C2E1E5CC5241 80 +57 0101000020E6100000AD6B99A9818B0DC07E06699A33364440 0101000020110F00007EE2C43DB01719C12BBF9D402DCC5241 172 +58 0101000020E610000098272DE67B960DC08EC657CCF4344440 0101000020110F0000352CE4F9022119C16660F49BC9CA5241 177 +59 0101000020E6100000DEAA210E56A90DC0B45845C72B354440 0101000020110F00003689FED4053119C1F0FE50F006CB5241 172 +60 0101000020E61000004CAD1CF03B8F0DC07A4C4F1EED364440 0101000020110F000048EE2CB5DA1A19C1E236513AFCCC5241 122 +\. + + +-- +-- TOC entry 5788 (class 0 OID 0) +-- Dependencies: 801 +-- Name: analysis_rent_listings_cartodb_id_seq; Type: SEQUENCE SET; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +SELECT pg_catalog.setval('analysis_rent_listings_cartodb_id_seq', 60, true); + + +-- +-- TOC entry 5618 (class 2606 OID 13870260) +-- Name: analysis_banks_pkey; Type: CONSTRAINT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +ALTER TABLE ONLY analysis_banks + ADD CONSTRAINT analysis_banks_pkey PRIMARY KEY (cartodb_id); + + +-- +-- TOC entry 5614 (class 2606 OID 13870243) +-- Name: analysis_rent_listings_pkey; Type: CONSTRAINT; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +ALTER TABLE ONLY analysis_rent_listings + ADD CONSTRAINT analysis_rent_listings_pkey PRIMARY KEY (cartodb_id); + + +-- +-- TOC entry 5619 (class 1259 OID 13870261) +-- Name: analysis_banks_the_geom_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_banks_the_geom_idx ON analysis_banks USING gist (the_geom); + + +-- +-- TOC entry 5620 (class 1259 OID 13870262) +-- Name: analysis_banks_the_geom_webmercator_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_banks_the_geom_webmercator_idx ON analysis_banks USING gist (the_geom_webmercator); + + +-- +-- TOC entry 5615 (class 1259 OID 13870244) +-- Name: analysis_rent_listings_the_geom_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_rent_listings_the_geom_idx ON analysis_rent_listings USING gist (the_geom); + + +-- +-- TOC entry 5616 (class 1259 OID 13870245) +-- Name: analysis_rent_listings_the_geom_webmercator_idx; Type: INDEX; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886; Tablespace: +-- + +CREATE INDEX analysis_rent_listings_the_geom_webmercator_idx ON analysis_rent_listings USING gist (the_geom_webmercator); + +-- +-- TOC entry 5783 (class 0 OID 0) +-- Dependencies: 804 +-- Name: analysis_banks; Type: ACL; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +REVOKE ALL ON TABLE analysis_banks FROM PUBLIC; +REVOKE ALL ON TABLE analysis_banks FROM :TESTUSER; +GRANT ALL ON TABLE analysis_banks TO :TESTUSER; +GRANT SELECT ON TABLE analysis_banks TO :PUBLICUSER; + + +-- +-- TOC entry 5785 (class 0 OID 0) +-- Dependencies: 802 +-- Name: analysis_rent_listings; Type: ACL; Schema: public; Owner: development_cartodb_user_359a4d9f-a063-4130-9674-799e90960886 +-- + +REVOKE ALL ON TABLE analysis_rent_listings FROM PUBLIC; +REVOKE ALL ON TABLE analysis_rent_listings FROM :TESTUSER; +GRANT ALL ON TABLE analysis_rent_listings TO :TESTUSER; +GRANT SELECT ON TABLE analysis_rent_listings TO :PUBLICUSER; + + +-- Completed on 2016-02-29 12:50:53 CET + +-- +-- PostgreSQL database dump complete +-- + +-- +GRANT SELECT, UPDATE, INSERT, DELETE ON cdb_analysis_catalog TO :TESTUSER; diff --git a/test/support/test-client.js b/test/support/test-client.js index 02128e36..1da653a7 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -12,9 +12,9 @@ var helper = require('./test_helper'); var CartodbWindshaft = require('../../lib/cartodb/server'); var serverOptions = require('../../lib/cartodb/server_options'); +serverOptions.analysis.batch.inlineExecution = true; var server = new CartodbWindshaft(serverOptions); - function TestClient(mapConfig, apiKey) { this.mapConfig = mapConfig; this.apiKey = apiKey; @@ -117,6 +117,111 @@ TestClient.prototype.getWidget = function(widgetName, params, callback) { ); }; +TestClient.prototype.getDataview = function(dataviewName, params, callback) { + var self = this; + + if (!callback) { + callback = params; + params = {}; + } + + var extraParams = {}; + if (this.apiKey) { + extraParams.api_key = this.apiKey; + } + if (params && params.filters) { + extraParams.filters = JSON.stringify(params.filters); + } + + var url = '/api/v1/map'; + if (Object.keys(extraParams).length > 0) { + url += '?' + qs.stringify(extraParams); + } + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + var parsedBody = JSON.parse(res.body); + var expectedDataviewsURLS = { + http: "/api/v1/map/" + parsedBody.layergroupid + "/dataview/" + dataviewName + }; + assert.ok(parsedBody.metadata.dataviews[dataviewName]); + assert.ok( + parsedBody.metadata.dataviews[dataviewName].url.http.match(expectedDataviewsURLS.http) + ); + return next(null, parsedBody.layergroupid); + } + ); + }, + function getDataviewResult(err, _layergroupId) { + assert.ifError(err); + + var next = this; + layergroupId = _layergroupId; + + var urlParams = { + own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 + }; + if (params && params.bbox) { + urlParams.bbox = params.bbox; + } + if (self.apiKey) { + urlParams.api_key = self.apiKey; + } + url = '/api/v1/map/' + layergroupId + '/dataview/' + dataviewName + '?' + qs.stringify(urlParams); + + assert.response(server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + next(null, JSON.parse(res.body)); + } + ); + }, + function finish(err, res) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + return callback(err, res); + } + ); +}; + TestClient.prototype.getTile = function(z, x, y, params, callback) { var self = this; @@ -198,7 +303,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { } }; - var isPng = format === 'png' || format === 'torque.png'; + var isPng = format.match(/png$/); if (isPng) { request.encoding = 'binary'; @@ -225,6 +330,53 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) { ); }; +TestClient.prototype.getLayergroup = function(expectedResponse, callback) { + var self = this; + + if (!callback) { + callback = expectedResponse; + expectedResponse = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + } + + var url = '/api/v1/map'; + + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(self.mapConfig) + }, + expectedResponse, + function(res, err) { + if (err) { + return callback(err); + } + + var parsedBody = JSON.parse(res.body); + + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + + return callback(null, parsedBody); + } + ); +}; + TestClient.prototype.drain = function(callback) { helper.deleteRedisKeys(this.keysToDelete, callback); };