diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 24c06875..ac3ac11c 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -13,7 +13,17 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ 'map_key', 'api_key', 'auth_token', - 'callback' + 'callback', + // widgets & filters + 'filters', // json + 'own_filter', // 0, 1 + 'bbox', // w,s,e,n + 'bins', // number + 'start', // number + 'end', // number + 'column_type', // string + // widgets search + 'q' ]; function BaseController(authApi, pgConnection) { @@ -191,7 +201,7 @@ BaseController.prototype.sendError = function(req, res, err, label) { var statusCode = findStatusCode(err); - debug('[%s ERROR] -- %d: %s', label, statusCode, err); + debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack); // If a callback was requested, force status to 200 if (req.query && req.query.callback) { diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 9194c05f..0a7e7698 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -17,6 +17,7 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry'); * @param {TileBackend} tileBackend * @param {PreviewBackend} previewBackend * @param {AttributesBackend} attributesBackend + * @param {WidgetBackend} widgetBackend * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {QueryTablesApi} queryTablesApi @@ -24,13 +25,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry'); * @constructor */ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, - surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) { + widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) { BaseController.call(this, authApi, pgConnection); this.mapStore = mapStore; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.attributesBackend = attributesBackend; + this.widgetBackend = widgetBackend; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.queryTablesApi = queryTablesApi; @@ -66,6 +68,72 @@ LayergroupController.prototype.register = function(app) { app.get(app.base_url_mapconfig + '/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware, this.bbox.bind(this)); + + // Undocumented/non-supported API endpoint methods. + // Use at your own peril. + app.get(app.base_url_mapconfig + + '/:token/:layer/widget/:widgetName', cors(), userMiddleware, + this.widget.bind(this)); + + app.get(app.base_url_mapconfig + + '/:token/:layer/widget/:widgetName/search', cors(), userMiddleware, + this.widgetSearch.bind(this)); +}; + +LayergroupController.prototype.widget = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function retrieveList(err) { + assert.ifError(err); + + var mapConfigProvider = new MapStoreMapConfigProvider( + self.mapStore, req.context.user, self.userLimitsApi, req.params + ); + self.widgetBackend.getWidget(mapConfigProvider, req.params, this); + }, + function finish(err, tile, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET WIDGET'); + } else { + self.sendResponse(req, res, tile, 200); + } + } + ); + +}; + +LayergroupController.prototype.widgetSearch = function(req, res) { + var self = this; + + step( + function setupParams() { + self.req2params(req, this); + }, + function retrieveList(err) { + assert.ifError(err); + + var mapConfigProvider = new MapStoreMapConfigProvider( + self.mapStore, req.context.user, self.userLimitsApi, req.params + ); + self.widgetBackend.search(mapConfigProvider, req.params, this); + }, + function finish(err, tile, stats) { + req.profiler.add(stats || {}); + + if (err) { + self.sendError(req, res, err, 'GET WIDGET'); + } else { + self.sendResponse(req, res, tile, 200); + } + } + ); + }; LayergroupController.prototype.attributes = function(req, res) { diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f6391db1..9fad27f6 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -306,6 +306,9 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); + // TODO this should take into account several URL patterns + addWidgetsUrl(username, layergroup); + if (req.method === 'GET') { var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables); var ttl = global.environment.varnish.layergroupTtl || 86400; @@ -324,3 +327,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la } ); }; + +function addWidgetsUrl(username, layergroup) { + + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (layer.widgets) { + Object.keys(layer.widgets).forEach(function(widgetName) { + var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName; + layer.widgets[widgetName].url = getUrls(username, resource); + }); + } + return layer; + }); + } + +} + +function getUrls(username, resource) { + var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url; + if (cdnUrl) { + return { + http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource, + https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource + }; + } else { + var port = global.environment.port; + return { + http: 'http://' + username + '.' + 'localhost.lan:' + port + '/api/v1/map/' + resource + }; + } +} diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e7824d2b..e4079574 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -131,7 +131,7 @@ module.exports = function(serverOptions) { var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); rendererStatsReporter.start(); - var attributesBackend = new windshaft.backend.Attributes(mapStore); + var attributesBackend = new windshaft.backend.Attributes(); var previewBackend = new windshaft.backend.Preview(rendererCache); var tileBackend = new windshaft.backend.Tile(rendererCache); var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); @@ -161,6 +161,7 @@ module.exports = function(serverOptions) { tileBackend, previewBackend, attributesBackend, + new windshaft.backend.Widget(), surrogateKeysCache, userLimitsApi, queryTablesApi, diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 94d2ec34..3ac81ae1 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -75,7 +75,7 @@ }, "type-is": { "version": "1.6.10", - "from": "type-is@~1.6.6", + "from": "type-is@~1.6.10", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.10.tgz", "dependencies": { "media-typer": { @@ -584,7 +584,7 @@ }, "inherits": { "version": "2.0.1", - "from": "inherits@2", + "from": "inherits@~2.0.1", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "isarray": { @@ -633,7 +633,7 @@ "dependencies": { "async": { "version": "1.5.2", - "from": "async@1.x", + "from": "async@^1.4.0", "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz" } } @@ -878,9 +878,9 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "1.6.1", - "from": "windshaft@1.6.1", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.6.1.tgz", + "version": "1.6.2", + "from": "https://github.com/CartoDB/Windshaft/tarball/master", + "resolved": "https://github.com/CartoDB/Windshaft/tarball/master", "dependencies": { "mapnik": { "version": "1.4.15-cdb6", @@ -3382,11 +3382,6 @@ } } } - }, - "semver": { - "version": "5.0.3", - "from": "semver@~5.0.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz" } } }, @@ -3467,6 +3462,11 @@ } } }, + "semver": { + "version": "5.0.3", + "from": "semver@~5.0.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz" + }, "carto": { "version": "0.15.1-cdb1", "from": "https://github.com/CartoDB/carto/tarball/0.15.1-cdb1", diff --git a/package.json b/package.json index 27972e43..7f539bef 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "node-statsd": "~0.0.7", "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "1.6.1", + "windshaft": "https://github.com/CartoDB/Windshaft/tarball/master", "step": "~0.0.6", "queue-async": "~1.0.7", "request": "~2.62.0", diff --git a/test/acceptance/widgets.js b/test/acceptance/widgets.js new file mode 100644 index 00000000..2f4483f5 --- /dev/null +++ b/test/acceptance/widgets.js @@ -0,0 +1,479 @@ +var assert = require('../support/assert'); +var step = require('step'); +var qs = require('querystring'); + +var helper = require(__dirname + '/../support/test_helper'); +var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); + +var CartodbWindshaft = require('../../lib/cartodb/server'); +var serverOptions = require('../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); +server.setMaxListeners(0); + + +describe('widgets', function() { + + var keysToDelete; + + beforeEach(function() { + keysToDelete = {}; + }); + + afterEach(function(done) { + helper.deleteRedisKeys(keysToDelete, done); + }); + + function getWidget(mapConfig, widgetName, params, callback) { + if (!callback) { + callback = params; + params = {}; + } + + var url = '/api/v1/map'; + if (params && params.filters) { + url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) }); + } + + var layergroupId; + step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: url, + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(mapConfig) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + var parsedBody = JSON.parse(res.body); + var expectedWidgetURLS = { + http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName + }; + assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]); + assert.ok( + parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http) + ); + return next(null, parsedBody.layergroupid); + } + ); + }, + function getWidgetResult(err, _layergroupId) { + assert.ifError(err); + + var next = this; + layergroupId = _layergroupId; + + var urlParams = { + own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 + }; + if (params && params.bbox) { + urlParams.bbox = params.bbox; + } + url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams); + + assert.response(server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost' + } + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return next(err); + } + + next(null, res); + } + ); + }, + function finish(err, res) { + keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + return callback(err, res); + } + ); + } + + + it("should expose layer list", function(done) { + + var listWidgetMapConfig = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + widgets: { + names: { + type: 'list', + options: { + columns: ['name'] + } + } + } + } + } + ] + }; + + getWidget(listWidgetMapConfig, 'names', function(err, res) { + if (err) { + return done(err); + } + + var expectedList = [ + {name:"Hawai"}, + {name:"El Estocolmo"}, + {name:"El Rey del Tallarín"}, + {name:"El Lacón"}, + {name:"El Pico"} + ]; + assert.deepEqual(JSON.parse(res.body).rows, expectedList); + + done(); + }); + }); + + it("should expose layer histogram", function(done) { + var histogramMapConfig = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + widgets: { + pop_max: { + type: 'histogram', + options: { + column: 'pop_max' + } + } + } + } + } + ] + }; + getWidget(histogramMapConfig, 'pop_max', function(err, res) { + if (err) { + return done(err); + } + + var histogram = JSON.parse(res.body); + assert.ok(histogram.bins.length); + + done(); + }); + }); + + describe('filters', function() { + + describe('category', function() { + var aggregationMapConfig = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + widgets: { + country_places_count: { + type: 'aggregation', + options: { + column: 'adm0_a3', + aggregation: 'count' + } + } + } + } + } + ] + }; + + it("should expose an aggregation", function(done) { + getWidget(aggregationMapConfig, 'country_places_count', { own_filter: 0 }, function(err, res) { + if (err) { + return done(err); + } + + var aggregation = JSON.parse(res.body); + assert.equal(aggregation.categories.length, 6); + assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false }); + + done(); + }); + }); + + it("should expose a filtered aggregation", function(done) { + var params = { + filters: { + layers: [ + {country_places_count: {accept: ['CAN']}} + ] + } + }; + getWidget(aggregationMapConfig, 'country_places_count', params, function(err, res) { + if (err) { + return done(err); + } + + var aggregation = JSON.parse(res.body); + assert.equal(aggregation.categories.length, 1); + assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false }); + + done(); + }); + }); + }); + + describe('range', function() { + var histogramMapConfig = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + widgets: { + country_places_histogram: { + type: 'histogram', + options: { + column: 'pop_max' + } + } + } + } + } + ] + }; + + it("should expose an histogram", function(done) { + getWidget(histogramMapConfig, 'country_places_histogram', { own_filter: 0 }, function(err, res) { + if (err) { + return done(err); + } + + var histogram = JSON.parse(res.body); + // notice min value + assert.deepEqual( + histogram.bins[0], + { bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 } + ); + + done(); + }); + }); + + it("should expose a filtered histogram", function(done) { + var params = { + filters: { + layers: [ + { + country_places_histogram: { min: 4000000 } + } + ] + } + }; + getWidget(histogramMapConfig, 'country_places_histogram', params, function(err, res) { + if (err) { + return done(err); + } + + var histogram = JSON.parse(res.body); + // notice min value + assert.deepEqual(histogram.bins[0], { + bin: 0, + freq: 62, + min: 4000000, + max: 9276403, + avg: 5815009.596774193 + }); + + done(); + }); + }); + }); + + describe('combine widget filters', function() { + var combinedWidgetsMapConfig = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }', + cartocss_version: '2.3.0', + widgets: { + country_places_count: { + type: 'aggregation', + options: { + column: 'adm0_a3', + aggregation: 'count' + } + }, + country_places_histogram: { + type: 'histogram', + options: { + column: 'pop_max' + } + } + } + } + } + ] + }; + + it("should expose a filtered aggregation", function(done) { + var params = { + filters: { + layers: [ + { + country_places_count: { reject: ['CHN'] } + } + ] + } + }; + getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) { + if (err) { + return done(err); + } + + var aggregation = JSON.parse(res.body); + + // first one would be CHN if reject filter wasn't applied + assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false }); + + // confirm 'CHN' was filtered out (reject) + assert.equal(aggregation.categories.reduce(function(sum, row) { + return sum + (row.category === 'CHN' ? 1 : 0); + }, 0), 0); + + done(); + }); + }); + + it("should expose a filtered aggregation", function(done) { + var params = { + filters: { + layers: [ + { + country_places_count: { reject: ['CHN'] }, + country_places_histogram: { min: 7000000 } + } + ] + } + }; + getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) { + if (err) { + return done(err); + } + + var aggregation = JSON.parse(res.body); + + // first one would be CHN if reject filter wasn't applied + assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false }); + + // confirm 'CHN' was filtered out (reject) + assert.equal(aggregation.categories.reduce(function(sum, row) { + return sum + (row.category === 'CHN' ? 1 : 0); + }, 0), 0); + + done(); + }); + }); + + it("should allow to filter by bounding box a filtered aggregation", function(done) { + var params = { + filters: { + layers: [ + { + country_places_histogram: { min: 50000 } + } + ] + }, + bbox: '-20,0,45,60' + }; + getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) { + if (err) { + return done(err); + } + + var aggregation = JSON.parse(res.body); + + // first one would be CHN if reject filter wasn't applied + assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false }); + + // confirm 'CHN' was filtered out (reject) + assert.equal(aggregation.categories.reduce(function(sum, row) { + return sum + (row.category === 'CHN' ? 1 : 0); + }, 0), 0); + + done(); + }); + }); + + it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) { + var params = { + filters: { + layers: [ + { + country_places_count: { reject: ['RUS'] }, + country_places_histogram: { min: 50000 } + } + ] + }, + bbox: '-20,0,45,60' + }; + getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) { + if (err) { + return done(err); + } + + var aggregation = JSON.parse(res.body); + + // first one would be CHN if reject filter wasn't applied + assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false }); + + // confirm 'CHN' was filtered out (reject) + assert.equal(aggregation.categories.reduce(function(sum, row) { + return sum + (row.category === 'CHN' ? 1 : 0); + }, 0), 0); + + done(); + }); + }); + }); + }); + +});