Merge pull request #361 from CartoDB/mapconfig-extension

Mapconfig widgets extension
This commit is contained in:
Raul Ochoa
2016-01-18 14:39:35 +01:00
7 changed files with 608 additions and 16 deletions
+12 -2
View File
@@ -13,7 +13,17 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
'map_key',
'api_key',
'auth_token',
'callback'
'callback',
// widgets & filters
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'bins', // number
'start', // number
'end', // number
'column_type', // string
// widgets search
'q'
];
function BaseController(authApi, pgConnection) {
@@ -191,7 +201,7 @@ BaseController.prototype.sendError = function(req, res, err, label) {
var statusCode = findStatusCode(err);
debug('[%s ERROR] -- %d: %s', label, statusCode, err);
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
// If a callback was requested, force status to 200
if (req.query && req.query.callback) {
+69 -1
View File
@@ -17,6 +17,7 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {WidgetBackend} widgetBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
@@ -24,13 +25,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @constructor
*/
function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.widgetBackend = widgetBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
@@ -66,6 +68,72 @@ LayergroupController.prototype.register = function(app) {
app.get(app.base_url_mapconfig +
'/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
this.bbox.bind(this));
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:widgetName', cors(), userMiddleware,
this.widget.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:widgetName/search', cors(), userMiddleware,
this.widgetSearch.bind(this));
};
LayergroupController.prototype.widget = function(req, res) {
var self = this;
step(
function setupParams() {
self.req2params(req, this);
},
function retrieveList(err) {
assert.ifError(err);
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, req.context.user, self.userLimitsApi, req.params
);
self.widgetBackend.getWidget(mapConfigProvider, req.params, this);
},
function finish(err, tile, stats) {
req.profiler.add(stats || {});
if (err) {
self.sendError(req, res, err, 'GET WIDGET');
} else {
self.sendResponse(req, res, tile, 200);
}
}
);
};
LayergroupController.prototype.widgetSearch = function(req, res) {
var self = this;
step(
function setupParams() {
self.req2params(req, this);
},
function retrieveList(err) {
assert.ifError(err);
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, req.context.user, self.userLimitsApi, req.params
);
self.widgetBackend.search(mapConfigProvider, req.params, this);
},
function finish(err, tile, stats) {
req.profiler.add(stats || {});
if (err) {
self.sendError(req, res, err, 'GET WIDGET');
} else {
self.sendResponse(req, res, tile, 200);
}
}
);
};
LayergroupController.prototype.attributes = function(req, res) {
+34
View File
@@ -306,6 +306,9 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
@@ -324,3 +327,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
}
);
};
function addWidgetsUrl(username, layergroup) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
if (layer.widgets) {
Object.keys(layer.widgets).forEach(function(widgetName) {
var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
layer.widgets[widgetName].url = getUrls(username, resource);
});
}
return layer;
});
}
}
function getUrls(username, resource) {
var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
if (cdnUrl) {
return {
http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource,
https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource
};
} else {
var port = global.environment.port;
return {
http: 'http://' + username + '.' + 'localhost.lan:' + port + '/api/v1/map/' + resource
};
}
}
+2 -1
View File
@@ -131,7 +131,7 @@ module.exports = function(serverOptions) {
var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
rendererStatsReporter.start();
var attributesBackend = new windshaft.backend.Attributes(mapStore);
var attributesBackend = new windshaft.backend.Attributes();
var previewBackend = new windshaft.backend.Preview(rendererCache);
var tileBackend = new windshaft.backend.Tile(rendererCache);
var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
@@ -161,6 +161,7 @@ module.exports = function(serverOptions) {
tileBackend,
previewBackend,
attributesBackend,
new windshaft.backend.Widget(),
surrogateKeysCache,
userLimitsApi,
queryTablesApi,
+11 -11
View File
@@ -75,7 +75,7 @@
},
"type-is": {
"version": "1.6.10",
"from": "type-is@~1.6.6",
"from": "type-is@~1.6.10",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.10.tgz",
"dependencies": {
"media-typer": {
@@ -584,7 +584,7 @@
},
"inherits": {
"version": "2.0.1",
"from": "inherits@2",
"from": "inherits@~2.0.1",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
},
"isarray": {
@@ -633,7 +633,7 @@
"dependencies": {
"async": {
"version": "1.5.2",
"from": "async@1.x",
"from": "async@^1.4.0",
"resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz"
}
}
@@ -878,9 +878,9 @@
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
},
"windshaft": {
"version": "1.6.1",
"from": "windshaft@1.6.1",
"resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.6.1.tgz",
"version": "1.6.2",
"from": "https://github.com/CartoDB/Windshaft/tarball/master",
"resolved": "https://github.com/CartoDB/Windshaft/tarball/master",
"dependencies": {
"mapnik": {
"version": "1.4.15-cdb6",
@@ -3382,11 +3382,6 @@
}
}
}
},
"semver": {
"version": "5.0.3",
"from": "semver@~5.0.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz"
}
}
},
@@ -3467,6 +3462,11 @@
}
}
},
"semver": {
"version": "5.0.3",
"from": "semver@~5.0.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.0.3.tgz"
},
"carto": {
"version": "0.15.1-cdb1",
"from": "https://github.com/CartoDB/carto/tarball/0.15.1-cdb1",
+1 -1
View File
@@ -29,7 +29,7 @@
"node-statsd": "~0.0.7",
"underscore" : "~1.6.0",
"dot": "~1.0.2",
"windshaft": "1.6.1",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/master",
"step": "~0.0.6",
"queue-async": "~1.0.7",
"request": "~2.62.0",
+479
View File
@@ -0,0 +1,479 @@
var assert = require('../support/assert');
var step = require('step');
var qs = require('querystring');
var helper = require(__dirname + '/../support/test_helper');
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
describe('widgets', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
helper.deleteRedisKeys(keysToDelete, done);
});
function getWidget(mapConfig, widgetName, params, callback) {
if (!callback) {
callback = params;
params = {};
}
var url = '/api/v1/map';
if (params && params.filters) {
url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
}
var layergroupId;
step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
var expectedWidgetURLS = {
http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
};
assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
assert.ok(
parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
);
return next(null, parsedBody.layergroupid);
}
);
},
function getWidgetResult(err, _layergroupId) {
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
var urlParams = {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
};
if (params && params.bbox) {
urlParams.bbox = params.bbox;
}
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
assert.response(server,
{
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
next(null, res);
}
);
},
function finish(err, res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
return callback(err, res);
}
);
}
it("should expose layer list", function(done) {
var listWidgetMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
names: {
type: 'list',
options: {
columns: ['name']
}
}
}
}
}
]
};
getWidget(listWidgetMapConfig, 'names', function(err, res) {
if (err) {
return done(err);
}
var expectedList = [
{name:"Hawai"},
{name:"El Estocolmo"},
{name:"El Rey del Tallarín"},
{name:"El Lacón"},
{name:"El Pico"}
];
assert.deepEqual(JSON.parse(res.body).rows, expectedList);
done();
});
});
it("should expose layer histogram", function(done) {
var histogramMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
pop_max: {
type: 'histogram',
options: {
column: 'pop_max'
}
}
}
}
}
]
};
getWidget(histogramMapConfig, 'pop_max', function(err, res) {
if (err) {
return done(err);
}
var histogram = JSON.parse(res.body);
assert.ok(histogram.bins.length);
done();
});
});
describe('filters', function() {
describe('category', function() {
var aggregationMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
country_places_count: {
type: 'aggregation',
options: {
column: 'adm0_a3',
aggregation: 'count'
}
}
}
}
}
]
};
it("should expose an aggregation", function(done) {
getWidget(aggregationMapConfig, 'country_places_count', { own_filter: 0 }, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
assert.equal(aggregation.categories.length, 6);
assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false });
done();
});
});
it("should expose a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{country_places_count: {accept: ['CAN']}}
]
}
};
getWidget(aggregationMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
assert.equal(aggregation.categories.length, 1);
assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false });
done();
});
});
});
describe('range', function() {
var histogramMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
country_places_histogram: {
type: 'histogram',
options: {
column: 'pop_max'
}
}
}
}
}
]
};
it("should expose an histogram", function(done) {
getWidget(histogramMapConfig, 'country_places_histogram', { own_filter: 0 }, function(err, res) {
if (err) {
return done(err);
}
var histogram = JSON.parse(res.body);
// notice min value
assert.deepEqual(
histogram.bins[0],
{ bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 }
);
done();
});
});
it("should expose a filtered histogram", function(done) {
var params = {
filters: {
layers: [
{
country_places_histogram: { min: 4000000 }
}
]
}
};
getWidget(histogramMapConfig, 'country_places_histogram', params, function(err, res) {
if (err) {
return done(err);
}
var histogram = JSON.parse(res.body);
// notice min value
assert.deepEqual(histogram.bins[0], {
bin: 0,
freq: 62,
min: 4000000,
max: 9276403,
avg: 5815009.596774193
});
done();
});
});
});
describe('combine widget filters', function() {
var combinedWidgetsMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
country_places_count: {
type: 'aggregation',
options: {
column: 'adm0_a3',
aggregation: 'count'
}
},
country_places_histogram: {
type: 'histogram',
options: {
column: 'pop_max'
}
}
}
}
}
]
};
it("should expose a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{
country_places_count: { reject: ['CHN'] }
}
]
}
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
it("should expose a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{
country_places_count: { reject: ['CHN'] },
country_places_histogram: { min: 7000000 }
}
]
}
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
it("should allow to filter by bounding box a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{
country_places_histogram: { min: 50000 }
}
]
},
bbox: '-20,0,45,60'
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) {
var params = {
filters: {
layers: [
{
country_places_count: { reject: ['RUS'] },
country_places_histogram: { min: 50000 }
}
]
},
bbox: '-20,0,45,60'
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
});
});
});