cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into geojson-renderer

Browse Source 
Conflicts:
	npm-shrinkwrap.json
	package.json
This commit is contained in:
Raul Ochoa
2016-01-18 14:52:01 +01:00
parent 3e601cc2e6 8b50097a12
commit e72bd77265
9 changed files with 663 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/MultiLayer-API.md
+2 -2
View File
@@ -17,8 +17,8 @@ Windshaft-CartoDB adds the following attributes in the response object
```json
{
http: 'http://cdn_url.com/'
https: 'https://secure.cdn_url.com/'
"http": "http://cdn_url.com/",
"https": "https://secure.cdn_url.com/"
}
```
lib/cartodb/controllers/base.js
+12 -2
View File
@@ -13,7 +13,17 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [
'map_key',
'api_key',
'auth_token',
'callback'
'callback',
// widgets & filters
'filters', // json
'own_filter', // 0, 1
'bbox', // w,s,e,n
'bins', // number
'start', // number
'end', // number
'column_type', // string
// widgets search
'q'
];
function BaseController(authApi, pgConnection) {
@@ -191,7 +201,7 @@ BaseController.prototype.sendError = function(req, res, err, label) {
var statusCode = findStatusCode(err);
debug('[%s ERROR] -- %d: %s', label, statusCode, err);
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
// If a callback was requested, force status to 200
if (req.query && req.query.callback) {
lib/cartodb/controllers/layergroup.js
+69 -1
View File
@@ -17,6 +17,7 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {WidgetBackend} widgetBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {QueryTablesApi} queryTablesApi
@@ -24,13 +25,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
* @constructor
*/
function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
BaseController.call(this, authApi, pgConnection);
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.widgetBackend = widgetBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.queryTablesApi = queryTablesApi;
@@ -66,6 +68,72 @@ LayergroupController.prototype.register = function(app) {
app.get(app.base_url_mapconfig +
'/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', cors(), userMiddleware,
this.bbox.bind(this));
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:widgetName', cors(), userMiddleware,
this.widget.bind(this));
app.get(app.base_url_mapconfig +
'/:token/:layer/widget/:widgetName/search', cors(), userMiddleware,
this.widgetSearch.bind(this));
};
LayergroupController.prototype.widget = function(req, res) {
var self = this;
step(
function setupParams() {
self.req2params(req, this);
},
function retrieveList(err) {
assert.ifError(err);
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, req.context.user, self.userLimitsApi, req.params
);
self.widgetBackend.getWidget(mapConfigProvider, req.params, this);
},
function finish(err, tile, stats) {
req.profiler.add(stats || {});
if (err) {
self.sendError(req, res, err, 'GET WIDGET');
} else {
self.sendResponse(req, res, tile, 200);
}
}
);
};
LayergroupController.prototype.widgetSearch = function(req, res) {
var self = this;
step(
function setupParams() {
self.req2params(req, this);
},
function retrieveList(err) {
assert.ifError(err);
var mapConfigProvider = new MapStoreMapConfigProvider(
self.mapStore, req.context.user, self.userLimitsApi, req.params
);
self.widgetBackend.search(mapConfigProvider, req.params, this);
},
function finish(err, tile, stats) {
req.profiler.add(stats || {});
if (err) {
self.sendError(req, res, err, 'GET WIDGET');
} else {
self.sendResponse(req, res, tile, 200);
}
}
);
};
LayergroupController.prototype.attributes = function(req, res) {
lib/cartodb/controllers/map.js
+34
View File
@@ -306,6 +306,9 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
// TODO this should take into account several URL patterns
addWidgetsUrl(username, layergroup);
if (req.method === 'GET') {
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
var ttl = global.environment.varnish.layergroupTtl || 86400;
@@ -324,3 +327,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
}
);
};
function addWidgetsUrl(username, layergroup) {
if (layergroup.metadata && Array.isArray(layergroup.metadata.layers)) {
layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) {
if (layer.widgets) {
Object.keys(layer.widgets).forEach(function(widgetName) {
var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName;
layer.widgets[widgetName].url = getUrls(username, resource);
});
}
return layer;
});
}
}
function getUrls(username, resource) {
var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url;
if (cdnUrl) {
return {
http: 'http://' + cdnUrl.http + '/' + username + '/api/v1/map/' + resource,
https: 'https://' + cdnUrl.https + '/' + username + '/api/v1/map/' + resource
};
} else {
var port = global.environment.port;
return {
http: 'http://' + username + '.' + 'localhost.lan:' + port + '/api/v1/map/' + resource
};
}
}
lib/cartodb/server.js
+2 -1
View File
@@ -131,7 +131,7 @@ module.exports = function(serverOptions) {
var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
rendererStatsReporter.start();
var attributesBackend = new windshaft.backend.Attributes(mapStore);
var attributesBackend = new windshaft.backend.Attributes();
var previewBackend = new windshaft.backend.Preview(rendererCache);
var tileBackend = new windshaft.backend.Tile(rendererCache);
var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
@@ -161,6 +161,7 @@ module.exports = function(serverOptions) {
tileBackend,
previewBackend,
attributesBackend,
new windshaft.backend.Widget(),
surrogateKeysCache,
userLimitsApi,
queryTablesApi,
npm-shrinkwrap.json
Generated
+61 -10
View File
@@ -192,7 +192,7 @@
"dependencies": {
"mime-types": {
"version": "2.1.9",
"from": "mime-types@~2.1.6",
"from": "mime-types@~2.1.8",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
@@ -272,9 +272,9 @@
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.0.tgz"
},
"methods": {
"version": "1.1.1",
"version": "1.1.2",
"from": "methods@~1.1.1",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.1.tgz"
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
},
"on-finished": {
"version": "2.3.0",
@@ -289,9 +289,9 @@
}
},
"parseurl": {
"version": "1.3.0",
"version": "1.3.1",
"from": "parseurl@~1.3.0",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.0.tgz"
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz"
},
"path-to-regexp": {
"version": "0.1.7",
@@ -365,9 +365,60 @@
}
},
"serve-static": {
"version": "1.10.0",
"version": "1.10.1",
"from": "serve-static@~1.10.0",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.0.tgz"
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.1.tgz",
"dependencies": {
"escape-html": {
"version": "1.0.3",
"from": "escape-html@~1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz"
},
"send": {
"version": "0.13.1",
"from": "send@0.13.1",
"resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
"dependencies": {
"depd": {
"version": "1.1.0",
"from": "depd@~1.1.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
},
"destroy": {
"version": "1.0.4",
"from": "destroy@~1.0.4",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
},
"http-errors": {
"version": "1.3.1",
"from": "http-errors@~1.3.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
"dependencies": {
"inherits": {
"version": "2.0.1",
"from": "inherits@~2.0.1",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
}
}
},
"mime": {
"version": "1.3.4",
"from": "mime@1.3.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
},
"ms": {
"version": "0.7.1",
"from": "ms@0.7.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
},
"statuses": {
"version": "1.2.1",
"from": "statuses@~1.2.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
}
}
}
}
},
"type-is": {
"version": "1.6.10",
@@ -381,7 +432,7 @@
},
"mime-types": {
"version": "2.1.9",
"from": "mime-types@~2.1.6",
"from": "mime-types@~2.1.8",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz",
"dependencies": {
"mime-db": {
@@ -774,9 +825,9 @@
}
},
"is-my-json-valid": {
"version": "2.12.3",
"version": "2.12.4",
"from": "is-my-json-valid@^2.12.0",
"resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.3.tgz",
"resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.12.4.tgz",
"dependencies": {
"generate-function": {
"version": "2.0.0",
test/acceptance/ported/multilayer_error_cases.js
-40
View File
@@ -64,46 +64,6 @@ describe('multilayer error cases', function() {
});
});
// See https://github.com/CartoDB/Windshaft/issues/154
it("mapnik tokens cannot be used with attributes service", function(done) {
var layergroup = {
version: '1.1.0',
layers: [
{ options: {
sql: 'select cartodb_id, 1 as n, the_geom, !bbox! as b from test_table limit 1',
cartocss: '#layer { marker-fill:red }',
cartocss_version: '2.0.1',
attributes: { id:'cartodb_id', columns:['n'] },
geom_column: 'the_geom'
} }
]
};
step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json; charset=utf-8' },
data: JSON.stringify(layergroup)
}, {}, function(res, err) { next(err, res); });
},
function do_check(err, res) {
assert.equal(res.statusCode, 400, res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
assert.equal(parsed.errors.length, 1);
var msg = parsed.errors[0];
assert.ok(msg.match(/Attribute service cannot be activated/), msg);
return null;
},
function finish(err) {
done(err);
}
);
});
it("layergroup with no cartocss_version", function(done) {
var layergroup = {
version: '1.0.0',
test/acceptance/ported/torque.js
+4 -2
View File
@@ -216,8 +216,10 @@ describe('torque', function() {
assert.ok(tm0,
'No layer 0 in "torque" in metadata:' + JSON.stringify(tm));
var expectedTorqueMetadata = {"start":0,"end":86400000,"data_steps":2,"column_type":"date"};
assert.deepEqual(tm0, expectedTorqueMetadata);
assert.deepEqual(meta.layers[0].meta, expectedTorqueMetadata);
Object.keys(expectedTorqueMetadata).forEach(function(k) {
assert.equal(tm0[k], expectedTorqueMetadata[k]);
assert.equal(meta.layers[0].meta[k], expectedTorqueMetadata[k]);
});
return null;
},
function do_get_tile(err)
test/acceptance/widgets.js
+479
View File
@@ -0,0 +1,479 @@
var assert = require('../support/assert');
var step = require('step');
var qs = require('querystring');
var helper = require(__dirname + '/../support/test_helper');
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
describe('widgets', function() {
var keysToDelete;
beforeEach(function() {
keysToDelete = {};
});
afterEach(function(done) {
helper.deleteRedisKeys(keysToDelete, done);
});
function getWidget(mapConfig, widgetName, params, callback) {
if (!callback) {
callback = params;
params = {};
}
var url = '/api/v1/map';
if (params && params.filters) {
url += '?' + qs.stringify({ filters: JSON.stringify(params.filters) });
}
var layergroupId;
step(
function createLayergroup() {
var next = this;
assert.response(server,
{
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(mapConfig)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
var parsedBody = JSON.parse(res.body);
var expectedWidgetURLS = {
http: "/api/v1/map/" + parsedBody.layergroupid + "/0/widget/" + widgetName
};
assert.ok(parsedBody.metadata.layers[0].widgets[widgetName]);
assert.ok(
parsedBody.metadata.layers[0].widgets[widgetName].url.http.match(expectedWidgetURLS.http)
);
return next(null, parsedBody.layergroupid);
}
);
},
function getWidgetResult(err, _layergroupId) {
assert.ifError(err);
var next = this;
layergroupId = _layergroupId;
var urlParams = {
own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1
};
if (params && params.bbox) {
urlParams.bbox = params.bbox;
}
url = '/api/v1/map/' + layergroupId + '/0/widget/' + widgetName + '?' + qs.stringify(urlParams);
assert.response(server,
{
url: url,
method: 'GET',
headers: {
host: 'localhost'
}
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
if (err) {
return next(err);
}
next(null, res);
}
);
},
function finish(err, res) {
keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
return callback(err, res);
}
);
}
it("should expose layer list", function(done) {
var listWidgetMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
names: {
type: 'list',
options: {
columns: ['name']
}
}
}
}
}
]
};
getWidget(listWidgetMapConfig, 'names', function(err, res) {
if (err) {
return done(err);
}
var expectedList = [
{name:"Hawai"},
{name:"El Estocolmo"},
{name:"El Rey del Tallarín"},
{name:"El Lacón"},
{name:"El Pico"}
];
assert.deepEqual(JSON.parse(res.body).rows, expectedList);
done();
});
});
it("should expose layer histogram", function(done) {
var histogramMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
pop_max: {
type: 'histogram',
options: {
column: 'pop_max'
}
}
}
}
}
]
};
getWidget(histogramMapConfig, 'pop_max', function(err, res) {
if (err) {
return done(err);
}
var histogram = JSON.parse(res.body);
assert.ok(histogram.bins.length);
done();
});
});
describe('filters', function() {
describe('category', function() {
var aggregationMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
country_places_count: {
type: 'aggregation',
options: {
column: 'adm0_a3',
aggregation: 'count'
}
}
}
}
}
]
};
it("should expose an aggregation", function(done) {
getWidget(aggregationMapConfig, 'country_places_count', { own_filter: 0 }, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
assert.equal(aggregation.categories.length, 6);
assert.deepEqual(aggregation.categories[0], { value: 769, category: 'USA', agg: false });
done();
});
});
it("should expose a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{country_places_count: {accept: ['CAN']}}
]
}
};
getWidget(aggregationMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
assert.equal(aggregation.categories.length, 1);
assert.deepEqual(aggregation.categories[0], { value: 256, category: 'CAN', agg: false });
done();
});
});
});
describe('range', function() {
var histogramMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
country_places_histogram: {
type: 'histogram',
options: {
column: 'pop_max'
}
}
}
}
}
]
};
it("should expose an histogram", function(done) {
getWidget(histogramMapConfig, 'country_places_histogram', { own_filter: 0 }, function(err, res) {
if (err) {
return done(err);
}
var histogram = JSON.parse(res.body);
// notice min value
assert.deepEqual(
histogram.bins[0],
{ bin: 0, freq: 6497, min: 0, max: 742572, avg: 113511.16823149147 }
);
done();
});
});
it("should expose a filtered histogram", function(done) {
var params = {
filters: {
layers: [
{
country_places_histogram: { min: 4000000 }
}
]
}
};
getWidget(histogramMapConfig, 'country_places_histogram', params, function(err, res) {
if (err) {
return done(err);
}
var histogram = JSON.parse(res.body);
// notice min value
assert.deepEqual(histogram.bins[0], {
bin: 0,
freq: 62,
min: 4000000,
max: 9276403,
avg: 5815009.596774193
});
done();
});
});
});
describe('combine widget filters', function() {
var combinedWidgetsMapConfig = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
cartocss_version: '2.3.0',
widgets: {
country_places_count: {
type: 'aggregation',
options: {
column: 'adm0_a3',
aggregation: 'count'
}
},
country_places_histogram: {
type: 'histogram',
options: {
column: 'pop_max'
}
}
}
}
}
]
};
it("should expose a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{
country_places_count: { reject: ['CHN'] }
}
]
}
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 769, category: "USA", agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
it("should expose a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{
country_places_count: { reject: ['CHN'] },
country_places_histogram: { min: 7000000 }
}
]
}
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 4, category: 'IND', agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
it("should allow to filter by bounding box a filtered aggregation", function(done) {
var params = {
filters: {
layers: [
{
country_places_histogram: { min: 50000 }
}
]
},
bbox: '-20,0,45,60'
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 96, category: "RUS", agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
it("should allow to filter by bounding box a filtered aggregation, with reject", function(done) {
var params = {
filters: {
layers: [
{
country_places_count: { reject: ['RUS'] },
country_places_histogram: { min: 50000 }
}
]
},
bbox: '-20,0,45,60'
};
getWidget(combinedWidgetsMapConfig, 'country_places_count', params, function(err, res) {
if (err) {
return done(err);
}
var aggregation = JSON.parse(res.body);
// first one would be CHN if reject filter wasn't applied
assert.deepEqual(aggregation.categories[0], { value: 77, category: "TUR", agg: false });
// confirm 'CHN' was filtered out (reject)
assert.equal(aggregation.categories.reduce(function(sum, row) {
return sum + (row.category === 'CHN' ? 1 : 0);
}, 0), 0);
done();
});
});
});
});
});