cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #683 from CartoDB/664-layergroup-stats

Browse Source 
Use windshaft-stats to extract info about layer metadata
This commit is contained in:
Daniel
2017-05-31 11:58:00 +02:00
committed by GitHub
parent 45471597bb 5a1623b667
commit 4f7ec0dd4d
20 changed files with 943 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -10,3 +10,4 @@ pids/
redis.pid
*.log
coverage/
.DS_Store

4
NEWS.md
View File

@@ -1,9 +1,11 @@
# Changelog
## 3.8.1
## 3.9.0
Released 2017-mm-dd
Announcements:
- Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1).
- Add support to retrieve info about layer stats in map instantiation.
- Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2).
- Remove promise polyfill from turbo-carto adapter

16
lib/cartodb/backends/layer-stats/empty-layer-stats.js Normal file
View File

@@ -0,0 +1,16 @@
function EmptyLayerStats(types) {
this._types = types || {};
}
EmptyLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
EmptyLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
setImmediate(function() {
callback(null, {});
});
};
module.exports = EmptyLayerStats;

23
lib/cartodb/backends/layer-stats/factory.js Normal file
View File

@@ -0,0 +1,23 @@
var LayerStats = require('./layer-stats');
var EmptyLayerStats = require('./empty-layer-stats');
var MapnikLayerStats = require('./mapnik-layer-stats');
var TorqueLayerStats = require('./torque-layer-stats');
module.exports = function LayerStatsFactory(type) {
var layerStatsIterator = [];
var selectedType = type || 'ALL';
if (selectedType === 'ALL') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
layerStatsIterator.push(new MapnikLayerStats());
layerStatsIterator.push(new TorqueLayerStats());
} else if (selectedType === 'mapnik') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
layerStatsIterator.push(new MapnikLayerStats());
} else if (selectedType === 'torque') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
layerStatsIterator.push(new TorqueLayerStats());
}
return new LayerStats(layerStatsIterator);
};

45
lib/cartodb/backends/layer-stats/layer-stats.js Normal file
View File

@@ -0,0 +1,45 @@
var queue = require('queue-async');
function LayerStats(layerStatsIterator) {
this.layerStatsIterator = layerStatsIterator;
}
LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
var self = this;
var stats = [];
if (!mapConfig.getLayers().length) {
return callback(null, stats);
}
var metaQueue = queue(mapConfig.getLayers().length);
mapConfig.getLayers().forEach(function (layer, layerId) {
var layerType = mapConfig.layerType(layerId);
for (var i = 0; i < self.layerStatsIterator.length; i++) {
if (self.layerStatsIterator[i].is(layerType)) {
var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
metaQueue.defer(getStats, layer, dbConnection);
break;
}
}
});
metaQueue.awaitAll(function (err, results) {
if (err) {
return callback(err);
}
if (!results) {
return callback(null, null);
}
mapConfig.getLayers().forEach(function (layer, layerIndex) {
stats[layerIndex] = results[layerIndex];
});
return callback(err, stats);
});
};
module.exports = LayerStats;

28
lib/cartodb/backends/layer-stats/mapnik-layer-stats.js Normal file
View File

@@ -0,0 +1,28 @@
var queryUtils = require('../../utils/query-utils');
function MapnikLayerStats () {
this._types = {
mapnik: true,
cartodb: true
};
}
MapnikLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
MapnikLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
// This query would gather stats for postgresql table if not exists
dbConnection.query(queryRowCountSql, function (err, res) {
if (err) {
return callback(null, {estimatedFeatureCount: -1});
} else {
// We decided that the relation is 1 row == 1 feature
return callback(null, {estimatedFeatureCount: res.rows[0].rows});
}
});
};
module.exports = MapnikLayerStats;

16
lib/cartodb/backends/layer-stats/torque-layer-stats.js Normal file
View File

@@ -0,0 +1,16 @@
function TorqueLayerStats() {
this._types = {
torque: true
};
}
TorqueLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
TorqueLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
return callback(null, {});
};
module.exports = TorqueLayerStats;

16
lib/cartodb/backends/stats.js Normal file
View File

@@ -0,0 +1,16 @@
var layerStats = require('./layer-stats/factory');
function StatsBackend() {
}
module.exports = StatsBackend;
StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
var enabledFeatures = global.environment.enabledFeatures;
var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
if (layerStatsEnabled) {
layerStats().getStats(mapConfig, dbConnection, callback);
} else {
return callback(null, []);
}
};

34
lib/cartodb/controllers/map.js
View File

@@ -20,6 +20,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
/**
* @param {AuthApi} authApi
* @param {PgConnection} pgConnection
@@ -30,10 +31,12 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
statsBackend) {
BaseController.call(this, authApi, pgConnection);
@@ -47,6 +50,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.mapConfigAdapter = mapConfigAdapter;
this.resourceLocator = new ResourceLocator(global.environment);
this.statsBackend = statsBackend;
}
util.inherits(MapController, BaseController);
@@ -214,7 +219,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
var mapConfigProvider;
var mapConfig;
step(
function setupParams(){
self.req2params(req, this);
@@ -249,7 +253,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
},
function afterLayergroupCreate(err, layergroup) {
assert.ifError(err);
self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
self.afterLayergroupCreate(req, res, mapConfig, layergroup,
mapConfigProvider.analysesResults,
this);
},
function finishTemplateInstantiation(err, layergroup) {
if (err) {
@@ -272,7 +278,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
);
};
MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
MapController.prototype.afterLayergroupCreate =
function(req, res, mapconfig, layergroup, analysesResults, callback) {
var self = this;
var username = req.context.user;
@@ -318,6 +325,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
var dbName = req.params.dbname;
var layergroupId = layergroup.layergroupid;
var dbConnection;
step(
function getPgConnection() {
@@ -325,7 +333,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
},
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this);
dbConnection = connection;
QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
req.profiler.done('queryTablesAndLastUpdated');
@@ -352,6 +361,21 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
return null;
},
function fetchLayersStats(err) {
assert.ifError(err);
var next = this;
self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) {
if (err) {
return next(err);
}
if (layersStats.length > 0) {
layergroup.metadata.layers.forEach(function (layer, index) {
layer.meta.stats = layersStats[index];
});
}
return next();
});
},
function finish(err) {
done(err);
}

7
lib/cartodb/server.js
View File

@@ -42,6 +42,8 @@ var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter'
var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
var MapConfigAdapter = require('./models/mapconfig/adapter');
var StatsBackend = require('./backends/stats');
module.exports = function(serverOptions) {
// Make stats client globally accessible
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@@ -151,6 +153,8 @@ module.exports = function(serverOptions) {
var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
var statsBackend = new StatsBackend();
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
@@ -209,7 +213,8 @@ module.exports = function(serverOptions) {
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
mapConfigAdapter
mapConfigAdapter,
statsBackend
).register(app);
new controller.NamedMaps(

26
lib/cartodb/utils/query-utils.js Normal file
View File

@@ -0,0 +1,26 @@
function prepareQuery(sql) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
return [
'SELECT * FROM CDB_QueryTablesText($windshaft$',
prepareQuery(query),
'$windshaft$) as tablenames'
].join('');
};
module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
};

5
package.json
View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "3.8.1",
"version": "3.9.0",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -39,7 +39,7 @@
"step-profiler": "~0.3.0",
"turbo-carto": "0.19.1",
"underscore": "~1.6.0",
"windshaft": "3.2.0",
"windshaft": "3.2.1",
"yargs": "~5.0.0"
},
"devDependencies": {
@@ -52,6 +52,7 @@
"strftime": "~0.8.2"
},
"scripts": {
"lint": "jshint lib test",
"preinstall": "make pre-install",
"test": "make test-all"
},

2
test/acceptance/multilayer.js
View File

@@ -1041,7 +1041,7 @@ describe(suiteName, function() {
);
});
}
// See https://github.com/CartoDB/Windshaft-cartodb/issues/91
// and https://github.com/CartoDB/Windshaft-cartodb/issues/38
it("tiles for private tables can be fetched with api_key", function(done) {

279
test/acceptance/stats/mapnik_stats_layergroup.js Normal file
View File

@@ -0,0 +1,279 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('Create mapnik layergroup', function() {
before(function() {
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerStats = true;
});
after(function() {
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var mapnikLayer1 = {
type: 'mapnik',
options: {
sql: 'select * from test_table limit 1',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer2 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_2 limit 2',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer3 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_3 limit 3',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer4 = {
type: 'mapnik',
options: {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id;'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var mapnikLayerGeomColumn = {
type: 'mapnik',
options: {
sql: 'select *, the_geom as my_geom from test_table_3 limit 2',
geom_column: 'my_geom',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
function typeLayerId(type, index) {
return type + '-' + mapnikBasicLayerId(index);
}
it('with one mapnik layer should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
});
});
it('with two mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
testClient.drain(done);
});
});
it('with three mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2,
mapnikLayer3
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2));
assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3);
testClient.drain(done);
});
});
it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer3,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
assert.ok(!layergroup.metadata.layers[2]);
testClient.drain(done);
});
});
it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[1].type, 'http');
testClient.drain(done);
});
});
it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss);
testClient.drain(done);
});
});
it('should work with different geom_column', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayerGeomColumn
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
testClient.drain(done);
});
});
it('should not include the stats part if the FF is disabled', function(done) {
global.environment.enabledFeatures.layerStats = false;
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.equal(layergroup.metadata.layers[2].type, 'http');
testClient.drain(done);
});
});
});

225
test/acceptance/stats/multilayer_stats.js Normal file
View File

@@ -0,0 +1,225 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('multilayer stats disabled', function() {
before(function () {
this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerMetadata = true;
global.environment.enabledFeatures.layerStats = false;
});
after(function () {
global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig;
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
function testLayerMetadataStats(testScenario) {
it(testScenario.desc, function(done) {
var mapConfig = {
version: '1.3.0',
layers: testScenario.layers
};
var testClient = new TestClient(mapConfig);
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
layergroup.metadata.layers.forEach(function (layer) {
if (layer.type !== 'torque' && layer.type !== 'mapnik') {
assert.ok(!('stats' in layer.meta));
} else if (layer.type !== 'torque') {
assert.ok(!('stats' in layer.meta));
assert.ok('cartocss' in layer.meta);
} else {
assert.ok('cartocss' in layer.meta);
// check torque metadata at least match in number
var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; });
if (torqueLayers.length) {
assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length);
}
}
});
testClient.drain(done);
});
});
}
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var torqueLayer = {
type: 'torque',
options: {
sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator",
cartocss: [
"Map {",
"-torque-frame-count:2;",
"-torque-resolution:3;",
"-torque-time-attribute:d;",
"-torque-aggregation-function:'count(id)';",
"}"
].join(' '),
cartocss_version: '2.0.1'
}
};
var mapnikLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikInteractivityLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var cartodbInteractivityLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbWadusInteractivityLayer = {
type: 'cartodb',
options: {
sql: sqlWadus,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'wadus'
}
};
var noTypeLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var noTypeInteractivityLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var testScenarios = [
{
desc: 'one layer, no interactivity',
layers: [cartodbLayer]
},
{
desc: 'two layers, different interactivity columns',
layers: [
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'torque + interactivity layers',
layers: [
torqueLayer,
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'interactivity + torque + interactivity',
layers: [
cartodbInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'http + interactivity + torque + no interactivity + torque + interactivity',
layers: [
httpLayer,
cartodbInteractivityLayer,
torqueLayer,
cartodbLayer,
torqueLayer,
cartodbWadusInteractivityLayer
]
},
{
desc: 'mapnik type – two layers, interactivity mix',
layers: [
mapnikLayer,
mapnikInteractivityLayer
]
},
{
desc: 'mapnik type – http + interactivity + torque + interactivity',
layers: [
httpLayer,
mapnikInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'no type – two layers, interactivity mix',
layers: [
noTypeLayer,
noTypeInteractivityLayer
]
},
{
desc: 'no type – http + interactivity + torque + interactivity',
layers: [
httpLayer,
noTypeInteractivityLayer,
torqueLayer,
noTypeInteractivityLayer
]
}
];
testScenarios.forEach(testLayerMetadataStats);
});

2
test/support/prepare_db.sh
View File

@@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount'
CURL_ARGS=""
for i in ${REMOTE_SQL_SCRIPTS}

2
test/support/sql/windshaft.test.sql
View File

@@ -649,3 +649,5 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int
END;
$$ LANGUAGE plpgsql;
GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER;
ANALYZE;

153
test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js Normal file
View File

@@ -0,0 +1,153 @@
var assert = require('assert');
var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
function getDbConnectionMock () {
return {
query: function(sql, callback) {
return callback(null, {
rows: [{rows: 1}]
});
}
};
}
describe('mapnik-layer-stats', function() {
beforeEach(function () {
this.dbConnectionMock = getDbConnectionMock();
this.rendererCacheMock = {};
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
}
]
};
var testMapConfigTwoLayers = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
]
};
var testMapConfigOneLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
]
};
var testMapConfigTwoLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_3', 'test_table_4']
}
},
]
};
it('should return 1 feature for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayers);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
it('should return 1 feature for one layers with two tables', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers and two tables', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
});

36
test/unit/cartodb/backends/layer-stats/torque-layer-stats.js Normal file
View File

@@ -0,0 +1,36 @@
var assert = require('assert');
var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
describe('torque-layer-stats', function () {
beforeEach(function () {
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'torque',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
}
},
]
};
it('should return torque stats for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layerId = 0;
var layer = mapConfig.getLayer(layerId);
var testSubject = new TorqueLayerStats();
testSubject.getStats(layer, {}, function (err, result) {
assert.ifError(err);
assert.deepEqual({}, result);
done();
});
});
});

72
yarn.lock
View File

@@ -2,7 +2,7 @@
# yarn lockfile v1
abaculus@cartodb/abaculus#2.0.3-cdb1:
"abaculus@github:cartodb/abaculus#2.0.3-cdb1":
version "2.0.3-cdb1"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf"
dependencies:
@@ -150,7 +150,7 @@ boom@2.x.x:
dependencies:
hoek "2.x.x"
brace-expansion@^1.0.0:
brace-expansion@^1.1.7:
version "1.1.7"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.7.tgz#3effc3c50e000531fb720eaff80f0ae8ef23cf59"
dependencies:
@@ -209,7 +209,7 @@ camshaft@0.55.2:
dot "^1.0.3"
request "^2.69.0"
canvas@cartodb/node-canvas#1.6.2-cdb2:
"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
version "1.6.2-cdb2"
resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
dependencies:
@@ -235,7 +235,7 @@ carto@CartoDB/carto#0.15.1-cdb1:
optimist "~0.6.0"
underscore "~1.6.0"
carto@cartodb/carto#0.15.1-cdb3:
"carto@github:cartodb/carto#0.15.1-cdb3":
version "0.15.1-cdb3"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
dependencies:
@@ -441,8 +441,8 @@ deep-equal@^1.0.0:
resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5"
deep-extend@~0.4.0:
version "0.4.1"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.1.tgz#efe4113d08085f4e6f9687759810f807469e2253"
version "0.4.2"
resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.4.2.tgz#48b699c27e334bf89f10892be432f6e4c7d34a7f"
deep-is@~0.1.3:
version "0.1.3"
@@ -682,7 +682,7 @@ fstream@^1.0.0, fstream@^1.0.10, fstream@^1.0.2:
mkdirp ">=0.5 0"
rimraf "2"
gauge@~2.7.1:
gauge@~2.7.3:
version "2.7.4"
resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7"
dependencies:
@@ -1002,8 +1002,8 @@ js-base64@^2.1.9:
resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.1.9.tgz#f0e80ae039a4bd654b5f281fc93f04a914a7fcce"
js-yaml@3.x, js-yaml@^3.4.6:
version "3.8.3"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.3.tgz#33a05ec481c850c8875929166fe1beb61c728766"
version "3.8.4"
resolved "https://registry.yarnpkg.com/js-yaml/-/js-yaml-3.8.4.tgz#520b4564f86573ba96662af85a8cafa7b4b5a6f6"
dependencies:
argparse "^1.0.7"
esprima "^3.1.1"
@@ -1061,8 +1061,8 @@ jsprim@^1.2.2:
verror "1.3.6"
kind-of@^3.0.2:
version "3.2.0"
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.0.tgz#b58abe4d5c044ad33726a8c1525b48cf891bff07"
version "3.2.2"
resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
dependencies:
is-buffer "^1.1.5"
@@ -1388,12 +1388,12 @@ normalize-package-data@^2.3.2:
validate-npm-package-license "^3.0.1"
npmlog@^4.0.2:
version "4.0.2"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.0.2.tgz#d03950e0e78ce1527ba26d2a7592e9348ac3e75f"
version "4.1.0"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.0.tgz#dc59bee85f64f00ed424efb2af0783df25d1c0b5"
dependencies:
are-we-there-yet "~1.1.2"
console-control-strings "~1.1.0"
gauge "~2.7.1"
gauge "~2.7.3"
set-blocking "~2.0.0"
number-is-nan@^1.0.0:
@@ -1404,14 +1404,10 @@ oauth-sign@~0.8.1:
version "0.8.2"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
object-assign@4.1.0:
object-assign@4.1.0, object-assign@^4.1.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0"
object-assign@^4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
object-keys@~0.4.0:
version "0.4.0"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336"
@@ -1749,7 +1745,7 @@ repeat-string@^1.5.2:
version "1.6.1"
resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637"
request@2.x, request@^2.55.0, request@~2.79.0:
request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0:
version "2.79.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de"
dependencies:
@@ -1774,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0:
tunnel-agent "~0.4.1"
uuid "^3.0.0"
request@^2.69.0, request@^2.81.0:
request@^2.81.0:
version "2.81.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0"
dependencies:
@@ -1839,22 +1835,22 @@ safe-json-stringify@~1:
version "1.0.4"
resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911"
"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
"semver@2 || 3 || 4 || 5", semver@~4.3.3:
version "4.3.6"
resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da"
semver@4.3.2:
version "4.3.2"
resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7"
semver@^5.1.0, semver@^5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
semver@~1.1.4:
version "1.1.4"
resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540"
semver@~4.3.3:
version "4.3.6"
resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da"
semver@~5.0.3:
version "5.0.3"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a"
@@ -2111,7 +2107,7 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2:
"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2":
version "2.3.1-cdb2"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313"
dependencies:
@@ -2119,7 +2115,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2:
mapnik-pool "~0.1.3"
sphericalmercator "1.0.x"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2:
"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2":
version "0.6.18-cdb2"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858"
dependencies:
@@ -2197,8 +2193,8 @@ type-is@~1.6.10, type-is@~1.6.6:
mime-types "~2.1.15"
uglify-js@^2.6:
version "2.8.22"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.22.tgz#d54934778a8da14903fa29a326fb24c0ab51a1a0"
version "2.8.26"
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.26.tgz#3a1db8ae0a0aba7f92e1ddadadbd0293d549f90e"
dependencies:
source-map "~0.5.1"
yargs "~3.10.0"
@@ -2269,10 +2265,10 @@ which@^1.1.1:
isexe "^2.0.0"
wide-align@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.0.tgz#40edde802a71fea1f070da3e62dcda2e7add96ad"
version "1.1.2"
resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.2.tgz#571e0f1b0604636ebc0dfc21b0339bbe31341710"
dependencies:
string-width "^1.0.1"
string-width "^1.0.2"
window-size@0.1.0:
version "0.1.0"
@@ -2282,9 +2278,9 @@ window-size@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@3.2.0:
version "3.2.0"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.0.tgz#c69f8d199bbeccc96e66b8256928e6fa87999803"
windshaft@3.2.1:
version "3.2.1"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.1.tgz#50a3afa6562315dd9e65e411660970e118f36c19"
dependencies:
abaculus cartodb/abaculus#2.0.3-cdb1
canvas cartodb/node-canvas#1.6.2-cdb2