From c65518cf41c135cd0289b8fb87176915862d7004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 20:16:43 +0200 Subject: [PATCH] Get back layer-stats from windshaft-stats --- .../backends/layer-stats/empty-layer-stats.js | 16 ++ lib/cartodb/backends/layer-stats/factory.js | 23 +++ .../backends/layer-stats/layer-stats.js | 45 ++++++ .../layer-stats/mapnik-layer-stats.js | 28 ++++ .../backends/layer-stats/query-utils.js | 26 +++ .../layer-stats/torque-layer-stats.js | 16 ++ lib/cartodb/backends/stats.js | 5 +- package.json | 1 - .../layer-stats/mapnik-layer-stats.js | 153 ++++++++++++++++++ .../layer-stats/torque-layer-stats.js | 36 +++++ yarn.lock | 30 +--- 11 files changed, 347 insertions(+), 32 deletions(-) create mode 100644 lib/cartodb/backends/layer-stats/empty-layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/factory.js create mode 100644 lib/cartodb/backends/layer-stats/layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/mapnik-layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/query-utils.js create mode 100644 lib/cartodb/backends/layer-stats/torque-layer-stats.js create mode 100644 test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js create mode 100644 test/unit/cartodb/backends/layer-stats/torque-layer-stats.js diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js new file mode 100644 index 00000000..0923b59c --- /dev/null +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -0,0 +1,16 @@ +function EmptyLayerStats(types) { + this._types = types || {}; +} + +EmptyLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +EmptyLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + process.nextTick(function() { + callback(null, {}); + }); +}; + +module.exports = EmptyLayerStats; diff --git a/lib/cartodb/backends/layer-stats/factory.js b/lib/cartodb/backends/layer-stats/factory.js new file mode 100644 index 00000000..8aacdb5a --- /dev/null +++ b/lib/cartodb/backends/layer-stats/factory.js @@ -0,0 +1,23 @@ +var LayerStats = require('./layer-stats'); +var EmptyLayerStats = require('./empty-layer-stats'); +var MapnikLayerStats = require('./mapnik-layer-stats'); +var TorqueLayerStats = require('./torque-layer-stats'); + +module.exports = function LayerStatsFactory(type) { + var layerStatsIterator = []; + var selectedType = type || 'ALL'; + + if (selectedType === 'ALL') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true })); + layerStatsIterator.push(new MapnikLayerStats()); + layerStatsIterator.push(new TorqueLayerStats()); + } else if (selectedType === 'mapnik') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true })); + layerStatsIterator.push(new MapnikLayerStats()); + } else if (selectedType === 'torque') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true })); + layerStatsIterator.push(new TorqueLayerStats()); + } + + return new LayerStats(layerStatsIterator); +}; diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js new file mode 100644 index 00000000..bc1787c2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -0,0 +1,45 @@ +var d3 = require('d3-queue'); + +function LayerStats(layerStatsIterator) { + this.layerStatsIterator = layerStatsIterator; +} + +LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) { + var self = this; + var stats = []; + + if (!mapConfig.getLayers().length) { + return callback(null, stats); + } + var metaQueue = d3.queue(mapConfig.getLayers().length); + mapConfig.getLayers().forEach(function (layer, layerId) { + var layerType = mapConfig.layerType(layerId); + + for (var i = 0; i < self.layerStatsIterator.length; i++) { + if (self.layerStatsIterator[i].is(layerType)) { + var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]); + metaQueue.defer(getStats, layer, dbConnection); + break; + } + } + }); + + metaQueue.awaitAll(function (err, results) { + if (err) { + return callback(err); + } + + if (!results) { + return callback(null, null); + } + + mapConfig.getLayers().forEach(function (layer, layerIndex) { + stats[layerIndex] = results[layerIndex]; + }); + + return callback(err, stats); + }); + +}; + +module.exports = LayerStats; diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..7dfd97af --- /dev/null +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,28 @@ +var queryUtils = require('./query-utils'); + +function MapnikLayerStats () { + this._types = { + mapnik: true, + cartodb: true + }; +} + +MapnikLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +MapnikLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); + // This query would gather stats for postgresql table if not exists + dbConnection.query(queryRowCountSql, function (err, res) { + if (err) { + return callback(null, {estimatedFeatureCount: -1}); + } else { + // We decided that the relation is 1 row == 1 feature + return callback(null, {estimatedFeatureCount: res.rows[0].rows}); + } + }); +}; + +module.exports = MapnikLayerStats; diff --git a/lib/cartodb/backends/layer-stats/query-utils.js b/lib/cartodb/backends/layer-stats/query-utils.js new file mode 100644 index 00000000..47d730f4 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/query-utils.js @@ -0,0 +1,26 @@ +function prepareQuery(sql) { + var affectedTableRegexCache = { + bbox: /!bbox!/g, + scale_denominator: /!scale_denominator!/g, + pixel_width: /!pixel_width!/g, + pixel_height: /!pixel_height!/g + }; + + return sql + .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') + .replace(affectedTableRegexCache.scale_denominator, '0') + .replace(affectedTableRegexCache.pixel_width, '1') + .replace(affectedTableRegexCache.pixel_height, '1'); +} + +module.exports.extractTableNames = function extractTableNames(query) { + return [ + 'SELECT * FROM CDB_QueryTablesText($windshaft$', + prepareQuery(query), + '$windshaft$) as tablenames' + ].join(''); +}; + +module.exports.getQueryRowCount = function getQueryRowEstimation(query) { + return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; +}; diff --git a/lib/cartodb/backends/layer-stats/torque-layer-stats.js b/lib/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..00b4def2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,16 @@ +function TorqueLayerStats() { + this._types = { + torque: true + }; +} + +TorqueLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +TorqueLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + return callback(null, {}); +}; + +module.exports = TorqueLayerStats; diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index eb800e3c..20f89a0f 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -1,4 +1,4 @@ -var windshaftStats = require('windshaft-stats'); +var layerStats = require('./layer-stats/factory'); function StatsBackend() { } @@ -9,8 +9,7 @@ StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; if (layerStatsEnabled) { - var layerStats = windshaftStats(); - layerStats.getStats(mapConfig, dbConnection, callback); + layerStats().getStats(mapConfig, dbConnection, callback); } else { return callback(null, null); } diff --git a/package.json b/package.json index 4affa452..c49f1fe4 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "turbo-carto": "0.19.0", "underscore": "~1.6.0", "windshaft": "cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea", - "windshaft-stats": "cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..6e47297d --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,153 @@ +var assert = require('assert'); +var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +function getDbConnectionMock () { + return { + query: function(sql, callback) { + return callback(null, { + rows: [{rows: 1}] + }); + } + }; +} + +describe('mapnik-layer-stats', function() { + + beforeEach(function () { + this.dbConnectionMock = getDbConnectionMock(); + this.rendererCacheMock = {}; + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + } + ] + }; + + var testMapConfigTwoLayers = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + ] + }; + + var testMapConfigOneLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + ] + }; + + var testMapConfigTwoLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_3', 'test_table_4'] + } + }, + ] + }; + + it('should return 1 feature for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayers); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); + + it('should return 1 feature for one layers with two tables', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers and two tables', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); +}); diff --git a/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..c9adfad9 --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +describe('torque-layer-stats', function () { + + beforeEach(function () { + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'torque', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + } + }, + ] + }; + + it('should return torque stats for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layerId = 0; + var layer = mapConfig.getLayer(layerId); + var testSubject = new TorqueLayerStats(); + testSubject.getStats(layer, {}, function (err, result) { + assert.ifError(err); + assert.deepEqual({}, result); + done(); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8d8ec0d0..a1b13067 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,7 +254,7 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" -cartodb-psql@0.8.0, cartodb-psql@~0.8.0: +cartodb-psql@0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" dependencies: @@ -411,10 +411,6 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" -d3-queue@~3.0.5: - version "3.0.7" - resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" - dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -443,12 +439,6 @@ debug@^1.0.4: dependencies: ms "0.6.2" -debug@~2.6.6: - version "2.6.6" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" - dependencies: - ms "0.7.3" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1312,10 +1302,6 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" -ms@0.7.3: - version "0.7.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" - mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1532,7 +1518,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2025,10 +2011,6 @@ step@~0.0.5, step@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" -step@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" - strftime@~0.8.2: version "0.8.4" resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" @@ -2288,14 +2270,6 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f: - version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/16a973ba9addf88312ba92047e41108be2c3434f" - dependencies: - cartodb-psql "~0.8.0" - d3-queue "~3.0.5" - debug "~2.6.6" - windshaft@cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea: version "3.1.3" resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/515efb2d9237a1511b81a61fd6f7524184df38ea"