From 05d3b3bf669ecc9e73fa40e29285fb2a6da4175a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 3 May 2017 17:21:08 +0200 Subject: [PATCH 01/29] Point windshaft to specific commit for buffer-size configuration --- package.json | 2 +- yarn.lock | 106 +++++++++++++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index 07f158b7..f91ac976 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "3.1.1", + "windshaft": "cartodb/windshaft#548-vector-buffer-size", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 1386aaaa..1addd775 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -22,8 +22,8 @@ accepts@~1.2.12: negotiator "0.5.3" ajv@^4.9.1: - version "4.11.7" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.7.tgz#8655a5d86d0824985cc471a1d913fb6729a0ec48" + version "4.11.8" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-4.11.8.tgz#82ffb02b29e662ae53bdc20af15947706739c536" dependencies: co "^4.6.0" json-stable-stringify "^1.0.1" @@ -205,7 +205,7 @@ camshaft@0.54.1: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -223,15 +223,7 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -239,6 +231,14 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -612,8 +612,8 @@ express@~4.13.3: vary "~1.0.1" extend@~3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4" + version "3.0.1" + resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" extsprintf@1.0.2: version "1.0.2" @@ -716,7 +716,7 @@ generate-object-property@^1.1.0: dependencies: is-property "^1.0.0" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -728,21 +728,17 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" getpass@^0.1.1: - version "0.1.6" - resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.6.tgz#283ffd9fc1256840875311c1b60e8c40187110e6" + version "0.1.7" + resolved "https://registry.yarnpkg.com/getpass/-/getpass-0.1.7.tgz#5eff8e3e684d569ae4cb2b1282604e8ba62149fa" dependencies: assert-plus "^1.0.0" -glob@3.2.3, "glob@~ 3.2.1": +glob@3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -781,6 +777,13 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" +"glob@~ 3.2.1": + version "3.2.11" + resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" + dependencies: + inherits "2" + minimatch "0.3" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -812,8 +815,8 @@ growl@1.8.1: resolved "https://registry.yarnpkg.com/growl/-/growl-1.8.1.tgz#4b2dec8d907e93db336624dcec0183502f8c9428" handlebars@^4.0.1: - version "4.0.6" - resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.6.tgz#2ce4484850537f9c97a8026d5399b935c4ed4ed7" + version "4.0.8" + resolved "https://registry.yarnpkg.com/handlebars/-/handlebars-4.0.8.tgz#22b875cd3f0e6cbea30314f144e82bc7a72ff420" dependencies: async "^1.4.0" optimist "^0.6.1" @@ -939,7 +942,7 @@ is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" -is-buffer@^1.0.2: +is-buffer@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc" @@ -1084,10 +1087,10 @@ jsprim@^1.2.2: verror "1.3.6" kind-of@^3.0.2: - version "3.1.0" - resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.1.0.tgz#475d698a5e49ff5e53d14e3e732429dc8bf4cf47" + version "3.2.0" + resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.0.tgz#b58abe4d5c044ad33726a8c1525b48cf891bff07" dependencies: - is-buffer "^1.0.2" + is-buffer "^1.1.5" lazy-cache@^1.0.3: version "1.0.4" @@ -1224,6 +1227,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" +minimatch@0.3: + version "0.3.0" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" + dependencies: + lru-cache "2" + sigmund "~1.0.0" + minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1393,14 +1403,10 @@ oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1517,7 +1523,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +"pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1828,14 +1834,18 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" +semver@^5.1.0, semver@^5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + semver@~1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/semver/-/semver-1.1.4.tgz#2e5a4e72bab03472cc97f72753b4508912ef5540" @@ -1844,10 +1854,6 @@ semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" -semver@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" - send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -2098,17 +2104,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb1": +tilelive-bridge@cartodb/tilelive-bridge#buffer-size: version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/9df4bde07f3cf9ffbda42657066559e2b8875e1b" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb1": +tilelive-mapnik@cartodb/tilelive-mapnik#fix-buffer-size-0: version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/c67400573572e836609dd75a7acec8f414a50dbe" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2269,9 +2275,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.1.1: +windshaft@cartodb/windshaft#548-vector-buffer-size: version "3.1.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.1.1.tgz#5e1b20fa9bcc6f4e04f67738b18a35cb19cadbd1" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2288,8 +2294,8 @@ windshaft@3.1.1: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb1 - tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb1 + tilelive-bridge cartodb/tilelive-bridge#buffer-size + tilelive-mapnik cartodb/tilelive-mapnik#fix-buffer-size-0 torque.js "~2.11.0" underscore "~1.6.0" From 3f6c8fa51c032232b97ab826ce76f960918b4989 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Mon, 8 May 2017 18:42:40 +0200 Subject: [PATCH 02/29] Use windshaft-stats to get layer stats --- .gitignore | 1 + lib/cartodb/controllers/map.js | 49 ++++++++++++++++--- .../provider/dummy-mapconfig-provider.js | 20 ++++++++ package.json | 1 + 4 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js diff --git a/.gitignore b/.gitignore index b39f2801..d49bbc6e 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ redis.pid test.log npm-debug.log coverage/ +.DS_Store diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b3ca2b4a..0cdf75f1 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); +var windshaftStats = require('windshaft-stats'); var QueryTables = require('cartodb-query-tables'); var ResourceLocator = require('../models/resource-locator'); @@ -19,6 +20,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); +var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconfig-provider'); /** * @param {AuthApi} authApi @@ -33,7 +35,8 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, - surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) { + surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, + rendererCache) { BaseController.call(this, authApi, pgConnection); @@ -47,6 +50,8 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.mapConfigAdapter = mapConfigAdapter; this.resourceLocator = new ResourceLocator(global.environment); + + this.rendererCache = rendererCache; } util.inherits(MapController, BaseController); @@ -161,7 +166,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, req.params, this); }, function finish(err, layergroup) { if (err) { @@ -214,6 +219,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var mapConfigProvider; var mapConfig; + var rendererParams; step( function setupParams(){ @@ -238,8 +244,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); mapConfigProvider.getMapConfig(this); }, - function createLayergroup(err, mapConfig_, rendererParams) { + function createLayergroup(err, mapConfig_, _rendererParams) { assert.ifError(err); + rendererParams = _rendererParams; mapConfig = mapConfig_; self.mapBackend.createLayergroup( mapConfig, rendererParams, @@ -249,7 +256,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, rendererParams, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -272,7 +279,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) { +MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { var self = this; var username = req.context.user; @@ -318,6 +325,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la var dbName = req.params.dbname; var layergroupId = layergroup.layergroupid; + var dbConnection; step( function getPgConnection() { @@ -325,7 +333,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la }, function getAffectedTablesAndLastUpdatedTime(err, connection) { assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this); + dbConnection = connection; + QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { req.profiler.done('queryTablesAndLastUpdated'); @@ -352,6 +361,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la return null; }, + function fetchLayersMetadata(err) { + assert.ifError(err); + var next = this; + var mapConfigProvider = new DummyMapConfigProvider(mapconfig, rendererParams); + var layerStats = windshaftStats(); + layerStats.getStats(self.rendererCache, rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { + if (err) { + return next(err); + } + if (layersStats) { + res.metadata = res.metadata || {}; + res.metadata.layers = layersStats; + + // backwards compatibility for torque + var torqueMetadata = layersStats.reduce(function(acc, layer, layerId) { + if (layer.type === 'torque') { + acc[layerId] = layer.meta; + } + return acc; + }, {}); + if (Object.keys(torqueMetadata).length) { + res.metadata.torque = torqueMetadata; + } + } + + return next(); + }); + }, function finish(err) { done(err); } diff --git a/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js b/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js new file mode 100644 index 00000000..0b8942c5 --- /dev/null +++ b/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js @@ -0,0 +1,20 @@ +var util = require('util'); +var MapStoreMapConfigProvider = require('windshaft').model.provider.MapStoreMapConfig; + +function DummyMapConfigProvider(mapConfig, params) { + MapStoreMapConfigProvider.call(this, undefined, params); + + this.mapConfig = mapConfig; +} + +util.inherits(DummyMapConfigProvider, MapStoreMapConfigProvider); + +module.exports = DummyMapConfigProvider; + +DummyMapConfigProvider.prototype.setParams = function(params) { + this.params = params; +}; + +DummyMapConfigProvider.prototype.getMapConfig = function(callback) { + return callback(null, this.mapConfig, this.params, {}); +}; diff --git a/package.json b/package.json index f91ac976..9bbb5125 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "turbo-carto": "0.19.0", "underscore": "~1.6.0", "windshaft": "cartodb/windshaft#548-vector-buffer-size", + "windshaft-stats": "0.0.1", "yargs": "~5.0.0" }, "devDependencies": { From 525c0f2afaa99cae395e8e12bd28f47f55e8c236 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 11:58:48 +0200 Subject: [PATCH 03/29] Added rendererCache as dependency for map controller --- lib/cartodb/controllers/map.js | 1 + lib/cartodb/server.js | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 0cdf75f1..b8242b59 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -32,6 +32,7 @@ var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconf * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables * @param {MapConfigAdapter} mapConfigAdapter + * @param {RendererCache} rendererCache * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index b695a367..e14c0389 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -207,7 +207,8 @@ module.exports = function(serverOptions) { surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache, - mapConfigAdapter + mapConfigAdapter, + rendererCache ).register(app); new controller.NamedMaps( From 568d6b5458e2803c5179a86e5df43402f46924c8 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 11:59:11 +0200 Subject: [PATCH 04/29] Include the layers metadata into the layergroup object --- lib/cartodb/controllers/map.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b8242b59..4bcc4911 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -372,8 +372,8 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la return next(err); } if (layersStats) { - res.metadata = res.metadata || {}; - res.metadata.layers = layersStats; + layergroup.metadata = layergroup.metadata || {}; + layergroup.metadata.layers = layersStats; // backwards compatibility for torque var torqueMetadata = layersStats.reduce(function(acc, layer, layerId) { @@ -383,7 +383,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la return acc; }, {}); if (Object.keys(torqueMetadata).length) { - res.metadata.torque = torqueMetadata; + layergroup.metadata.torque = torqueMetadata; } } From 15c68711aa44bac74027a21f883259f5152b9ea3 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 12:31:16 +0200 Subject: [PATCH 05/29] Create stats backend to decouple stats logic from map controller --- lib/cartodb/backends/stats.js | 12 ++++++++++++ lib/cartodb/controllers/map.js | 10 ++++------ lib/cartodb/server.js | 6 +++++- 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 lib/cartodb/backends/stats.js diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js new file mode 100644 index 00000000..2179db1b --- /dev/null +++ b/lib/cartodb/backends/stats.js @@ -0,0 +1,12 @@ +var windshaftStats = require('windshaft-stats'); + +function StatsBackend(rendererCache) { + this.rendererCache = rendererCache; +}; + +module.exports = StatsBackend; + +StatsBackend.prototype.getStats = function(params, dbConnection, mapConfigProvider, callback) { + var layerStats = windshaftStats(); + layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); +}; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4bcc4911..f3211190 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -2,7 +2,6 @@ var _ = require('underscore'); var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); -var windshaftStats = require('windshaft-stats'); var QueryTables = require('cartodb-query-tables'); var ResourceLocator = require('../models/resource-locator'); @@ -32,12 +31,12 @@ var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconf * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables * @param {MapConfigAdapter} mapConfigAdapter - * @param {RendererCache} rendererCache + * @param {StatsBackend} statsBackend * @constructor */ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, - rendererCache) { + statsBackend) { BaseController.call(this, authApi, pgConnection); @@ -52,7 +51,7 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.mapConfigAdapter = mapConfigAdapter; this.resourceLocator = new ResourceLocator(global.environment); - this.rendererCache = rendererCache; + this.statsBackend = statsBackend; } util.inherits(MapController, BaseController); @@ -366,8 +365,7 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la assert.ifError(err); var next = this; var mapConfigProvider = new DummyMapConfigProvider(mapconfig, rendererParams); - var layerStats = windshaftStats(); - layerStats.getStats(self.rendererCache, rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { + self.statsBackend.getStats(rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { if (err) { return next(err); } diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index e14c0389..5ce31f13 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -41,6 +41,8 @@ var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter' var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter'); var MapConfigAdapter = require('./models/mapconfig/adapter'); +var StatsBackend = require('./backends/stats'); + module.exports = function(serverOptions) { // Make stats client globally accessible global.statsClient = StatsClient.getInstance(serverOptions.statsd); @@ -150,6 +152,8 @@ module.exports = function(serverOptions) { var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); + var statsBackend = new StatsBackend(rendererCache); + var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; @@ -208,7 +212,7 @@ module.exports = function(serverOptions) { userLimitsApi, layergroupAffectedTablesCache, mapConfigAdapter, - rendererCache + statsBackend ).register(app); new controller.NamedMaps( From 759d28f12f40854f5bcb8ce44c92f2c8e9343265 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 12:51:22 +0200 Subject: [PATCH 06/29] Include feature flag to enable/disable stats --- lib/cartodb/backends/stats.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 2179db1b..d2e989e3 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -7,6 +7,14 @@ function StatsBackend(rendererCache) { module.exports = StatsBackend; StatsBackend.prototype.getStats = function(params, dbConnection, mapConfigProvider, callback) { - var layerStats = windshaftStats(); - layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + var enabledFeatures = global.environment.enabledFeatures; + var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; + var layerStats; + if (layerMetadataEnabled) { + layerStats = windshaftStats(); + layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + } else { + layerStats = windshaftStats('torque'); + layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + } }; From b25540720ca2e5e2b8dd3f285354b0b7ff522a6a Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 14:36:35 +0200 Subject: [PATCH 07/29] Added acceptance tests for stats functionallity --- .../stats/mapnik_stats_layergroup.js | 260 ++++++++++++++++++ test/acceptance/stats/multilayer_stats.js | 219 +++++++++++++++ test/support/sql/windshaft.test.sql | 2 + 3 files changed, 481 insertions(+) create mode 100644 test/acceptance/stats/mapnik_stats_layergroup.js create mode 100644 test/acceptance/stats/multilayer_stats.js diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js new file mode 100644 index 00000000..746f6c5c --- /dev/null +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -0,0 +1,260 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('Create mapnik layergroup', function() { + before(function() { + this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + global.environment.enabledFeatures.layerMetadata = true; + }); + + after(function() { + global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + }); + + var cartocssVersion = '2.3.0'; + var cartocss = '#layer { line-width:16; }'; + + var mapnikLayer1 = { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer2 = { + type: 'mapnik', + options: { + sql: 'select * from test_table_2 limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer3 = { + type: 'mapnik', + options: { + sql: 'select * from test_table_3 limit 2', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikLayer4 = { + type: 'mapnik', + options: { + sql: [ + 'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address', + ' from test_table t1, test_table_2 t2', + ' where t1.cartodb_id = t2.cartodb_id;' + ].join(''), + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var httpLayer = { + type: 'http', + options: { + urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png', + subdomains: ['a','b','c'] + } + }; + + var mapnikLayerGeomColumn = { + type: 'mapnik', + options: { + sql: 'select *, the_geom as my_geom from test_table_3 limit 2', + geom_column: 'my_geom', + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + function mapnikBasicLayerId(index) { + return 'layer' + index; + } + function typeLayerId(type, index) { + return type + '-' + mapnikBasicLayerId(index); + } + + it('with one mapnik layer should response with meta-stats for that layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + mapnikLayer2 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + testClient.drain(done); + }); + }); + + it('with three mapnik layer should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + mapnikLayer2, + mapnikLayer3 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); + assert.equal(layergroup.metadata.layers[2].meta.stats[0].features, 5); + testClient.drain(done); + }); + }); + + it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer4, + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + testClient.drain(done); + }); + }); + + it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer3, + mapnikLayer4 + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); + assert.ok(!layergroup.metadata.layers[2]); + testClient.drain(done); + }); + }); + + it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayer1, + httpLayer + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[1].type, 'http'); + testClient.drain(done); + }); + }); + + it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + httpLayer, + mapnikLayer1 + ] + }); + + testClient.getLayergroup(function (err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[0].type, 'http'); + assert.ok(!layergroup.metadata.layers[0].meta.cartocss); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); + assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); + testClient.drain(done); + }); + }); + + it('should work with different geom_column', function(done) { + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + mapnikLayerGeomColumn + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); + // we don't care about stats here as is an aliased column + assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('features')); + testClient.drain(done); + }); + }); +}); diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js new file mode 100644 index 00000000..4571bf6a --- /dev/null +++ b/test/acceptance/stats/multilayer_stats.js @@ -0,0 +1,219 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + +describe('multilayer stats disabled', function() { + + before(function () { + global.environment.enabledFeatures.layerMetadata = false; + }); + + after(function () { + global.environment.enabledFeatures.layerMetadata = true; + }); + + + function testLayerMetadataStats(testScenario) { + + it(testScenario.desc, function(done) { + var mapConfig = { + version: '1.3.0', + layers: testScenario.layers + }; + + var testClient = new TestClient(mapConfig); + + testClient.getLayergroup(function(err, layergroup) { + assert.ifError(err); + layergroup.metadata.layers.forEach(function (layer) { + if (layer.type !== 'torque' && layer.type !== 'mapnik') { + assert.ok('stats' in layer.meta); + } else if (layer.type !== 'torque') { + assert.ok('stats' in layer.meta); + assert.ok('cartocss' in layer.meta); + } else { + assert.ok('cartocss' in layer.meta); + // check torque metadata at least match in number + var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; }); + if (torqueLayers.length) { + assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length); + } + } + }); + + testClient.drain(done); + }); + }); + } + + var cartocssVersion = '2.3.0'; + var cartocss = '#layer { line-width:16; }'; + var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + + var httpLayer = { + type: 'http', + options: { + urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png', + subdomains: ['a','b','c'] + } + }; + + var torqueLayer = { + type: 'torque', + options: { + sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator", + cartocss: [ + "Map {", + "-torque-frame-count:2;", + "-torque-resolution:3;", + "-torque-time-attribute:d;", + "-torque-aggregation-function:'count(id)';", + "}" + ].join(' '), + cartocss_version: '2.0.1' + } + }; + + var mapnikLayer = { + type: 'mapnik', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var mapnikInteractivityLayer = { + type: 'mapnik', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var cartodbLayer = { + type: 'cartodb', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var cartodbInteractivityLayer = { + type: 'cartodb', + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var cartodbWadusInteractivityLayer = { + type: 'cartodb', + options: { + sql: sqlWadus, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'wadus' + } + }; + + var noTypeLayer = { + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss + } + }; + + var noTypeInteractivityLayer = { + options: { + sql: sql, + cartocss_version: cartocssVersion, + cartocss: cartocss, + interactivity: 'i' + } + }; + + var testScenarios = [ + { + desc: 'one layer, no interactivity', + layers: [cartodbLayer] + }, + { + desc: 'two layers, different interactivity columns', + layers: [ + cartodbWadusInteractivityLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'torque + interactivity layers', + layers: [ + torqueLayer, + cartodbWadusInteractivityLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'interactivity + torque + interactivity', + layers: [ + cartodbInteractivityLayer, + torqueLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'http + interactivity + torque + no interactivity + torque + interactivity', + layers: [ + httpLayer, + cartodbInteractivityLayer, + torqueLayer, + cartodbLayer, + torqueLayer, + cartodbWadusInteractivityLayer + ] + }, + { + desc: 'mapnik type – two layers, interactivity mix', + layers: [ + mapnikLayer, + mapnikInteractivityLayer + ] + }, + { + desc: 'mapnik type – http + interactivity + torque + interactivity', + layers: [ + httpLayer, + mapnikInteractivityLayer, + torqueLayer, + cartodbInteractivityLayer + ] + }, + { + desc: 'no type – two layers, interactivity mix', + layers: [ + noTypeLayer, + noTypeInteractivityLayer + ] + }, + { + desc: 'no type – http + interactivity + torque + interactivity', + layers: [ + httpLayer, + noTypeInteractivityLayer, + torqueLayer, + noTypeInteractivityLayer + ] + } + ]; + + testScenarios.forEach(testLayerMetadataStats); + +}); diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index eb495028..91484950 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -649,3 +649,5 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int END; $$ LANGUAGE plpgsql; GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER; + +ANALYZE; From 474d68687c60e71633d5a7b81a00d3aed37bb7cd Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 14:47:39 +0200 Subject: [PATCH 08/29] Include vertexCount in the stats tests --- .../stats/mapnik_stats_layergroup.js | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 746f6c5c..78d873af 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -92,7 +92,8 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -109,9 +110,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -129,11 +132,14 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -149,8 +155,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -167,11 +175,15 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -188,11 +200,14 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); @@ -212,7 +227,8 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -233,7 +249,8 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].features, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -253,7 +270,8 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('features')); + assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); + assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('vertexCount')); testClient.drain(done); }); }); From b87e4428018de06f1516f4a9862c526a366c7fdf Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 18:24:11 +0200 Subject: [PATCH 09/29] Remove vertex count from stats tests --- .../stats/mapnik_stats_layergroup.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 78d873af..4a327a01 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -93,7 +93,6 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -111,10 +110,8 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -133,13 +130,10 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].vertexCount, 5); testClient.drain(done); }); }); @@ -156,9 +150,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -176,14 +168,10 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); testClient.drain(done); }); }); @@ -201,13 +189,10 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].vertexCount, 5); assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); @@ -228,7 +213,6 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -250,7 +234,6 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].vertexCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -271,7 +254,6 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('vertexCount')); testClient.drain(done); }); }); From 7d5b6b08206515c7e3f7bcc9f2e12eb14eb2162e Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 9 May 2017 18:24:24 +0200 Subject: [PATCH 10/29] Lint changes and yarn.lock --- lib/cartodb/backends/stats.js | 2 +- lib/cartodb/controllers/map.js | 7 ++-- package.json | 1 + test/acceptance/multilayer.js | 2 +- test/acceptance/stats/multilayer_stats.js | 6 ++-- yarn.lock | 40 +++++++++-------------- 6 files changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index d2e989e3..d8f83c72 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -2,7 +2,7 @@ var windshaftStats = require('windshaft-stats'); function StatsBackend(rendererCache) { this.rendererCache = rendererCache; -}; +} module.exports = StatsBackend; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f3211190..f52d839d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -256,7 +256,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, rendererParams, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, + mapConfigProvider.analysesResults, + rendererParams, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -279,7 +281,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { +MapController.prototype.afterLayergroupCreate = +function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { var self = this; var username = req.context.user; diff --git a/package.json b/package.json index 9bbb5125..b14b07d3 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "strftime": "~0.8.2" }, "scripts": { + "lint": "jshint lib test", "preinstall": "make pre-install", "test": "make test-all" }, diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 275ebab7..210418ac 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1041,7 +1041,7 @@ describe(suiteName, function() { ); }); } - + // See https://github.com/CartoDB/Windshaft-cartodb/issues/91 // and https://github.com/CartoDB/Windshaft-cartodb/issues/38 it("tiles for private tables can be fetched with api_key", function(done) { diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js index 4571bf6a..bfd76394 100644 --- a/test/acceptance/stats/multilayer_stats.js +++ b/test/acceptance/stats/multilayer_stats.js @@ -49,8 +49,10 @@ describe('multilayer stats disabled', function() { var cartocssVersion = '2.3.0'; var cartocss = '#layer { line-width:16; }'; - var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; - var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " + + "st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; + var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " + + "st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator"; var httpLayer = { type: 'http', diff --git a/yarn.lock b/yarn.lock index 1addd775..e8a320d2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -223,6 +223,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -231,14 +239,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -738,7 +738,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@3.2.3: +glob@3.2.3, "glob@~ 3.2.1": version "3.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.3.tgz#e313eeb249c7affaa5c475286b0e115b59839467" dependencies: @@ -777,13 +777,6 @@ glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" -"glob@~ 3.2.1": - version "3.2.11" - resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d" - dependencies: - inherits "2" - minimatch "0.3" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1227,13 +1220,6 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -minimatch@0.3: - version "0.3.0" - resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.3.0.tgz#275d8edaac4f1bb3326472089e7949c8394699dd" - dependencies: - lru-cache "2" - sigmund "~1.0.0" - minimatch@1.0.x: version "1.0.0" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-1.0.0.tgz#e0dd2120b49e1b724ce8d714c520822a9438576d" @@ -1523,7 +1509,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2275,6 +2261,12 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" +windshaft-stats@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/windshaft-stats/-/windshaft-stats-0.0.1.tgz#5af57a2ea71c67cdf6d8628941c88b826100d7cb" + dependencies: + debug "~2.2.0" + windshaft@cartodb/windshaft#548-vector-buffer-size: version "3.1.1" resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" From daf19c5e27479e5c8acfc5b6c73ef593cd9b1552 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 10 May 2017 17:17:01 +0200 Subject: [PATCH 11/29] Stats backend only provides stats not metadata --- lib/cartodb/backends/stats.js | 16 ++++++------- lib/cartodb/controllers/map.js | 24 +++++-------------- .../provider/dummy-mapconfig-provider.js | 20 ---------------- lib/cartodb/server.js | 2 +- .../stats/mapnik_stats_layergroup.js | 3 +++ test/acceptance/stats/multilayer_stats.js | 12 ++++++---- 6 files changed, 25 insertions(+), 52 deletions(-) delete mode 100644 lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index d8f83c72..374de6ff 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -1,20 +1,18 @@ var windshaftStats = require('windshaft-stats'); -function StatsBackend(rendererCache) { - this.rendererCache = rendererCache; +function StatsBackend() { } module.exports = StatsBackend; -StatsBackend.prototype.getStats = function(params, dbConnection, mapConfigProvider, callback) { +StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; - var layerStats; - if (layerMetadataEnabled) { - layerStats = windshaftStats(); - layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; + if (layerMetadataEnabled && layerStatsEnabled) { + var layerStats = windshaftStats(); + layerStats.getStats(mapConfig, params, dbConnection, callback); } else { - layerStats = windshaftStats('torque'); - layerStats.getStats(this.rendererCache, params, dbConnection, mapConfigProvider, callback); + callback(null, null); } }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f52d839d..353b459d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -19,7 +19,7 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider'); -var DummyMapConfigProvider = require('../models/mapconfig/provider/dummy-mapconfig-provider'); + /** * @param {AuthApi} authApi @@ -364,30 +364,18 @@ function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callb return null; }, - function fetchLayersMetadata(err) { + function fetchLayersStats(err) { assert.ifError(err); var next = this; - var mapConfigProvider = new DummyMapConfigProvider(mapconfig, rendererParams); - self.statsBackend.getStats(rendererParams, dbConnection, mapConfigProvider, function(err, layersStats) { + self.statsBackend.getStats(mapconfig, rendererParams, dbConnection, function(err, layersStats) { if (err) { return next(err); } if (layersStats) { - layergroup.metadata = layergroup.metadata || {}; - layergroup.metadata.layers = layersStats; - - // backwards compatibility for torque - var torqueMetadata = layersStats.reduce(function(acc, layer, layerId) { - if (layer.type === 'torque') { - acc[layerId] = layer.meta; - } - return acc; - }, {}); - if (Object.keys(torqueMetadata).length) { - layergroup.metadata.torque = torqueMetadata; - } + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); } - return next(); }); }, diff --git a/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js b/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js deleted file mode 100644 index 0b8942c5..00000000 --- a/lib/cartodb/models/mapconfig/provider/dummy-mapconfig-provider.js +++ /dev/null @@ -1,20 +0,0 @@ -var util = require('util'); -var MapStoreMapConfigProvider = require('windshaft').model.provider.MapStoreMapConfig; - -function DummyMapConfigProvider(mapConfig, params) { - MapStoreMapConfigProvider.call(this, undefined, params); - - this.mapConfig = mapConfig; -} - -util.inherits(DummyMapConfigProvider, MapStoreMapConfigProvider); - -module.exports = DummyMapConfigProvider; - -DummyMapConfigProvider.prototype.setParams = function(params) { - this.params = params; -}; - -DummyMapConfigProvider.prototype.getMapConfig = function(callback) { - return callback(null, this.mapConfig, this.params, {}); -}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 5ce31f13..8cf42220 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -152,7 +152,7 @@ module.exports = function(serverOptions) { var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis); - var statsBackend = new StatsBackend(rendererCache); + var statsBackend = new StatsBackend(); var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache(); app.layergroupAffectedTablesCache = layergroupAffectedTablesCache; diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 4a327a01..1feb477e 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -6,11 +6,14 @@ var TestClient = require('../../support/test-client'); describe('Create mapnik layergroup', function() { before(function() { this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerStats = true; }); after(function() { global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); var cartocssVersion = '2.3.0'; diff --git a/test/acceptance/stats/multilayer_stats.js b/test/acceptance/stats/multilayer_stats.js index bfd76394..ced63b0e 100644 --- a/test/acceptance/stats/multilayer_stats.js +++ b/test/acceptance/stats/multilayer_stats.js @@ -6,11 +6,15 @@ var TestClient = require('../../support/test-client'); describe('multilayer stats disabled', function() { before(function () { - global.environment.enabledFeatures.layerMetadata = false; + this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; + this.layerStatsConfig = global.environment.enabledFeatures.layerStats; + global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerStats = false; }); after(function () { - global.environment.enabledFeatures.layerMetadata = true; + global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; + global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); @@ -28,9 +32,9 @@ describe('multilayer stats disabled', function() { assert.ifError(err); layergroup.metadata.layers.forEach(function (layer) { if (layer.type !== 'torque' && layer.type !== 'mapnik') { - assert.ok('stats' in layer.meta); + assert.ok(!('stats' in layer.meta)); } else if (layer.type !== 'torque') { - assert.ok('stats' in layer.meta); + assert.ok(!('stats' in layer.meta)); assert.ok('cartocss' in layer.meta); } else { assert.ok('cartocss' in layer.meta); From 968677e27566f386bffb9325a14b795cfc8b03ef Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 10 May 2017 17:26:12 +0200 Subject: [PATCH 12/29] package.json poiting to staging branches --- package.json | 4 ++-- yarn.lock | 42 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index b14b07d3..6d9065a7 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#548-vector-buffer-size", - "windshaft-stats": "0.0.1", + "windshaft": "cartodb/windshaft#644_layergroup_stats", + "windshaft-stats": "cartodb/windshaft-stats#adapt_existing_code", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index e8a320d2..cbdecfba 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,6 +254,15 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" +cartodb-psql@~0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" + dependencies: + debug "~2.2.0" + pg cartodb/node-postgres#6.1.2-cdb1 + step "~0.0.6" + underscore "~1.6.0" + cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" @@ -402,6 +411,10 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" +d3-queue@~3.0.5: + version "3.0.7" + resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -430,6 +443,12 @@ debug@^1.0.4: dependencies: ms "0.6.2" +debug@~2.6.6: + version "2.6.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" + dependencies: + ms "0.7.3" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1293,6 +1312,10 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1509,7 +1532,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2002,6 +2025,10 @@ step@~0.0.5, step@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" +step@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" + strftime@~0.8.2: version "0.8.4" resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" @@ -2261,15 +2288,18 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@0.0.1: +windshaft-stats@cartodb/windshaft-stats#adapt_existing_code: version "0.0.1" - resolved "https://registry.yarnpkg.com/windshaft-stats/-/windshaft-stats-0.0.1.tgz#5af57a2ea71c67cdf6d8628941c88b826100d7cb" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/f56056e8f212d133a68340b0367a6e4a7c721c7f" dependencies: - debug "~2.2.0" + cartodb-psql "~0.8.0" + d3-queue "~3.0.5" + debug "~2.6.6" + step "~1.0.0" -windshaft@cartodb/windshaft#548-vector-buffer-size: +windshaft@cartodb/windshaft#644_layergroup_stats: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/1f318899ba04f770d6bb14256b49a6d8fc0091b0" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 61f9ea6e865c45e335608a0ea757c92465f4af56 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 12:47:35 +0200 Subject: [PATCH 13/29] Changed stats name from featureCount to estimatedFeatureCount --- .../stats/mapnik_stats_layergroup.js | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 1feb477e..8bee7fee 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -152,8 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -170,11 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -191,12 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -215,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -236,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -256,7 +251,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); + assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); testClient.drain(done); }); }); From 35df0c3a686ed608295eafaf7a455bc500222406 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 14:26:00 +0200 Subject: [PATCH 14/29] Change from branch to commits for staging dependencies --- package.json | 4 ++-- yarn.lock | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 6d9065a7..217812fc 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#644_layergroup_stats", - "windshaft-stats": "cartodb/windshaft-stats#adapt_existing_code", + "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", + "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index cbdecfba..57e245f1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1532,7 +1532,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2288,18 +2288,18 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#adapt_existing_code: +windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/f56056e8f212d133a68340b0367a6e4a7c721c7f" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" dependencies: cartodb-psql "~0.8.0" d3-queue "~3.0.5" debug "~2.6.6" step "~1.0.0" -windshaft@cartodb/windshaft#644_layergroup_stats: +windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/1f318899ba04f770d6bb14256b49a6d8fc0091b0" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From f6fff6953e2100414f7f7b9dad8352626da0ddb0 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:50:57 +0200 Subject: [PATCH 15/29] We leave only one feature flag for stats --- lib/cartodb/backends/stats.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 374de6ff..489c1740 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -7,9 +7,8 @@ module.exports = StatsBackend; StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; - var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; - if (layerMetadataEnabled && layerStatsEnabled) { + if (layerStatsEnabled) { var layerStats = windshaftStats(); layerStats.getStats(mapConfig, params, dbConnection, callback); } else { From 44cca385381f39ae1752440471ea1f5355e97d9c Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:51:45 +0200 Subject: [PATCH 16/29] Change to tests for layer stats because now uses CDB_EstimateRowCount fuction --- .../stats/mapnik_stats_layergroup.js | 30 +++++++++---------- test/support/prepare_db.sh | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 8bee7fee..77288d51 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -22,7 +22,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer1 = { type: 'mapnik', options: { - sql: 'select * from test_table limit 2', + sql: 'select * from test_table limit 1', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -40,7 +40,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer3 = { type: 'mapnik', options: { - sql: 'select * from test_table_3 limit 2', + sql: 'select * from test_table_3 limit 3', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3); testClient.drain(done); }); }); @@ -152,7 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -169,9 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -188,10 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -210,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -231,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 4fbf4b7a..dcf1cba7 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' - REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount' CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} From 7596df96ed836ac3a0029c7959eb09392ce8f9e5 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:58:13 +0200 Subject: [PATCH 17/29] package.json for staging --- package.json | 4 ++-- yarn.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 217812fc..adaa4cc1 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", - "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", + "windshaft": "cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd", + "windshaft-stats": "cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 57e245f1..0f4c6c2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2288,18 +2288,18 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: +windshaft-stats@cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d: version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/5f41e0a69763692f45f76cbae7f1e784186e1e7d" dependencies: cartodb-psql "~0.8.0" d3-queue "~3.0.5" debug "~2.6.6" step "~1.0.0" -windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: +windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/0675de5edcb2a24179dcd6224c3f309b68f17bbd" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 1a7fd9bf3170aba93d8a3e02715789df2679abe7 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 12:47:35 +0200 Subject: [PATCH 18/29] Changed stats name from featureCount to estimatedFeatureCount --- .../stats/mapnik_stats_layergroup.js | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 1feb477e..8bee7fee 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -152,8 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -170,11 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[0].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); testClient.drain(done); }); }); @@ -191,12 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); - assert.equal(layergroup.metadata.layers[1].meta.stats[1].featureCount, 5); - assert.ok(!layergroup.metadata.layers[1].meta.stats[2]); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -215,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -236,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats[0].featureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); @@ -256,7 +251,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); // we don't care about stats here as is an aliased column - assert.ok(layergroup.metadata.layers[0].meta.stats[0].hasOwnProperty('featureCount')); + assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount')); testClient.drain(done); }); }); From 522c86e6f2c3f59f09da0c7e56013d0f660597d6 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 11 May 2017 14:26:00 +0200 Subject: [PATCH 19/29] Change from branch to commits for staging dependencies --- package.json | 4 ++-- yarn.lock | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index b14b07d3..217812fc 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#548-vector-buffer-size", - "windshaft-stats": "0.0.1", + "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", + "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index e8a320d2..8d02a27e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2261,15 +2261,15 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@0.0.1: +windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: version "0.0.1" - resolved "https://registry.yarnpkg.com/windshaft-stats/-/windshaft-stats-0.0.1.tgz#5af57a2ea71c67cdf6d8628941c88b826100d7cb" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" dependencies: debug "~2.2.0" -windshaft@cartodb/windshaft#548-vector-buffer-size: +windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/09493bfff94371989532bef6dff734e4cd5cf058" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 59585b5cd9fca14d21cdbdae96557c4272f00e2b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:50:57 +0200 Subject: [PATCH 20/29] We leave only one feature flag for stats --- lib/cartodb/backends/stats.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 374de6ff..489c1740 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -7,9 +7,8 @@ module.exports = StatsBackend; StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; - var layerMetadataEnabled = enabledFeatures ? enabledFeatures.layerMetadata : false; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; - if (layerMetadataEnabled && layerStatsEnabled) { + if (layerStatsEnabled) { var layerStats = windshaftStats(); layerStats.getStats(mapConfig, params, dbConnection, callback); } else { From 5417933eccdfac42e6d9fc4770c3268da966439b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 12:51:45 +0200 Subject: [PATCH 21/29] Change to tests for layer stats because now uses CDB_EstimateRowCount fuction --- .../stats/mapnik_stats_layergroup.js | 30 +++++++++---------- test/support/prepare_db.sh | 2 +- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 8bee7fee..77288d51 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -22,7 +22,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer1 = { type: 'mapnik', options: { - sql: 'select * from test_table limit 2', + sql: 'select * from test_table limit 1', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -40,7 +40,7 @@ describe('Create mapnik layergroup', function() { var mapnikLayer3 = { type: 'mapnik', options: { - sql: 'select * from test_table_3 limit 2', + sql: 'select * from test_table_3 limit 3', cartocss_version: cartocssVersion, cartocss: cartocss } @@ -95,7 +95,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); testClient.drain(done); }); }); @@ -112,9 +112,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); testClient.drain(done); }); }); @@ -132,11 +132,11 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2); assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2)); - assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3); testClient.drain(done); }); }); @@ -152,7 +152,7 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -169,9 +169,9 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); testClient.drain(done); }); }); @@ -188,10 +188,10 @@ describe('Create mapnik layergroup', function() { testClient.getLayergroup(function(err, layergroup) { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3); assert.ok(!layergroup.metadata.layers[0].meta.stats[1]); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1)); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 10); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); assert.ok(!layergroup.metadata.layers[2]); testClient.drain(done); }); @@ -210,7 +210,7 @@ describe('Create mapnik layergroup', function() { assert.ok(!err); assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[0].type, 'mapnik'); - assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[1].type, 'http'); testClient.drain(done); @@ -231,7 +231,7 @@ describe('Create mapnik layergroup', function() { assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); assert.equal(layergroup.metadata.layers[0].type, 'http'); assert.ok(!layergroup.metadata.layers[0].meta.cartocss); - assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5); + assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1); assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 4fbf4b7a..dcf1cba7 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database" LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish' - REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ' + REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount' CURL_ARGS="" for i in ${REMOTE_SQL_SCRIPTS} From 5969c99e8a5168f2afb0c6f2126c381562ca70a2 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 16:39:54 +0200 Subject: [PATCH 22/29] Removed not used parameters for layer stats --- lib/cartodb/backends/stats.js | 6 +++--- lib/cartodb/controllers/map.js | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 489c1740..eb800e3c 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -5,13 +5,13 @@ function StatsBackend() { module.exports = StatsBackend; -StatsBackend.prototype.getStats = function(mapConfig, params, dbConnection, callback) { +StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; if (layerStatsEnabled) { var layerStats = windshaftStats(); - layerStats.getStats(mapConfig, params, dbConnection, callback); + layerStats.getStats(mapConfig, dbConnection, callback); } else { - callback(null, null); + return callback(null, null); } }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 353b459d..cd5bbed3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -367,7 +367,7 @@ function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callb function fetchLayersStats(err) { assert.ifError(err); var next = this; - self.statsBackend.getStats(mapconfig, rendererParams, dbConnection, function(err, layersStats) { + self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) { if (err) { return next(err); } From 422867762b236ef1eccb80f5da5694d4d625ba9f Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Fri, 12 May 2017 17:14:07 +0200 Subject: [PATCH 23/29] package.json for staging --- package.json | 4 ++-- yarn.lock | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 217812fc..de4f6259 100644 --- a/package.json +++ b/package.json @@ -39,8 +39,8 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef", - "windshaft-stats": "cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95", + "windshaft": "cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd", + "windshaft-stats": "cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 8d02a27e..3092b78f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,6 +254,15 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" +cartodb-psql@~0.8.0: + version "0.8.0" + resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" + dependencies: + debug "~2.2.0" + pg cartodb/node-postgres#6.1.2-cdb1 + step "~0.0.6" + underscore "~1.6.0" + cartodb-query-tables@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz#b4d672accde04da5b890a5d56a87b761fa7eec44" @@ -402,6 +411,10 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" +d3-queue@~3.0.5: + version "3.0.7" + resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" + dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -430,6 +443,12 @@ debug@^1.0.4: dependencies: ms "0.6.2" +debug@~2.6.6: + version "2.6.6" + resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" + dependencies: + ms "0.7.3" + decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1293,6 +1312,10 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" +ms@0.7.3: + version "0.7.3" + resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" + mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1509,7 +1532,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1: +pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2261,15 +2284,17 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#0d269b812b7665e9b2fd3fee4deaa37fb7c80f95: +windshaft-stats@cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f: version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/0d269b812b7665e9b2fd3fee4deaa37fb7c80f95" + resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/16a973ba9addf88312ba92047e41108be2c3434f" dependencies: - debug "~2.2.0" + cartodb-psql "~0.8.0" + d3-queue "~3.0.5" + debug "~2.6.6" -windshaft@cartodb/windshaft#f91c03e888ab774af6443f04970f6a74ff12a1ef: +windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/f91c03e888ab774af6443f04970f6a74ff12a1ef" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/0675de5edcb2a24179dcd6224c3f309b68f17bbd" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 29361f5392a1f09b4975cd4db9b51c113cd4afa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 16 May 2017 12:07:17 +0200 Subject: [PATCH 24/29] Use windhshaft with latest changes for layer stats --- package.json | 2 +- yarn.lock | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/package.json b/package.json index adaa4cc1..8ad0eda7 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.0", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd", + "windshaft": "cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea", "windshaft-stats": "cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d", "yargs": "~5.0.0" }, diff --git a/yarn.lock b/yarn.lock index 0f4c6c2a..05642fd5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,7 +254,7 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" -cartodb-psql@~0.8.0: +cartodb-psql@0.8.0, cartodb-psql@~0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" dependencies: @@ -2117,17 +2117,17 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#buffer-size: +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb1: version "2.3.1-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/9df4bde07f3cf9ffbda42657066559e2b8875e1b" + resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3f76c278c782e93d79045870387a0a06bace720b" dependencies: mapnik "~3.5.0" mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#fix-buffer-size-0: +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb1: version "0.6.18-cdb1" - resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/c67400573572e836609dd75a7acec8f414a50dbe" + resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/cf7e5b4633db653a889a6c6e6a5ddcbcf4ddc3b5" dependencies: generic-pool "~2.4.0" mapnik "3.5.14" @@ -2297,14 +2297,14 @@ windshaft-stats@cartodb/windshaft-stats#5f41e0a69763692f45f76cbae7f1e784186e1e7d debug "~2.6.6" step "~1.0.0" -windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: - version "3.1.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/0675de5edcb2a24179dcd6224c3f309b68f17bbd" +windshaft@cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea: + version "3.1.3" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/515efb2d9237a1511b81a61fd6f7524184df38ea" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 carto cartodb/carto#0.15.1-cdb3 - cartodb-psql "0.7.1" + cartodb-psql "0.8.0" debug "~2.2.0" dot "~1.0.2" grainstore "~1.6.0" @@ -2316,8 +2316,8 @@ windshaft@cartodb/windshaft#0675de5edcb2a24179dcd6224c3f309b68f17bbd: sphericalmercator "1.0.4" step "~0.0.6" tilelive "5.12.2" - tilelive-bridge cartodb/tilelive-bridge#buffer-size - tilelive-mapnik cartodb/tilelive-mapnik#fix-buffer-size-0 + tilelive-bridge cartodb/tilelive-bridge#2.3.1-cdb1 + tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb1 torque.js "~2.11.0" underscore "~1.6.0" From c65518cf41c135cd0289b8fb87176915862d7004 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 17 May 2017 20:16:43 +0200 Subject: [PATCH 25/29] Get back layer-stats from windshaft-stats --- .../backends/layer-stats/empty-layer-stats.js | 16 ++ lib/cartodb/backends/layer-stats/factory.js | 23 +++ .../backends/layer-stats/layer-stats.js | 45 ++++++ .../layer-stats/mapnik-layer-stats.js | 28 ++++ .../backends/layer-stats/query-utils.js | 26 +++ .../layer-stats/torque-layer-stats.js | 16 ++ lib/cartodb/backends/stats.js | 5 +- package.json | 1 - .../layer-stats/mapnik-layer-stats.js | 153 ++++++++++++++++++ .../layer-stats/torque-layer-stats.js | 36 +++++ yarn.lock | 30 +--- 11 files changed, 347 insertions(+), 32 deletions(-) create mode 100644 lib/cartodb/backends/layer-stats/empty-layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/factory.js create mode 100644 lib/cartodb/backends/layer-stats/layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/mapnik-layer-stats.js create mode 100644 lib/cartodb/backends/layer-stats/query-utils.js create mode 100644 lib/cartodb/backends/layer-stats/torque-layer-stats.js create mode 100644 test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js create mode 100644 test/unit/cartodb/backends/layer-stats/torque-layer-stats.js diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js new file mode 100644 index 00000000..0923b59c --- /dev/null +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -0,0 +1,16 @@ +function EmptyLayerStats(types) { + this._types = types || {}; +} + +EmptyLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +EmptyLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + process.nextTick(function() { + callback(null, {}); + }); +}; + +module.exports = EmptyLayerStats; diff --git a/lib/cartodb/backends/layer-stats/factory.js b/lib/cartodb/backends/layer-stats/factory.js new file mode 100644 index 00000000..8aacdb5a --- /dev/null +++ b/lib/cartodb/backends/layer-stats/factory.js @@ -0,0 +1,23 @@ +var LayerStats = require('./layer-stats'); +var EmptyLayerStats = require('./empty-layer-stats'); +var MapnikLayerStats = require('./mapnik-layer-stats'); +var TorqueLayerStats = require('./torque-layer-stats'); + +module.exports = function LayerStatsFactory(type) { + var layerStatsIterator = []; + var selectedType = type || 'ALL'; + + if (selectedType === 'ALL') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true })); + layerStatsIterator.push(new MapnikLayerStats()); + layerStatsIterator.push(new TorqueLayerStats()); + } else if (selectedType === 'mapnik') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true })); + layerStatsIterator.push(new MapnikLayerStats()); + } else if (selectedType === 'torque') { + layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true })); + layerStatsIterator.push(new TorqueLayerStats()); + } + + return new LayerStats(layerStatsIterator); +}; diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js new file mode 100644 index 00000000..bc1787c2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -0,0 +1,45 @@ +var d3 = require('d3-queue'); + +function LayerStats(layerStatsIterator) { + this.layerStatsIterator = layerStatsIterator; +} + +LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) { + var self = this; + var stats = []; + + if (!mapConfig.getLayers().length) { + return callback(null, stats); + } + var metaQueue = d3.queue(mapConfig.getLayers().length); + mapConfig.getLayers().forEach(function (layer, layerId) { + var layerType = mapConfig.layerType(layerId); + + for (var i = 0; i < self.layerStatsIterator.length; i++) { + if (self.layerStatsIterator[i].is(layerType)) { + var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]); + metaQueue.defer(getStats, layer, dbConnection); + break; + } + } + }); + + metaQueue.awaitAll(function (err, results) { + if (err) { + return callback(err); + } + + if (!results) { + return callback(null, null); + } + + mapConfig.getLayers().forEach(function (layer, layerIndex) { + stats[layerIndex] = results[layerIndex]; + }); + + return callback(err, stats); + }); + +}; + +module.exports = LayerStats; diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..7dfd97af --- /dev/null +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,28 @@ +var queryUtils = require('./query-utils'); + +function MapnikLayerStats () { + this._types = { + mapnik: true, + cartodb: true + }; +} + +MapnikLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +MapnikLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql); + // This query would gather stats for postgresql table if not exists + dbConnection.query(queryRowCountSql, function (err, res) { + if (err) { + return callback(null, {estimatedFeatureCount: -1}); + } else { + // We decided that the relation is 1 row == 1 feature + return callback(null, {estimatedFeatureCount: res.rows[0].rows}); + } + }); +}; + +module.exports = MapnikLayerStats; diff --git a/lib/cartodb/backends/layer-stats/query-utils.js b/lib/cartodb/backends/layer-stats/query-utils.js new file mode 100644 index 00000000..47d730f4 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/query-utils.js @@ -0,0 +1,26 @@ +function prepareQuery(sql) { + var affectedTableRegexCache = { + bbox: /!bbox!/g, + scale_denominator: /!scale_denominator!/g, + pixel_width: /!pixel_width!/g, + pixel_height: /!pixel_height!/g + }; + + return sql + .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') + .replace(affectedTableRegexCache.scale_denominator, '0') + .replace(affectedTableRegexCache.pixel_width, '1') + .replace(affectedTableRegexCache.pixel_height, '1'); +} + +module.exports.extractTableNames = function extractTableNames(query) { + return [ + 'SELECT * FROM CDB_QueryTablesText($windshaft$', + prepareQuery(query), + '$windshaft$) as tablenames' + ].join(''); +}; + +module.exports.getQueryRowCount = function getQueryRowEstimation(query) { + return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; +}; diff --git a/lib/cartodb/backends/layer-stats/torque-layer-stats.js b/lib/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..00b4def2 --- /dev/null +++ b/lib/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,16 @@ +function TorqueLayerStats() { + this._types = { + torque: true + }; +} + +TorqueLayerStats.prototype.is = function (type) { + return this._types[type] ? this._types[type] : false; +}; + +TorqueLayerStats.prototype.getStats = +function (layer, dbConnection, callback) { + return callback(null, {}); +}; + +module.exports = TorqueLayerStats; diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index eb800e3c..20f89a0f 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -1,4 +1,4 @@ -var windshaftStats = require('windshaft-stats'); +var layerStats = require('./layer-stats/factory'); function StatsBackend() { } @@ -9,8 +9,7 @@ StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { var enabledFeatures = global.environment.enabledFeatures; var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false; if (layerStatsEnabled) { - var layerStats = windshaftStats(); - layerStats.getStats(mapConfig, dbConnection, callback); + layerStats().getStats(mapConfig, dbConnection, callback); } else { return callback(null, null); } diff --git a/package.json b/package.json index 4affa452..c49f1fe4 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,6 @@ "turbo-carto": "0.19.0", "underscore": "~1.6.0", "windshaft": "cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea", - "windshaft-stats": "cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js new file mode 100644 index 00000000..6e47297d --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -0,0 +1,153 @@ +var assert = require('assert'); +var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +function getDbConnectionMock () { + return { + query: function(sql, callback) { + return callback(null, { + rows: [{rows: 1}] + }); + } + }; +} + +describe('mapnik-layer-stats', function() { + + beforeEach(function () { + this.dbConnectionMock = getDbConnectionMock(); + this.rendererCacheMock = {}; + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + } + ] + }; + + var testMapConfigTwoLayers = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0' + } + }, + ] + }; + + var testMapConfigOneLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + ] + }; + + var testMapConfigTwoLayerTwoTables = { + version: '1.5.0', + layers: [ + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_1', 'test_table_2'] + } + }, + { + type: 'mapnik', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + affected_tables: ['test_table_3', 'test_table_4'] + } + }, + ] + }; + + it('should return 1 feature for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayers); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); + + it('should return 1 feature for one layers with two tables', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables); + var layer = mapConfig.getLayer(0); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer, this.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + + it('should return 1 feature for two layers and two tables', function(done) { + var self = this; + var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables); + var layer0 = mapConfig.getLayer(0); + var layer1 = mapConfig.getLayer(1); + var testSubject = new MapnikLayerStats(); + testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) { + assert.ifError(err); + assert.equal(result.estimatedFeatureCount, 1); + done(); + }); + }); + }); +}); diff --git a/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js new file mode 100644 index 00000000..c9adfad9 --- /dev/null +++ b/test/unit/cartodb/backends/layer-stats/torque-layer-stats.js @@ -0,0 +1,36 @@ +var assert = require('assert'); +var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats'); +var MapConfig = require('windshaft').model.MapConfig; + +describe('torque-layer-stats', function () { + + beforeEach(function () { + this.params = {}; + }); + + var testMapConfigOneLayer = { + version: '1.5.0', + layers: [ + { + type: 'torque', + options: { + sql: 'select * from test_table limit 2', + cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }', + cartocss_version: '2.3.0', + } + }, + ] + }; + + it('should return torque stats for one layer', function(done) { + var mapConfig = MapConfig.create(testMapConfigOneLayer); + var layerId = 0; + var layer = mapConfig.getLayer(layerId); + var testSubject = new TorqueLayerStats(); + testSubject.getStats(layer, {}, function (err, result) { + assert.ifError(err); + assert.deepEqual({}, result); + done(); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 8d8ec0d0..a1b13067 100644 --- a/yarn.lock +++ b/yarn.lock @@ -254,7 +254,7 @@ cartodb-psql@0.7.1, cartodb-psql@~0.7.1: step "~0.0.6" underscore "~1.6.0" -cartodb-psql@0.8.0, cartodb-psql@~0.8.0: +cartodb-psql@0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/cartodb-psql/-/cartodb-psql-0.8.0.tgz#d3811f706dae2c3bc82365c5d25af13c4235ba37" dependencies: @@ -411,10 +411,6 @@ d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" -d3-queue@~3.0.5: - version "3.0.7" - resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-3.0.7.tgz#c93a2e54b417c0959129d7d73f6cf7d4292e7618" - dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -443,12 +439,6 @@ debug@^1.0.4: dependencies: ms "0.6.2" -debug@~2.6.6: - version "2.6.6" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.6.tgz#a9fa6fbe9ca43cf1e79f73b75c0189cbb7d6db5a" - dependencies: - ms "0.7.3" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -1312,10 +1302,6 @@ ms@0.7.1: version "0.7.1" resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098" -ms@0.7.3: - version "0.7.3" - resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.3.tgz#708155a5e44e33f5fd0fc53e81d0d40a91be1fff" - mv@~2: version "2.1.1" resolved "https://registry.yarnpkg.com/mv/-/mv-2.1.1.tgz#ae6ce0d6f6d5e0a4f7d893798d03c1ea9559b6a2" @@ -1532,7 +1518,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -pg@cartodb/node-postgres#6.1.2-cdb1, "pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -2025,10 +2011,6 @@ step@~0.0.5, step@~0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/step/-/step-0.0.6.tgz#143e7849a5d7d3f4a088fe29af94915216eeede2" -step@~1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/step/-/step-1.0.0.tgz#b300e9d2ae9057d4d78633aae2303813a94bdff2" - strftime@~0.8.2: version "0.8.4" resolved "https://registry.yarnpkg.com/strftime/-/strftime-0.8.4.tgz#86b15949845e7de20c0c3d69db2b74fb73e4d25e" @@ -2288,14 +2270,6 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft-stats@cartodb/windshaft-stats#16a973ba9addf88312ba92047e41108be2c3434f: - version "0.0.1" - resolved "https://codeload.github.com/cartodb/windshaft-stats/tar.gz/16a973ba9addf88312ba92047e41108be2c3434f" - dependencies: - cartodb-psql "~0.8.0" - d3-queue "~3.0.5" - debug "~2.6.6" - windshaft@cartodb/windshaft#515efb2d9237a1511b81a61fd6f7524184df38ea: version "3.1.3" resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/515efb2d9237a1511b81a61fd6f7524184df38ea" From ed56094be2c3fc214dd7a3ed8f143df8cf1944b9 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Thu, 18 May 2017 11:51:12 +0200 Subject: [PATCH 26/29] PR changes --- lib/cartodb/backends/layer-stats/empty-layer-stats.js | 2 +- lib/cartodb/backends/layer-stats/layer-stats.js | 4 ++-- .../backends/layer-stats/mapnik-layer-stats.js | 2 +- lib/cartodb/controllers/map.js | 11 ++++------- .../{backends/layer-stats => utils}/query-utils.js | 0 5 files changed, 8 insertions(+), 11 deletions(-) rename lib/cartodb/{backends/layer-stats => utils}/query-utils.js (100%) diff --git a/lib/cartodb/backends/layer-stats/empty-layer-stats.js b/lib/cartodb/backends/layer-stats/empty-layer-stats.js index 0923b59c..0760c0b6 100644 --- a/lib/cartodb/backends/layer-stats/empty-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/empty-layer-stats.js @@ -8,7 +8,7 @@ EmptyLayerStats.prototype.is = function (type) { EmptyLayerStats.prototype.getStats = function (layer, dbConnection, callback) { - process.nextTick(function() { + setImmediate(function() { callback(null, {}); }); }; diff --git a/lib/cartodb/backends/layer-stats/layer-stats.js b/lib/cartodb/backends/layer-stats/layer-stats.js index bc1787c2..2464fb22 100644 --- a/lib/cartodb/backends/layer-stats/layer-stats.js +++ b/lib/cartodb/backends/layer-stats/layer-stats.js @@ -1,4 +1,4 @@ -var d3 = require('d3-queue'); +var queue = require('queue-async'); function LayerStats(layerStatsIterator) { this.layerStatsIterator = layerStatsIterator; @@ -11,7 +11,7 @@ LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) { if (!mapConfig.getLayers().length) { return callback(null, stats); } - var metaQueue = d3.queue(mapConfig.getLayers().length); + var metaQueue = queue(mapConfig.getLayers().length); mapConfig.getLayers().forEach(function (layer, layerId) { var layerType = mapConfig.layerType(layerId); diff --git a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js index 7dfd97af..c060f964 100644 --- a/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js +++ b/lib/cartodb/backends/layer-stats/mapnik-layer-stats.js @@ -1,4 +1,4 @@ -var queryUtils = require('./query-utils'); +var queryUtils = require('../../utils/query-utils'); function MapnikLayerStats () { this._types = { diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index cd5bbed3..3bb72574 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -166,7 +166,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, req.params, this); + self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this); }, function finish(err, layergroup) { if (err) { @@ -219,8 +219,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var mapConfigProvider; var mapConfig; - var rendererParams; - step( function setupParams(){ self.req2params(req, this); @@ -244,9 +242,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); mapConfigProvider.getMapConfig(this); }, - function createLayergroup(err, mapConfig_, _rendererParams) { + function createLayergroup(err, mapConfig_, rendererParams) { assert.ifError(err); - rendererParams = _rendererParams; mapConfig = mapConfig_; self.mapBackend.createLayergroup( mapConfig, rendererParams, @@ -258,7 +255,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn assert.ifError(err); self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, - rendererParams, this); + this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -282,7 +279,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }; MapController.prototype.afterLayergroupCreate = -function(req, res, mapconfig, layergroup, analysesResults, rendererParams, callback) { +function(req, res, mapconfig, layergroup, analysesResults, callback) { var self = this; var username = req.context.user; diff --git a/lib/cartodb/backends/layer-stats/query-utils.js b/lib/cartodb/utils/query-utils.js similarity index 100% rename from lib/cartodb/backends/layer-stats/query-utils.js rename to lib/cartodb/utils/query-utils.js From 440953b1cd6a81c7e8923edf3bfabd6756ce74ce Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Tue, 30 May 2017 10:47:47 +0200 Subject: [PATCH 27/29] If we have the stats FF disbabled return empty array instead of null But we keep checking for elements in the returned object because we don't want to include the stats property if the FF is disabled --- lib/cartodb/backends/stats.js | 2 +- lib/cartodb/controllers/map.js | 2 +- .../stats/mapnik_stats_layergroup.js | 27 ++++++++++++++++--- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/backends/stats.js b/lib/cartodb/backends/stats.js index 20f89a0f..b0385bac 100644 --- a/lib/cartodb/backends/stats.js +++ b/lib/cartodb/backends/stats.js @@ -11,6 +11,6 @@ StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) { if (layerStatsEnabled) { layerStats().getStats(mapConfig, dbConnection, callback); } else { - return callback(null, null); + return callback(null, []); } }; diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 3bb72574..282c3145 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -368,7 +368,7 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) { if (err) { return next(err); } - if (layersStats) { + if (layersStats.length > 0) { layergroup.metadata.layers.forEach(function (layer, index) { layer.meta.stats = layersStats[index]; }); diff --git a/test/acceptance/stats/mapnik_stats_layergroup.js b/test/acceptance/stats/mapnik_stats_layergroup.js index 77288d51..cdac7bb1 100644 --- a/test/acceptance/stats/mapnik_stats_layergroup.js +++ b/test/acceptance/stats/mapnik_stats_layergroup.js @@ -5,14 +5,11 @@ var TestClient = require('../../support/test-client'); describe('Create mapnik layergroup', function() { before(function() { - this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata; this.layerStatsConfig = global.environment.enabledFeatures.layerStats; - global.environment.enabledFeatures.layerMetadata = true; global.environment.enabledFeatures.layerStats = true; }); after(function() { - global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig; global.environment.enabledFeatures.layerStats = this.layerStatsConfig; }); @@ -255,4 +252,28 @@ describe('Create mapnik layergroup', function() { testClient.drain(done); }); }); + + it('should not include the stats part if the FF is disabled', function(done) { + global.environment.enabledFeatures.layerStats = false; + var testClient = new TestClient({ + version: '1.4.0', + layers: [ + httpLayer, + mapnikLayer1, + httpLayer + ] + }); + + testClient.getLayergroup(function(err, layergroup) { + assert.ok(!err); + assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0)); + assert.equal(layergroup.metadata.layers[0].type, 'http'); + assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0)); + assert.equal(layergroup.metadata.layers[1].type, 'mapnik'); + assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats')); + assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1)); + assert.equal(layergroup.metadata.layers[2].type, 'http'); + testClient.drain(done); + }); + }); }); From da3f30dd9f121d57cc3b86192cdd1db16d4af2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 31 May 2017 11:27:41 +0200 Subject: [PATCH 28/29] Update NEWS --- NEWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b8e12ce5..744e524c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,9 +1,10 @@ # Changelog -## 3.8.1 +## 3.9.0 Released 2017-mm-dd Announcements: + - Add support to retrieve info about layer stats in map instantiation. - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). - Remove promise polyfill from turbo-carto adapter From 2c509d97b1532ad514b4664eda96c2842cf9d42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 31 May 2017 11:46:46 +0200 Subject: [PATCH 29/29] Point windshaft to the last released version --- NEWS.md | 1 + package.json | 4 ++-- yarn.lock | 38 +++++++++++++++++++------------------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/NEWS.md b/NEWS.md index 744e524c..e0de3ca5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,7 @@ Released 2017-mm-dd Announcements: + - Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1). - Add support to retrieve info about layer stats in map instantiation. - Upgrades camshaft to [0.55.1](https://github.com/CartoDB/camshaft/releases/tag/0.55.1). - Remove promise polyfill from turbo-carto adapter diff --git a/package.json b/package.json index 48db22e6..f8f5a56c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "3.8.1", + "version": "3.9.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -39,7 +39,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.19.1", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#9624bb093cb2f7056ddcbacdfa5f4c276ed79edb", + "windshaft": "3.2.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 6d05a362..158b2529 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -209,7 +209,7 @@ camshaft@0.55.1: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -227,7 +227,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@cartodb/carto#0.15.1-cdb3: +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -235,14 +243,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -carto@cartodb/carto#0.15.1-cdb3: - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -1256,7 +1256,7 @@ mime@~1.2.11: version "3.0.3" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" dependencies: - brace-expansion "^1.1.7" + brace-expansion "^1.0.0" minimist@0.0.8, minimist@~0.0.1: version "0.0.8" @@ -1524,7 +1524,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "~1.0.0" -"pg@github:cartodb/node-postgres#6.1.2-cdb1": +pg@cartodb/node-postgres#6.1.2-cdb1: version "6.1.2" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3c81aea432ce58d20a795786c58bbb14f68f9689" dependencies: @@ -1745,7 +1745,7 @@ repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" -request@2.x, request@^2.55.0, request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1770,7 +1770,7 @@ request@2.x, request@^2.55.0, request@~2.79.0: tunnel-agent "~0.4.1" uuid "^3.0.0" -request@^2.69.0, request@^2.81.0: +request@^2.81.0: version "2.81.0" resolved "https://registry.yarnpkg.com/request/-/request-2.81.0.tgz#c6928946a0e06c5f8d6f8a9333469ffda46298a0" dependencies: @@ -2107,7 +2107,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb2": version "2.3.1-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/0346c634875ac87dbf8316cb81ac46d2c30fe313" dependencies: @@ -2115,7 +2115,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb2: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb2": version "0.6.18-cdb2" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/46f1adefee90f3f46c0ede5e0833f8522634a858" dependencies: @@ -2278,9 +2278,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#9624bb093cb2f7056ddcbacdfa5f4c276ed79edb: +windshaft@3.2.1: version "3.2.1" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/9624bb093cb2f7056ddcbacdfa5f4c276ed79edb" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.2.1.tgz#50a3afa6562315dd9e65e411660970e118f36c19" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2