From ad1abb28afea3c0c16f23ffd438cf9c9dedfee36 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 28 Sep 2017 12:08:22 +0200 Subject: [PATCH 001/167] Add configuration flag to enable/disable direct PgSQL MVT --- config/environments/development.js.example | 3 +++ config/environments/production.js.example | 3 +++ config/environments/staging.js.example | 3 +++ config/environments/test.js.example | 3 +++ lib/cartodb/server.js | 3 ++- lib/cartodb/server_options.js | 1 + 6 files changed, 15 insertions(+), 1 deletion(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 48d03592..f09a3808 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -107,6 +107,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/production.js.example b/config/environments/production.js.example index ba7fec6f..1dd80212 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -101,6 +101,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 1792779c..7dd9f5b0 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -101,6 +101,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 374467c0..b266c923 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -100,6 +100,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 350e52a0..723289ba 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -154,7 +154,8 @@ module.exports = function(serverOptions) { grainstore: serverOptions.grainstore, mapnik: serverOptions.renderer.mapnik }, - http: serverOptions.renderer.http + http: serverOptions.renderer.http, + mvt: serverOptions.renderer.mvt }); // initialize render cache diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 6d8575a7..25e60c2f 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -81,6 +81,7 @@ module.exports = { statsInterval: rendererConfig.statsInterval }, renderer: { + mvt: rendererConfig.mvt, mapnik: _.defaults(rendererConfig.mapnik, { geojson: { dbPoolParams: { From e5cae8b8e3e530fc891cfccb1be0e4c9cdbcfcd0 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 28 Sep 2017 13:24:50 +0200 Subject: [PATCH 002/167] Added flag documentation --- config/environments/development.js.example | 3 +++ config/environments/production.js.example | 3 +++ config/environments/staging.js.example | 3 +++ config/environments/test.js.example | 3 +++ 4 files changed, 12 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index f09a3808..2bc6d2d6 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -108,6 +108,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 1dd80212..87214c75 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -102,6 +102,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 7dd9f5b0..5e05c83f 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -102,6 +102,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index b266c923..c01d52d1 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -101,6 +101,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { From 54632485787b666978a1a691e1d413a8c8ad9fe5 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 28 Sep 2017 17:57:59 +0200 Subject: [PATCH 003/167] Changed PostGIS MVT flag name --- config/environments/development.js.example | 10 +++++++++- config/environments/production.js.example | 10 +++++++++- config/environments/staging.js.example | 10 +++++++++- config/environments/test.js.example | 10 +++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 2bc6d2d6..91cf2a39 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -111,7 +111,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 87214c75..7d7a00cb 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -105,7 +105,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 5e05c83f..3d1098fb 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -105,7 +105,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend diff --git a/config/environments/test.js.example b/config/environments/test.js.example index c01d52d1..ea788a22 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -104,7 +104,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend From 226653207ab3d11bf00e507c1368645afa2da945 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 12:17:55 +0200 Subject: [PATCH 004/167] target windshaft master branch --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a3ca339d..649759cd 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.0", "underscore": "~1.6.0", - "windshaft": "3.3.2", + "windshaft": "cartodb/windshaft", "yargs": "~5.0.0" }, "devDependencies": { @@ -58,7 +58,8 @@ "scripts": { "lint": "jshint lib test", "preinstall": "make pre-install", - "test": "make test-all" + "test": "make test-all", + "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn" }, "engines": { "node": ">=6.9", From 72a9a3e0970d93ed98df7f5707876edac42fbba0 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 12:26:19 +0200 Subject: [PATCH 005/167] updated deps --- yarn.lock | 110 +++++++++++++++--------------------------------------- 1 file changed, 31 insertions(+), 79 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6cae1a4a..d6e719a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,11 +10,7 @@ mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" - -abbrev@1.0.x: +abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -226,7 +222,7 @@ camshaft@0.59.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -244,6 +240,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@cartodb/carto#0.15.1-cdb3: + version "0.15.1-cdb3" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "1.8.3" + "carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" @@ -252,14 +256,6 @@ carto@0.16.3: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "1.8.3" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -520,11 +516,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" - -dot@~1.0.2: +dot@^1.0.3, dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -754,7 +746,7 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -766,10 +758,6 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -780,7 +768,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1: +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -811,17 +799,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1323,13 +1300,13 @@ mime@~1.3.4: version "1.3.6" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8: +minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1337,10 +1314,6 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -minimist@~0.0.1: - version "0.0.10" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" - minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1476,14 +1449,10 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1661,8 +1630,8 @@ postcss@5.0.19: supports-color "^3.1.2" postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.17" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b" + version "5.2.18" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.18.tgz#badfa1497d46244f6390f58b319830d9107853c5" dependencies: chalk "^1.1.3" js-base64 "^2.1.9" @@ -1771,7 +1740,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1: +readable-stream@1.1, readable-stream@~1.1.9: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1801,15 +1770,6 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@~1.1.9: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1919,9 +1879,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" @@ -1935,10 +1895,6 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - send@0.16.1: version "0.16.1" resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" @@ -2036,14 +1992,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4: +sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" -sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" - split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2141,7 +2093,7 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" -supports-color@3.1.2: +supports-color@3.1.2, supports-color@^3.1.0, supports-color@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" dependencies: @@ -2151,7 +2103,7 @@ supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" -supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: +supports-color@^3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.2.3.tgz#65ac0504b3954171d8a64946b2ae3cbb8a5f54f6" dependencies: @@ -2189,7 +2141,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2197,7 +2149,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2358,9 +2310,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.3.tgz#0582e6a0d9cf91c533134787ace64a3337200e33" +windshaft@cartodb/windshaft: + version "4.0.0" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/5b21e24e66c8e24cb88640b606aaf5fefcd3f365" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 54b7ee85c2e2d0e97ee85ddfe9ce96dae8849cfa Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 14:38:43 +0200 Subject: [PATCH 006/167] Geojson tests adapted to MVT --- .../analysis/analysis-layers-geojson.js | 81 ---- test/acceptance/geojson-renderer.js | 343 -------------- test/acceptance/mvt.js | 433 +++++++++++++++++- test/support/test-client.js | 2 +- 4 files changed, 431 insertions(+), 428 deletions(-) delete mode 100644 test/acceptance/analysis/analysis-layers-geojson.js delete mode 100644 test/acceptance/geojson-renderer.js diff --git a/test/acceptance/analysis/analysis-layers-geojson.js b/test/acceptance/analysis/analysis-layers-geojson.js deleted file mode 100644 index 59a0900b..00000000 --- a/test/acceptance/analysis/analysis-layers-geojson.js +++ /dev/null @@ -1,81 +0,0 @@ -require('../../support/test_helper'); - -var assert = require('../../support/assert'); -var TestClient = require('../../support/test-client'); - -describe('analysis-layers-dataviews-geojson', function() { - - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - - var CARTOCSS = [ - "#points {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 1.0;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" - ].join('\n'); - - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "2570e105-7b37-40d2-bdf4-1af889598745" - }, - "cartocss": CARTOCSS, - "cartocss_version": "2.3.0" - } - } - ], - { - pop_max_histogram: { - source: { - id: '2570e105-7b37-40d2-bdf4-1af889598745' - }, - type: 'histogram', - options: { - column: 'pop_max' - } - } - }, - [ - { - "id": "2570e105-7b37-40d2-bdf4-1af889598745", - "type": "source", - "params": { - "query": "select * from populated_places_simple_reduced" - } - } - ] - ); - - it('should get pop_max column from dataview', function(done) { - var testClient = new TestClient(mapConfig, 1234); - - testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(Array.isArray(geojson.features)); - assert.ok(geojson.features.length > 0); - var feature = geojson.features[0]; - assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); - - testClient.drain(done); - }); - }); - -}); diff --git a/test/acceptance/geojson-renderer.js b/test/acceptance/geojson-renderer.js deleted file mode 100644 index 089e09b5..00000000 --- a/test/acceptance/geojson-renderer.js +++ /dev/null @@ -1,343 +0,0 @@ -require('../support/test_helper'); - -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); - - -describe('use only needed columns', function() { - - function getFeatureByCartodbId(features, cartodbId) { - for (var i = 0, len = features.length; i < len; i++) { - if (features[i].properties.cartodb_id === cartodbId) { - return features[i]; - } - } - return {}; - } - - var options = { format: 'geojson', layer: 0 }; - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - it('with aggregation widget, interactivity and cartocss columns', function(done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', - cartocss_version: '2.0.1', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_min" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373, - pop_min: 57586 - }); - done(); - }); - }); - - it('should not duplicate columns', function(done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: ['#layer0 {', - 'marker-fill: red;', - 'marker-width: 10;', - '[name="Madrid"] { marker-fill: green; } ', - '[pop_max>100000] { marker-fill: black; } ', - '}'].join('\n'), - cartocss_version: '2.3.0', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_max" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373 - }); - done(); - }); - }); - - it('with formula widget, no interactivity and no cartocss columns', function(done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id', - widgets: { - pop_max_f: { - type: 'formula', - options: { - column: 'pop_max', - operation: 'count' - } - } - } - } - }] - }; - - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max: 71373 - }); - done(); - }); - }); - it('with cartocss with multiple expressions', function(done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + - '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id' - } - }] - }; - - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max:71373, - name:"Mardin", - adm0name:"Turkey" - }); - done(); - }); - }); - - it('should work with mapnik substitution tokens', function(done) { - var cartocss = [ - "#layer {", - " line-width: 2;", - " line-color: #3B3B58;", - " line-opacity: 1;", - " polygon-opacity: 0.7;", - " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", - "}" - ].join('\n'); - - var sql = [ - 'WITH hgrid AS (', - ' SELECT CDB_HexagonGrid(', - ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', - ' greatest(!pixel_width!,!pixel_height!) * 100', - ' ) as cell', - ')', - 'SELECT', - ' hgrid.cell as the_geom_webmercator,', - ' count(1) as points_count,', - ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', - ' 1 as cartodb_id', - 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', - 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', - 'GROUP BY hgrid.cell' - ].join('\n'); - - var mapConfig = { - "version": "1.4.0", - "layers": [ - { - "type": 'mapnik', - "options": { - "cartocss_version": '2.3.0', - "sql": sql, - "cartocss": cartocss - } - } - ] - }; - - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(geojson); - assert.equal(geojson.features.length, 5); - - done(); - }); - }); - - it('should skip empty and null columns for geojson tiles', function(done) { - - var mapConfig = { - "analyses": [ - { - "id": "a0", - "params": { - "query": "SELECT * FROM test_table" - }, - "type": "source" - } - ], - "dataviews": { - "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "74f590f8-625c-4e95-922f-34ad3e9919c0": { - "options": { - "aggregation": "sum", - "aggregationColumn": "cartodb_id", - "column": "name" - }, - "source": { - "id": "a0" - }, - "type": "aggregation" - }, - "98a75757-3006-400a-b028-fb613a6c0b69": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "sum" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { - "options": { - "aggregationColumn": null, - "bins": "12", - "column": "cartodb_id" - }, - "source": { - "id": "a0" - }, - "type": "histogram" - }, - "ebc0653f-3581-469c-8b31-c969e440a865": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - } - }, - "layers": [ - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" - }, - "type": "http" - }, - { - "options": { - "attributes": { - "columns": [ - "name", - "address" - ], - "id": "cartodb_id" - }, - "cartocss": "#layer { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0", - "interactivity": "cartodb_id", - "layer_name": "wadus", - "source": { - "id": "a0" - } - }, - "type": "cartodb" - }, - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" - }, - "type": "http" - } - ] - }; - - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(geojson); - assert.equal(geojson.features.length, 5); - - assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']); - - done(); - }); - }); - -}); diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b5a59cd8..b6a04d79 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -1,9 +1,9 @@ require('../support/test_helper'); -const assert = require('../support/assert'); -const TestClient = require('../support/test-client'); +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); -function createMapConfig (sql = TestClient.SQL.ONE_POINT) { +function createMapConfig(sql = TestClient.SQL.ONE_POINT) { return { version: '1.6.0', layers: [{ @@ -19,6 +19,86 @@ function createMapConfig (sql = TestClient.SQL.ONE_POINT) { } describe('mvt', function () { + + describe('analysis-layers-dataviews-mvt', function () { + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var CARTOCSS = [ + "#points {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 1.0;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", + "}" + ].join('\n'); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": CARTOCSS, + "cartocss_version": "2.3.0" + } + } + ], + { + pop_max_histogram: { + source: { + id: '2570e105-7b37-40d2-bdf4-1af889598745' + }, + type: 'histogram', + options: { + column: 'pop_max' + } + } + }, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ); + + it('should get pop_max column from dataview', function (done) { + var testClient = new TestClient(mapConfig, 1234); + + testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(Array.isArray(geojson.features)); + assert.ok(geojson.features.length > 0); + var feature = geojson.features[0]; + assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); + + testClient.drain(done); + }); + }); + + }); + + const testCases = [ { desc: 'should get empty mvt with code 204 (no content)', @@ -60,4 +140,351 @@ describe('mvt', function () { }); }); }); + + + + describe('use only needed columns', function () { + + function getFeatureByCartodbId(features, cartodbId) { + for (var i = 0, len = features.length; i < len; i++) { + if (features[i].properties.cartodb_id === cartodbId) { + return features[i]; + } + } + return {}; + } + + var options = { format: 'mvt', layer: 0 }; + + afterEach(function (done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + it('with aggregation widget, interactivity and cartocss columns', function (done) { + var widgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', + cartocss_version: '2.0.1', + widgets: { + adm0name: { + type: 'aggregation', + options: { + column: 'adm0name', + aggregation: 'sum', + aggregationColumn: 'pop_max' + } + } + }, + interactivity: "cartodb_id,pop_min" + } + }] + }; + + this.testClient = new TestClient(widgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + name: 'Mardin', + adm0name: 'Turkey', + pop_max: 71373, + pop_min: 57586 + }); + done(); + }); + }); + + it('should not duplicate columns', function (done) { + var widgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: ['#layer0 {', + 'marker-fill: red;', + 'marker-width: 10;', + '[name="Madrid"] { marker-fill: green; } ', + '[pop_max>100000] { marker-fill: black; } ', + '}'].join('\n'), + cartocss_version: '2.3.0', + widgets: { + adm0name: { + type: 'aggregation', + options: { + column: 'adm0name', + aggregation: 'sum', + aggregationColumn: 'pop_max' + } + } + }, + interactivity: "cartodb_id,pop_max" + } + }] + }; + + this.testClient = new TestClient(widgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + name: 'Mardin', + adm0name: 'Turkey', + pop_max: 71373 + }); + done(); + }); + }); + + it('with formula widget, no interactivity and no cartocss columns', function (done) { + var formulaWidgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id', + widgets: { + pop_max_f: { + type: 'formula', + options: { + column: 'pop_max', + operation: 'count' + } + } + } + } + }] + }; + + this.testClient = new TestClient(formulaWidgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + pop_max: 71373 + }); + done(); + }); + }); + it('with cartocss with multiple expressions', function (done) { + var formulaWidgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + + '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + + '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + + '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id' + } + }] + }; + + this.testClient = new TestClient(formulaWidgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + pop_max: 71373, + name: "Mardin", + adm0name: "Turkey" + }); + done(); + }); + }); + + it('should work with mapnik substitution tokens', function (done) { + var cartocss = [ + "#layer {", + " line-width: 2;", + " line-color: #3B3B58;", + " line-opacity: 1;", + " polygon-opacity: 0.7;", + " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", + "}" + ].join('\n'); + + var sql = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + var mapConfig = { + "version": "1.4.0", + "layers": [ + { + "type": 'mapnik', + "options": { + "cartocss_version": '2.3.0', + "sql": sql, + "cartocss": cartocss + } + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(geojsonTile); + assert.equal(geojsonTile.features.length, 5); + + done(); + }); + }); + + it('should skip empty and null columns for geojson tiles', function (done) { + + var mapConfig = { + "analyses": [ + { + "id": "a0", + "params": { + "query": "SELECT * FROM test_table" + }, + "type": "source" + } + ], + "dataviews": { + "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "avg" + }, + "source": { + "id": "a0" + }, + "type": "formula" + }, + "74f590f8-625c-4e95-922f-34ad3e9919c0": { + "options": { + "aggregation": "sum", + "aggregationColumn": "cartodb_id", + "column": "name" + }, + "source": { + "id": "a0" + }, + "type": "aggregation" + }, + "98a75757-3006-400a-b028-fb613a6c0b69": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "sum" + }, + "source": { + "id": "a0" + }, + "type": "formula" + }, + "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { + "options": { + "aggregationColumn": null, + "bins": "12", + "column": "cartodb_id" + }, + "source": { + "id": "a0" + }, + "type": "histogram" + }, + "ebc0653f-3581-469c-8b31-c969e440a865": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "avg" + }, + "source": { + "id": "a0" + }, + "type": "formula" + } + }, + "layers": [ + { + "options": { + "subdomains": "abcd", + "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" + }, + "type": "http" + }, + { + "options": { + "attributes": { + "columns": [ + "name", + "address" + ], + "id": "cartodb_id" + }, + "cartocss": "#layer { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0", + "interactivity": "cartodb_id", + "layer_name": "wadus", + "source": { + "id": "a0" + } + }, + "type": "cartodb" + }, + { + "options": { + "subdomains": "abcd", + "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" + }, + "type": "http" + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(geojsonTile); + assert.equal(geojsonTile.features.length, 5); + + assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']); + + done(); + }); + }); + + }); }); + diff --git a/test/support/test-client.js b/test/support/test-client.js index 21918665..b3c6566b 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -20,7 +20,7 @@ const MAPNIK_SUPPORTED_FORMATS = { 'png': true, 'png32': true, 'grid.json': true, - 'geojson': true, + 'mvt': true, 'mvt': true } From 34d9e5a4eb3aaf9e72c282589b253e259ed2476a Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 16:08:05 +0200 Subject: [PATCH 007/167] Fix MVT test --- test/acceptance/mvt.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b6a04d79..dbfadd26 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -87,9 +87,9 @@ describe('mvt', function () { var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); assert.ok(!err, err); - assert.ok(Array.isArray(geojson.features)); - assert.ok(geojson.features.length > 0); - var feature = geojson.features[0]; + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + var feature = geojsonTile.features[0]; assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); testClient.drain(done); From f387f2ee6f2b5169ffb1bfba3016dc065f5181f9 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 16:08:31 +0200 Subject: [PATCH 008/167] Testing dockerified --- Dockerfile | 33 +++++++++++++++++++++++++++++++++ package.json | 7 ++++++- run_tests.sh | 2 +- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3b4fd258 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:xenial + + +RUN apt-get -y update +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN apt-get -y update +RUN apt-get -y install curl wget protobuf-c-compiler + +#Use UTF8 to avoid encoding problems with pgsql +RUN apt-get -y install locales +ENV LANG C.UTF-8 +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +#Add 6.X node ppa +RUN curl -sL https://deb.nodesource.com/setup_6.x | bash +RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev + +# Install PostGIS 2.4 from sources +RUN wget http://download.osgeo.org/postgis/source/postgis-2.4.0.tar.gz && tar xvfz postgis-2.4.0.tar.gz && cd postgis-2.4.0 && ./configure && make && make install && cd .. && rm -rf postgis-2.4.0 + +# Configure PostgreSQL +RUN echo "local all all trust" > /etc/postgresql/9.5/main/pg_hba.conf +RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf +RUN echo "host all all ::1/128 trust" >> /etc/postgresql/9.5/main/pg_hba.conf +RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf + +RUN apt-get -y remove wget protobuf-c-compiler +RUN apt-get -y autoremove + +WORKDIR /srv +CMD export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && npm install && export PGUSER=postgres && /etc/init.d/postgresql start && createdb template_postgis && createuser publicuser && psql -c "CREATE EXTENSION postgis" template_postgis && npm test diff --git a/package.json b/package.json index 9f0581e4..8ff78dce 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,12 @@ "lint": "jshint lib test", "preinstall": "make pre-install", "test": "make test-all", - "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn" + "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", + "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", + "docker-pull": "docker pull cartoimages/windshaft-carto-testing", + "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing", + "docker-build": "docker build -t cartoimages/windshaft-carto-testing .", + "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, "engines": { "node": ">=6.9", diff --git a/run_tests.sh b/run_tests.sh index 92e8621b..d1c63614 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS} else echo "Running tests" - mocha -u tdd -t 5000 ${TESTS} + ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS} fi ret=$? From bf637ccd5bed211fa5ef31e30a7e52a761d8eed9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 5 Oct 2017 17:06:31 +0200 Subject: [PATCH 009/167] Implement stats middleware removing some duplicated code --- lib/cartodb/controllers/base.js | 14 --------- lib/cartodb/middleware/error-middleware.js | 10 ------- lib/cartodb/middleware/stats.js | 33 ++++++++++++++++++++++ lib/cartodb/server.js | 12 ++++---- 4 files changed, 40 insertions(+), 29 deletions(-) create mode 100644 lib/cartodb/middleware/stats.js diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index 7462f6ba..46158eee 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -1,18 +1,13 @@ -var debug = require('debug')('windshaft:cartodb'); - function BaseController() { } module.exports = BaseController; -// jshint maxcomplexity:9 BaseController.prototype.send = function(req, res, body, status, headers) { if (req.params.dbhost) { res.set('X-Served-By-DB-Host', req.params.dbhost); } - res.set('X-Tiler-Profiler', req.profiler.toJSONString()); - if (headers) { res.set(headers); } @@ -28,13 +23,4 @@ BaseController.prototype.send = function(req, res, body, status, headers) { } else { res.send(body); } - - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - debug("error sending profiling stats: " + err); - } }; -// jshint maxcomplexity:6 diff --git a/lib/cartodb/middleware/error-middleware.js b/lib/cartodb/middleware/error-middleware.js index 71f6c411..acf8d99d 100644 --- a/lib/cartodb/middleware/error-middleware.js +++ b/lib/cartodb/middleware/error-middleware.js @@ -35,8 +35,6 @@ module.exports = function errorMiddleware (/* options */) { res.set('X-Served-By-DB-Host', res.locals.dbhost); } - res.set('X-Tiler-Profiler', req.profiler.toJSONString()); - res.status(statusCode); if (req.query && req.query.callback) { @@ -44,14 +42,6 @@ module.exports = function errorMiddleware (/* options */) { } else { res.json(errorResponseBody); } - - try { - // May throw due to dns, see - // See http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - debug("error sending profiling stats: " + err); - } }; }; diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/middleware/stats.js new file mode 100644 index 00000000..98dde45b --- /dev/null +++ b/lib/cartodb/middleware/stats.js @@ -0,0 +1,33 @@ +const Profiler = require('../stats/profiler_proxy'); +const debug = require('debug')('windshaft:cartodb:stats'); +const methods = ['send', 'json', 'jsonp']; + +module.exports = function statsMiddleware(options) { + const { enabled = true, statsClient } = options; + + return function stats(req, res, next) { + req.profiler = new Profiler({ + statsd_client: statsClient, + profile: enabled + }); + + methods.forEach((method) => { + const originalFn = res[method]; + + res[method] = function (body) { + res.set('X-Tiler-Profiler', req.profiler.toJSONString()); + + originalFn.call(this, body); + + try { + // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + debug("error sending profiling stats: " + err); + } + }; + }); + + next(); + }; +}; \ No newline at end of file diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 15c7190e..377fff69 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -12,7 +12,8 @@ var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); var FastlyCacheBackend = require('./cache/backend/fastly'); var StatsClient = require('./stats/client'); -var Profiler = require('./stats/profiler_proxy'); +const stats = require('./middleware/stats'); + var RendererStatsReporter = require('./stats/reporter/renderer'); var windshaft = require('windshaft'); @@ -362,10 +363,6 @@ function bootstrap(opts) { app.use(function bootstrap$prepareRequestResponse(req, res, next) { req.context = req.context || {}; - req.profiler = new Profiler({ - statsd_client: global.statsClient, - profile: opts.useProfiler - }); if (global.environment && global.environment.api_hostname) { res.set('X-Served-By-Host', global.environment.api_hostname); @@ -374,6 +371,11 @@ function bootstrap(opts) { next(); }); + app.use(stats({ + enabled: opts.useProfiler, + statsClient: global.statsClient + })); + app.use(lzmaMiddleware); // temporary measure until we upgrade to newer version expressjs so we can check err.status From cab2d6d5d4e51e120a3912db6cce616c502dcca8 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 11:03:08 +0200 Subject: [PATCH 010/167] package.json docker-bash script added --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 8ff78dce..5af0b9d3 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", "docker-pull": "docker pull cartoimages/windshaft-carto-testing", "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing", + "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-carto-testing bash", "docker-build": "docker build -t cartoimages/windshaft-carto-testing .", "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, From 4e6253b71794e5acf7fb164c4599d6ca08886854 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:02:16 +0200 Subject: [PATCH 011/167] Return HTTP 204 for empty tiles --- lib/cartodb/controllers/layergroup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 9644aa06..f6089a00 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -313,7 +313,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.sendResponse(req, res, tile, 200, headers); + this.sendResponse(req, res, tile, tile.length===0? 204:200, headers); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } From d29da0bcc35ab995b880c23081442068779c9f6c Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:07:24 +0200 Subject: [PATCH 012/167] Test both MVT paths: mapnik and PostGIS --- test/acceptance/mvt.js | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index dbfadd26..444c72fe 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -2,6 +2,7 @@ require('../support/test_helper'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); function createMapConfig(sql = TestClient.SQL.ONE_POINT) { return { @@ -18,7 +19,14 @@ function createMapConfig(sql = TestClient.SQL.ONE_POINT) { }; } -describe('mvt', function () { +describe('mvt (mapnik)', mvt(false)); +describe('mvt (postgis)', mvt(true)); + +function mvt(usePostGIS) { +return function () { + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); describe('analysis-layers-dataviews-mvt', function () { @@ -81,7 +89,7 @@ describe('mvt', function () { ); it('should get pop_max column from dataview', function (done) { - var testClient = new TestClient(mapConfig, 1234); + var testClient = new TestClient(mapConfig); testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); @@ -128,23 +136,25 @@ describe('mvt', function () { testCases.forEach(function (test) { it(test.desc, done => { - const testClient = new TestClient(test.mapConfig, 1234); + var testClient = new TestClient(test.mapConfig); const { z, x, y } = test.coords; const { format, response } = test; testClient.getTile(z, x, y, { format, response }, (err, res) => { assert.ifError(err); - - assert.equal(res.statusCode, test.response.status); testClient.drain(done); }); }); }); + if (usePostGIS){ + describe('use only needed columns', onlyNeededColumns); + }else{ + describe.skip('use only needed columns', onlyNeededColumns); + } - describe('use only needed columns', function () { - + function onlyNeededColumns() { function getFeatureByCartodbId(features, cartodbId) { for (var i = 0, len = features.length; i < len; i++) { if (features[i].properties.cartodb_id === cartodbId) { @@ -268,7 +278,6 @@ describe('mvt', function () { } }] }; - this.testClient = new TestClient(formulaWidgetMapConfig); this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); @@ -485,6 +494,6 @@ describe('mvt', function () { }); }); - }); -}); - + } +} +} \ No newline at end of file From 78671aa499eabbfccc5e3c2a57238beb20ea8a32 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:07:47 +0200 Subject: [PATCH 013/167] remove redundant format support --- test/support/test-client.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index b3c6566b..1f5272f6 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -20,7 +20,6 @@ const MAPNIK_SUPPORTED_FORMATS = { 'png': true, 'png32': true, 'grid.json': true, - 'mvt': true, 'mvt': true } From 43885f130bd56adcd1d6fb16578eb48a53b7f472 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:19:00 +0200 Subject: [PATCH 014/167] Fix HTTP status code distinction between 200 and 204 --- lib/cartodb/controllers/layergroup.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index f6089a00..ed0f3eb5 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -278,6 +278,10 @@ LayergroupController.prototype.tileOrLayer = function (req, res) { ); }; +function getStatusCode(tile, format){ + return tile.length===0 && format==='mvt'? 204:200; +} + // This function is meant for being called as the very last // step by all endpoints serving tiles or grids LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) { @@ -313,7 +317,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.sendResponse(req, res, tile, tile.length===0? 204:200, headers); + this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } From a706fd81ba33d512061c54873fb247766f679242 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:21:34 +0200 Subject: [PATCH 015/167] Restore MVT path configuration after each suite pass --- test/acceptance/mvt.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 444c72fe..af67f8bf 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -24,9 +24,13 @@ describe('mvt (postgis)', mvt(true)); function mvt(usePostGIS) { return function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; before(function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; }); + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); describe('analysis-layers-dataviews-mvt', function () { From cac16f8b66069af278a3fd6989e4ab10ec20d84b Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:30:35 +0200 Subject: [PATCH 016/167] Travis dockerified --- .travis.yml | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f3993a2..d55cc94a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,14 @@ +sudo: required dist: trusty -addons: - postgresql: "9.5" - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - postgresql-9.5-postgis-2.3 - - postgresql-plpython-9.5 - - pkg-config - - libcairo2-dev - - libjpeg8-dev - - libgif-dev - - libpango1.0-dev - - g++-4.9 + +services: + - docker before_install: - - createdb template_postgis - - createuser publicuser - - psql -c "CREATE EXTENSION postgis" template_postgis + - docker pull dmanzanares/windshaft-carto-testing -env: - - NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9 +script: + - docker run -v `pwd`:/srv dmanzanares/windshaft-carto-testing + +language: generic -language: node_js -node_js: - - "6" From 7f66189164964101398c2caddd948d19cd18db4f Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:44:28 +0200 Subject: [PATCH 017/167] Fix travis.yaml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d55cc94a..b946378f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,10 @@ services: - docker before_install: - - docker pull dmanzanares/windshaft-carto-testing + - docker pull cartoimages/windshaft-carto-testing script: - - docker run -v `pwd`:/srv dmanzanares/windshaft-carto-testing + - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing language: generic From f456237aa7197aef5f5cb2323faaa7da1c7e6817 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:53:47 +0000 Subject: [PATCH 018/167] Drain client on after hook --- test/acceptance/buffer-size-format.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index c32f62d1..236478b8 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -124,21 +124,28 @@ describe('buffer size per format', function () { } ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.mapConfig, 1234); + this.testClient = new TestClient(test.mapConfig, 1234); var coords = test.coords; var options = { format: test.format, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: // tile.save(test.fixturePath); test.assert(tile, function (err) { assert.ifError(err); - testClient.drain(done); + return done(); }); }); }); From c004e105efd6e63a51c8322052d45963d52147e4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 16:17:26 +0000 Subject: [PATCH 019/167] Drain client on after hooks --- test/acceptance/buffer-size-format.js | 37 +++++++++++++++------------ 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 236478b8..e41cf1d3 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -143,10 +143,7 @@ describe('buffer size per format', function () { assert.ifError(err); // To generate images use: // tile.save(test.fixturePath); - test.assert(tile, function (err) { - assert.ifError(err); - return done(); - }); + test.assert(tile, done); }); }); }); @@ -267,23 +264,27 @@ describe('buffer size per format for named maps', function () { } ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.template, 1234); + this.testClient = new TestClient(test.template, 1234); var coords = test.coords; var options = { format: test.format, placeholders: test.placeholders, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png'); - test.assert(tile, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + test.assert(tile, done); }); }); }); @@ -423,25 +424,29 @@ describe('buffer size per format for named maps w/o placeholders', function () { ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.template, 1234); + this.testClient = new TestClient(test.template, 1234); var coords = test.coords; var options = { format: test.format, placeholders: test.placeholders, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: //tile.save(test.fixturePath); // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); - test.assert(tile, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + test.assert(tile, done); }); }); }); From 28cb05e45b5f3b6123149d2dea7342edf3df3627 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 18:24:11 +0200 Subject: [PATCH 020/167] Use windshaft docker image --- Dockerfile | 13 ++++++++----- docker-test.sh | 10 ++++++++++ package.json | 7 +++---- 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 docker-test.sh diff --git a/Dockerfile b/Dockerfile index 3b4fd258..d16731c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,23 @@ FROM ubuntu:xenial - +# Add external repos RUN apt-get -y update RUN apt-get install -y software-properties-common RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test RUN apt-get -y update -RUN apt-get -y install curl wget protobuf-c-compiler -#Use UTF8 to avoid encoding problems with pgsql +# Use UTF8 to avoid encoding problems with pgsql RUN apt-get -y install locales ENV LANG C.UTF-8 RUN locale-gen en_US.UTF-8 RUN update-locale LANG=en_US.UTF-8 -#Add 6.X node ppa +# Add 6.X node ppa +RUN apt-get -y install curl RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev + +# Install dependencies +RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev wget protobuf-c-compiler # Install PostGIS 2.4 from sources RUN wget http://download.osgeo.org/postgis/source/postgis-2.4.0.tar.gz && tar xvfz postgis-2.4.0.tar.gz && cd postgis-2.4.0 && ./configure && make && make install && cd .. && rm -rf postgis-2.4.0 @@ -26,6 +28,7 @@ RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/ RUN echo "host all all ::1/128 trust" >> /etc/postgresql/9.5/main/pg_hba.conf RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf +# Clean RUN apt-get -y remove wget protobuf-c-compiler RUN apt-get -y autoremove diff --git a/docker-test.sh b/docker-test.sh new file mode 100644 index 00000000..e340369e --- /dev/null +++ b/docker-test.sh @@ -0,0 +1,10 @@ +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres + +npm install + + /etc/init.d/postgresql start + +createdb template_postgis && createuser publicuser +psql -c "CREATE EXTENSION postgis" template_postgis + +npm test \ No newline at end of file diff --git a/package.json b/package.json index 5af0b9d3..b65a64d1 100644 --- a/package.json +++ b/package.json @@ -61,10 +61,9 @@ "test": "make test-all", "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", - "docker-pull": "docker pull cartoimages/windshaft-carto-testing", - "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing", - "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-carto-testing bash", - "docker-build": "docker build -t cartoimages/windshaft-carto-testing .", + "docker-pull": "docker pull cartoimages/windshaft-testing", + "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh", + "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-testing bash", "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, "engines": { From 0e20958220aed2c3e8261c302b76608cc976ef04 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 18:24:29 +0200 Subject: [PATCH 021/167] Remove dockerfile --- Dockerfile | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d16731c5..00000000 --- a/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM ubuntu:xenial - -# Add external repos -RUN apt-get -y update -RUN apt-get install -y software-properties-common -RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test -RUN apt-get -y update - -# Use UTF8 to avoid encoding problems with pgsql -RUN apt-get -y install locales -ENV LANG C.UTF-8 -RUN locale-gen en_US.UTF-8 -RUN update-locale LANG=en_US.UTF-8 - -# Add 6.X node ppa -RUN apt-get -y install curl -RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - -# Install dependencies -RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev wget protobuf-c-compiler - -# Install PostGIS 2.4 from sources -RUN wget http://download.osgeo.org/postgis/source/postgis-2.4.0.tar.gz && tar xvfz postgis-2.4.0.tar.gz && cd postgis-2.4.0 && ./configure && make && make install && cd .. && rm -rf postgis-2.4.0 - -# Configure PostgreSQL -RUN echo "local all all trust" > /etc/postgresql/9.5/main/pg_hba.conf -RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf -RUN echo "host all all ::1/128 trust" >> /etc/postgresql/9.5/main/pg_hba.conf -RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf - -# Clean -RUN apt-get -y remove wget protobuf-c-compiler -RUN apt-get -y autoremove - -WORKDIR /srv -CMD export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && npm install && export PGUSER=postgres && /etc/init.d/postgresql start && createdb template_postgis && createuser publicuser && psql -c "CREATE EXTENSION postgis" template_postgis && npm test From 1ba1c488faec90479b87ac05ec4defbbcf640265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sat, 7 Oct 2017 19:02:26 +0200 Subject: [PATCH 022/167] Do not decorate response methods to set header and send stats --- lib/cartodb/middleware/stats.js | 24 ++-- package.json | 1 + yarn.lock | 222 ++++---------------------------- 3 files changed, 36 insertions(+), 211 deletions(-) diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/middleware/stats.js index 98dde45b..0bd8c5e0 100644 --- a/lib/cartodb/middleware/stats.js +++ b/lib/cartodb/middleware/stats.js @@ -1,6 +1,6 @@ const Profiler = require('../stats/profiler_proxy'); const debug = require('debug')('windshaft:cartodb:stats'); -const methods = ['send', 'json', 'jsonp']; +const onHeaders = require('on-headers'); module.exports = function statsMiddleware(options) { const { enabled = true, statsClient } = options; @@ -11,21 +11,15 @@ module.exports = function statsMiddleware(options) { profile: enabled }); - methods.forEach((method) => { - const originalFn = res[method]; + onHeaders(res, () => res.set('X-Tiler-Profiler', req.profiler.toJSONString())); - res[method] = function (body) { - res.set('X-Tiler-Profiler', req.profiler.toJSONString()); - - originalFn.call(this, body); - - try { - // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166 - req.profiler.sendStats(); - } catch (err) { - debug("error sending profiling stats: " + err); - } - }; + res.on('finish', () => { + try { + // May throw due to dns, see: http://github.com/CartoDB/Windshaft/issues/166 + req.profiler.sendStats(); + } catch (err) { + debug("error sending profiling stats: " + err); + } }); next(); diff --git a/package.json b/package.json index 02c94e52..1c3c0bbb 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "lru-cache": "2.6.5", "lzma": "~2.3.2", "node-statsd": "~0.0.7", + "on-headers": "^1.0.1", "queue-async": "~1.0.7", "redis-mpool": "0.4.1", "request": "~2.79.0", diff --git a/yarn.lock b/yarn.lock index 624dde8c..f3fedcce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10,11 +10,7 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1: - version "1.1.0" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.0.tgz#d0554c2256636e2f56e7c2e5ad183f859428d81f" - -abbrev@1.0.x: +abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -32,15 +28,6 @@ ajv@^4.9.1: co "^4.6.0" json-stable-stringify "^1.0.1" -ajv@^5.1.0: - version "5.2.2" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.2.2.tgz#47c68d69e86f5d953103b0074a9430dc63da5e39" - dependencies: - co "^4.6.0" - fast-deep-equal "^1.0.0" - json-schema-traverse "^0.3.0" - json-stable-stringify "^1.0.1" - align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -114,11 +101,7 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws-sign2@~0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" - -aws4@^1.2.1, aws4@^1.6.0: +aws4@^1.2.1: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -163,18 +146,6 @@ boom@2.x.x: dependencies: hoek "2.x.x" -boom@4.x.x: - version "4.3.1" - resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" - dependencies: - hoek "4.x.x" - -boom@5.x.x: - version "5.2.0" - resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" - dependencies: - hoek "4.x.x" - brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -368,16 +339,12 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0: +commander@2.9.0, commander@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: graceful-readlink ">= 1.0.0" -commander@^2.9.0: - version "2.11.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563" - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -418,12 +385,6 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" -cryptiles@3.x.x: - version "3.1.2" - resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" - dependencies: - boom "5.x.x" - d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -438,7 +399,7 @@ date-now@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b" -debug@2.2.0, debug@~2.2.0: +debug@2.2.0, debug@^2.2.0, debug@~2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da" dependencies: @@ -456,12 +417,6 @@ debug@^1.0.4: dependencies: ms "2.0.0" -debug@^2.2.0: - version "2.6.9" - resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" - dependencies: - ms "2.0.0" - decamelize@^1.0.0, decamelize@^1.1.1: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -532,11 +487,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" - -dot@~1.0.2: +dot@^1.0.3, dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -647,7 +598,7 @@ express@~4.13.3: utils-merge "1.0.0" vary "~1.0.1" -extend@~3.0.0, extend@~3.0.1: +extend@~3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -655,10 +606,6 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" -fast-deep-equal@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" - fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -697,14 +644,6 @@ form-data@~2.1.1: combined-stream "^1.0.5" mime-types "^2.1.12" -form-data@~2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf" - dependencies: - asynckit "^0.4.0" - combined-stream "^1.0.5" - mime-types "^2.1.12" - forwarded@~0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.2.tgz#98c23dab1175657b8c0573e8ceccd91b0ff18c84" @@ -790,7 +729,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1: +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -821,17 +760,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -872,10 +800,6 @@ har-schema@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-1.0.5.tgz#d263135f43307c02c602afc8fe95970c0151369e" -har-schema@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92" - har-validator@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d" @@ -892,13 +816,6 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" -har-validator@~5.0.3: - version "5.0.3" - resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" - dependencies: - ajv "^5.1.0" - har-schema "^2.0.0" - has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -922,15 +839,6 @@ hawk@3.1.3, hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" -hawk@~6.0.2: - version "6.0.2" - resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" - dependencies: - boom "4.x.x" - cryptiles "3.x.x" - hoek "4.x.x" - sntp "2.x.x" - hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -942,10 +850,6 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" -hoek@4.x.x: - version "4.2.0" - resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" - hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -975,14 +879,6 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" -http-signature@~1.2.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" - dependencies: - assert-plus "^1.0.0" - jsprim "^1.2.2" - sshpk "^1.7.0" - husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -1122,10 +1018,6 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" -json-schema-traverse@^0.3.0: - version "0.3.1" - resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" - json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1339,13 +1231,13 @@ mime-db@~1.30.0: version "1.30.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01" -mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.17, mime-types@~2.1.6, mime-types@~2.1.7: +mime-types@^2.1.12, mime-types@~2.1.15, mime-types@~2.1.6, mime-types@~2.1.7: version "2.1.17" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a" dependencies: mime-db "~1.30.0" -mime@1.3.4: +mime@1.3.4, mime@~1.3.4: version "1.3.4" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53" @@ -1353,17 +1245,13 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" -mime@~1.3.4: - version "1.3.6" - resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" - -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8: +minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1371,10 +1259,6 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -minimist@~0.0.1: - version "0.0.10" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" - minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1510,7 +1394,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1, oauth-sign@~0.8.2: +oauth-sign@~0.8.1: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1532,6 +1416,10 @@ on-finished@~2.3.0: dependencies: ee-first "1.1.1" +on-headers@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/on-headers/-/on-headers-1.0.1.tgz#928f5d0f470d49342651ea6794b0857c100693f7" + once@1.x, once@^1.3.0, once@^1.3.3: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" @@ -1617,10 +1505,6 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" -performance-now@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1776,10 +1660,6 @@ qs@~6.4.0: version "6.4.0" resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233" -qs@~6.5.1: - version "6.5.1" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8" - queue-async@~1.0.7: version "1.0.7" resolved "https://registry.yarnpkg.com/queue-async/-/queue-async-1.0.7.tgz#22ae0a1dac4a92f5bcd4634f993c682a2a810945" @@ -1820,7 +1700,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1: +readable-stream@1.1, readable-stream@~1.1.9: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1850,15 +1730,6 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@~1.1.9: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1903,34 +1774,7 @@ request@2.81.0: tunnel-agent "^0.6.0" uuid "^3.0.0" -request@2.x, request@^2.55.0, request@^2.69.0: - version "2.82.0" - resolved "https://registry.yarnpkg.com/request/-/request-2.82.0.tgz#2ba8a92cd7ac45660ea2b10a53ae67cd247516ea" - dependencies: - aws-sign2 "~0.7.0" - aws4 "^1.6.0" - caseless "~0.12.0" - combined-stream "~1.0.5" - extend "~3.0.1" - forever-agent "~0.6.1" - form-data "~2.3.1" - har-validator "~5.0.3" - hawk "~6.0.2" - http-signature "~1.2.0" - is-typedarray "~1.0.0" - isstream "~0.1.2" - json-stringify-safe "~5.0.1" - mime-types "~2.1.17" - oauth-sign "~0.8.2" - performance-now "^2.1.0" - qs "~6.5.1" - safe-buffer "^5.1.1" - stringstream "~0.0.5" - tough-cookie "~2.3.2" - tunnel-agent "^0.6.0" - uuid "^3.1.0" - -request@~2.79.0: +request@2.x, request@^2.55.0, request@^2.69.0, request@~2.79.0: version "2.79.0" resolved "https://registry.yarnpkg.com/request/-/request-2.79.0.tgz#4dfe5bf6be8b8cdc37fcf93e04b65577722710de" dependencies: @@ -1985,7 +1829,7 @@ rimraf@~2.4.0: dependencies: glob "^6.0.1" -safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.0.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853" @@ -1993,9 +1837,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" @@ -2009,10 +1853,6 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - send@0.13.1: version "0.13.1" resolved "https://registry.yarnpkg.com/send/-/send-0.13.1.tgz#a30d5f4c82c8a9bae9ad00a1d9b1bdbe6f199ed7" @@ -2077,12 +1917,6 @@ sntp@1.x.x: dependencies: hoek "2.x.x" -sntp@2.x.x: - version "2.0.2" - resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" - dependencies: - hoek "4.x.x" - source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -2117,14 +1951,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4: +sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" -sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" - split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2202,7 +2032,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4, stringstream@~0.0.5: +stringstream@~0.0.4: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2307,7 +2137,7 @@ torque.js@~2.11.0: dependencies: carto CartoDB/carto#0.15.1-cdb1 -tough-cookie@~2.3.0, tough-cookie@~2.3.2: +tough-cookie@~2.3.0: version "2.3.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561" dependencies: @@ -2400,7 +2230,7 @@ utils-merge@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.0.tgz#0294fb922bb9375153541c4f7096231f287c8af8" -uuid@^3.0.0, uuid@^3.1.0: +uuid@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" From 7289394f6ab7231f425622ec76f46e6f2acff6a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sat, 7 Oct 2017 19:16:15 +0200 Subject: [PATCH 023/167] Missing EOL --- lib/cartodb/middleware/stats.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/middleware/stats.js b/lib/cartodb/middleware/stats.js index 0bd8c5e0..489ee645 100644 --- a/lib/cartodb/middleware/stats.js +++ b/lib/cartodb/middleware/stats.js @@ -24,4 +24,4 @@ module.exports = function statsMiddleware(options) { next(); }; -}; \ No newline at end of file +}; From 28501f6b9d8c6a78312ce35af67669f12495f6f1 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 14:46:01 +0200 Subject: [PATCH 024/167] MVT PostGIS limit test fixed --- test/acceptance/user-render-timeout-limit.js | 84 +++++++++++--------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 45e41f89..eaf98805 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -2,6 +2,7 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; @@ -200,49 +201,56 @@ describe('user render timeout limit', function () { }); }); - describe('vector', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); - }); - - afterEach(function (done) { - this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); + describe('vector (PostGIS)', vector(true)); + describe('vector (mapnik)', vector(false)); + function vector(usePostGIS) { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + return function () { + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(50, done); }); - }); - it('layergroup creation works but vector tile request fails due to render timeout', function (done) { - const params = { - format: 'mvt', - response: { - status: 429, - headers: { - 'Content-Type': 'application/json; charset=utf-8' + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); } - } - }; - - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { - assert.ifError(err); - - assert.deepEqual(tile, { - errors: ['You are over platform\'s limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'render', - message: 'You are over platform\'s limits. Please contact us to know more details' - }] + this.testClient.drain(done); }); - - done(); }); - }); - }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details' + }] + }); + + done(); + }); + }); + }; + } describe('interativity', function () { beforeEach(function (done) { From 6af83d7630e23f6432bdcfd1828f9e7858756c64 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 14:54:03 +0200 Subject: [PATCH 025/167] Use yarn instead of npm in docker --- docker-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-test.sh b/docker-test.sh index e340369e..6bf5360e 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,6 +1,7 @@ export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres -npm install +npm install -g yarn@0.27.5 +yarn /etc/init.d/postgresql start From 1766cd0ad4235dcc4c5ce034427dd85a1aa4ed6f Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 15:42:24 +0200 Subject: [PATCH 026/167] Buffersize test fixed to test PostGIS and mapnik MVT paths --- test/acceptance/buffer-size-format.js | 88 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index e41cf1d3..b888e094 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -3,6 +3,7 @@ require('../support/test_helper'); var fs = require('fs'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); var mapnik = require('windshaft').mapnik; var IMAGE_TOLERANCE_PER_MIL = 5; @@ -131,21 +132,35 @@ describe('buffer size per format', function () { return done(); }); + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { - it(test.desc, function (done) { - this.testClient = new TestClient(test.mapConfig, 1234); - var coords = test.coords; - var options = { - format: test.format, - layers: test.layers - }; - this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - // tile.save(test.fixturePath); - test.assert(tile, done); + var testFn = (usePostGIS) => { + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + it(test.desc, function (done) { + this.testClient = new TestClient(test.mapConfig, 1234); + var coords = test.coords; + var options = { + format: test.format, + layers: test.layers + }; + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + // tile.save(test.fixturePath); + test.assert(tile, done); + }); }); - }); + }; + if (test.format === 'mvt'){ + testFn(true); + } + testFn(false); }); }); @@ -431,23 +446,36 @@ describe('buffer size per format for named maps w/o placeholders', function () { return done(); }); + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { - it(test.desc, function (done) { - this.testClient = new TestClient(test.template, 1234); - var coords = test.coords; - var options = { - format: test.format, - placeholders: test.placeholders, - layers: test.layers - }; - this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - //tile.save(test.fixturePath); - // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); - // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); - test.assert(tile, done); - }); - }); + var testFn = (usePostGIS) => { + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + it(test.desc, function (done) { + this.testClient = new TestClient(test.template, 1234); + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders, + layers: test.layers + }; + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save(test.fixturePath); + // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); + // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); + test.assert(tile, done); + }); + }); + }; + if (test.format === 'mvt'){ + testFn(true); + } + testFn(false); }); }); From 82648df21cc05a4e31be11e645ad50d4142bcca6 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 15:49:51 +0200 Subject: [PATCH 027/167] Fix jshint --- test/acceptance/mvt.js | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index af67f8bf..cccfbcf4 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -144,7 +144,7 @@ return function () { const { z, x, y } = test.coords; const { format, response } = test; - testClient.getTile(z, x, y, { format, response }, (err, res) => { + testClient.getTile(z, x, y, { format, response }, err => { assert.ifError(err); testClient.drain(done); }); @@ -185,7 +185,8 @@ return function () { type: 'mapnik', options: { sql: 'select * from populated_places_simple_reduced', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', + cartocss: + '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', cartocss_version: '2.0.1', widgets: { adm0name: { @@ -336,20 +337,20 @@ return function () { ].join('\n'); var sql = [ - 'WITH hgrid AS (', - ' SELECT CDB_HexagonGrid(', - ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', - ' greatest(!pixel_width!,!pixel_height!) * 100', - ' ) as cell', - ')', - 'SELECT', - ' hgrid.cell as the_geom_webmercator,', - ' count(1) as points_count,', - ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', - ' 1 as cartodb_id', - 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', - 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', - 'GROUP BY hgrid.cell' + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' ].join('\n'); var mapConfig = { @@ -499,5 +500,5 @@ return function () { }); } -} +}; } \ No newline at end of file From 382790153585c24ba1ec0d276db8b3d68892fb63 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 15:53:50 +0200 Subject: [PATCH 028/167] Skip test to ignore MVT v2 compliance --- test/acceptance/mvt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index cccfbcf4..2bead7e4 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -325,7 +325,7 @@ return function () { }); }); - it('should work with mapnik substitution tokens', function (done) { + it.skip('should work with mapnik substitution tokens', function (done) { var cartocss = [ "#layer {", " line-width: 2;", From d77739dfa43a7ea5a9ed8870e41a5d1a20271b9a Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 16:46:23 +0200 Subject: [PATCH 029/167] PostGIS MVT v2 compliance test skipped without skipping mapnik --- test/acceptance/mvt.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 2bead7e4..883f050f 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -325,7 +325,8 @@ return function () { }); }); - it.skip('should work with mapnik substitution tokens', function (done) { + var skipOnPostGIS = usePostGIS ? it.skip: it; + skipOnPostGIS('should work with mapnik substitution tokens', function (done) { var cartocss = [ "#layer {", " line-width: 2;", From 8a88b29665ae3499efd10df3f6722f718cdaa43b Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 17:16:29 +0200 Subject: [PATCH 030/167] update dependencies --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index d6e719a2..cf95c05b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -240,6 +240,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -248,14 +256,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -1572,7 +1572,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: From b18bf967fdcee108d6ea8bda8eaeeb3a1def81a4 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 18:27:57 +0200 Subject: [PATCH 031/167] Workaround lack of template creation clean-up --- test/acceptance/buffer-size-format.js | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index b888e094..2366d035 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -135,15 +135,10 @@ describe('buffer size per format', function () { const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { var testFn = (usePostGIS) => { - before(function () { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - }); - after(function () { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - }); - - it(test.desc, function (done) { + it(test.desc, function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; this.testClient = new TestClient(test.mapConfig, 1234); + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; var coords = test.coords; var options = { format: test.format, @@ -449,14 +444,11 @@ describe('buffer size per format for named maps w/o placeholders', function () { const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { var testFn = (usePostGIS) => { - before(function () { + it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) { serverOptions.renderer.mvt.usePostGIS = usePostGIS; - }); - after(function () { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - }); - it(test.desc, function (done) { + test.template.name += '_1'; this.testClient = new TestClient(test.template, 1234); + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; var coords = test.coords; var options = { format: test.format, From 8cc4fe5b563f1c1bbe601c4ab8632aa14da5fb13 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Wed, 11 Oct 2017 11:34:27 +0200 Subject: [PATCH 032/167] Target Windshaft ~4.0.0, yarn.lock updated --- package.json | 2 +- yarn.lock | 106 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 80 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index b65a64d1..26f3ed73 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.1", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft", + "windshaft": "~4.0.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index cf95c05b..6d36931f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -222,7 +226,7 @@ camshaft@0.59.2: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -240,15 +244,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -516,7 +520,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -602,8 +610,8 @@ exit@0.1.2, exit@0.1.x: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" express@~4.16.0: - version "4.16.1" - resolved "https://registry.yarnpkg.com/express/-/express-4.16.1.tgz#6b33b560183c9b253b7b62144df33a4654ac9ed0" + version "4.16.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" dependencies: accepts "~1.3.4" array-flatten "1.1.1" @@ -746,7 +754,7 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -758,6 +766,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -768,7 +780,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -799,6 +811,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1300,13 +1323,13 @@ mime@~1.3.4: version "1.3.6" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1314,6 +1337,10 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1340,7 +1367,11 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@~2.18.1: +moment@^2.10.6: + version "2.19.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.0.tgz#44f675ef6b944942762581b1c179fb679e599d67" + +moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1449,10 +1480,14 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1572,7 +1607,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1740,7 +1775,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1770,6 +1805,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1879,9 +1923,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1895,6 +1939,10 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + send@0.16.1: version "0.16.1" resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" @@ -1992,10 +2040,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2093,7 +2145,7 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" -supports-color@3.1.2, supports-color@^3.1.0, supports-color@^3.1.2: +supports-color@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" dependencies: @@ -2103,7 +2155,7 @@ supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" -supports-color@^3.2.3: +supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.2.3.tgz#65ac0504b3954171d8a64946b2ae3cbb8a5f54f6" dependencies: @@ -2141,7 +2193,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2149,7 +2201,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2310,9 +2362,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft: +windshaft@~4.0.0: version "4.0.0" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/5b21e24e66c8e24cb88640b606aaf5fefcd3f365" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.0.tgz#b28ffb3775db3a4f2578e79d73c9b854a1ae5854" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 67ab12e8e779f7341f4a7eb4c10d2b0b944e364f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 11:08:46 +0200 Subject: [PATCH 033/167] Add environment variable to check whether to run test against postgis 2.4 or not --- docker-test.sh | 2 +- test/acceptance/mvt.js | 6 ++++-- test/acceptance/user-render-timeout-limit.js | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker-test.sh b/docker-test.sh index 6bf5360e..8cc8c093 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -8,4 +8,4 @@ yarn createdb template_postgis && createuser publicuser psql -c "CREATE EXTENSION postgis" template_postgis -npm test \ No newline at end of file +POSTGIS_VERSION=2.4 npm test \ No newline at end of file diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 883f050f..ded1772e 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -20,7 +20,9 @@ function createMapConfig(sql = TestClient.SQL.ONE_POINT) { } describe('mvt (mapnik)', mvt(false)); -describe('mvt (postgis)', mvt(true)); +if (process.env.POSTGIS_VERSION === '2.4') { + describe('mvt (postgis)', mvt(true)); +} function mvt(usePostGIS) { return function () { @@ -185,7 +187,7 @@ return function () { type: 'mapnik', options: { sql: 'select * from populated_places_simple_reduced', - cartocss: + cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', cartocss_version: '2.0.1', widgets: { diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index eaf98805..0bb45c2b 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -201,7 +201,10 @@ describe('user render timeout limit', function () { }); }); - describe('vector (PostGIS)', vector(true)); + if (process.env.POSTGIS_VERSION === '2.4') { + describe('vector (PostGIS)', vector(true)); + } + describe('vector (mapnik)', vector(false)); function vector(usePostGIS) { const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; From f8ff41be01cfd9bbf76efe91ff8a376937645b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 12:15:28 +0200 Subject: [PATCH 034/167] Do not run test if postgis version is lower than or equal to 2.4 --- docker-test.sh | 4 ++-- test/acceptance/buffer-size-format.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-test.sh b/docker-test.sh index 8cc8c093..00cc63b3 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,4 +1,4 @@ -export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres export POSTGIS_VERSION=2.4 npm install -g yarn@0.27.5 yarn @@ -8,4 +8,4 @@ yarn createdb template_postgis && createuser publicuser psql -c "CREATE EXTENSION postgis" template_postgis -POSTGIS_VERSION=2.4 npm test \ No newline at end of file +npm test \ No newline at end of file diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 2366d035..ba8955e7 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -152,7 +152,7 @@ describe('buffer size per format', function () { }); }); }; - if (test.format === 'mvt'){ + if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ testFn(true); } testFn(false); @@ -465,7 +465,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { }); }); }; - if (test.format === 'mvt'){ + if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ testFn(true); } testFn(false); From a1c08f9bf74f2c80a61312cc3b95970d11da7058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 12:53:23 +0200 Subject: [PATCH 035/167] Revert docker-test config --- docker-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-test.sh b/docker-test.sh index 00cc63b3..6bf5360e 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,4 +1,4 @@ -export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres export POSTGIS_VERSION=2.4 +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres npm install -g yarn@0.27.5 yarn From 515146bf285d40dffeae54592f504be6a817ba8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 13:12:06 +0200 Subject: [PATCH 036/167] Configure travis to export environment variable (POSTGIS_VERSION) --- .travis.yml | 3 +++ docker-test.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b946378f..9ac20ae7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,5 +10,8 @@ before_install: script: - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing +after_success: + - export POSTGIS_VERSION=2.4 + language: generic diff --git a/docker-test.sh b/docker-test.sh index 6bf5360e..8cc8c093 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -8,4 +8,4 @@ yarn createdb template_postgis && createuser publicuser psql -c "CREATE EXTENSION postgis" template_postgis -npm test \ No newline at end of file +POSTGIS_VERSION=2.4 npm test \ No newline at end of file From 294a222669b39318a76f4cb0b1b2f65a56548eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 13:20:30 +0200 Subject: [PATCH 037/167] Configure travis to export environment variable (POSTGIS_VERSION) --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ac20ae7..d881854c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,7 @@ before_install: - docker pull cartoimages/windshaft-carto-testing script: - - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing - -after_success: - - export POSTGIS_VERSION=2.4 + - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing -e POSTGIS_VERSION=2.4 language: generic From c2bf7b075ca859d9687f6000ddad1f73c056b313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 13:24:46 +0200 Subject: [PATCH 038/167] Fix travis to export environment variable (POSTGIS_VERSION) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d881854c..5bc0cf1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ before_install: - docker pull cartoimages/windshaft-carto-testing script: - - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing -e POSTGIS_VERSION=2.4 + - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing language: generic From 3068ff1ea4abdce9d4e68853c4d12d64f79ed3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 15:16:21 +0200 Subject: [PATCH 039/167] Update NEWS --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 16cb601e..92ef4bff 100644 --- a/NEWS.md +++ b/NEWS.md @@ -7,6 +7,7 @@ Released 2017-mm-dd - Use express error handler middleware to respond in case of something went wrong. - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params. - Move `LZMA` decompression to its own middleware. + - Implement stats middleware removing some duplicated code while sending response. ## 4.0.0 From 1207764c1871b19903c7baa8b1d081b7034b07b8 Mon Sep 17 00:00:00 2001 From: csobier Date: Mon, 16 Oct 2017 15:54:49 -0400 Subject: [PATCH 040/167] added map tile rendering and MVT info to Maps API doc- WIP --- docs/Map-API.md | 1 + docs/general_concepts.md | 20 +++++++++++++++++++- docs/static_maps_api.md | 3 ++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/docs/Map-API.md b/docs/Map-API.md index 2058740f..456cf379 100644 --- a/docs/Map-API.md +++ b/docs/Map-API.md @@ -17,3 +17,4 @@ You can create two types of maps with the Maps API: * [Anonymous Maps](anonymous_maps.md) * [Named Maps](named_maps.md) * [Static Maps API](static_maps_api.md) +* [MapConfig File Format]([local file in the docs repo](https://github.com/CartoDB/docs/blob/master/_app/_mapsapi/06-mapconfig.md)) diff --git a/docs/general_concepts.md b/docs/general_concepts.md index 0044096e..e34bfbd7 100644 --- a/docs/general_concepts.md +++ b/docs/general_concepts.md @@ -22,6 +22,24 @@ Errors are reported using standard HTTP codes and extended information encoded i If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object. -## CORS support +## CORS Support All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method. + +## Map Tile Rendering + +Map tiles create the graphical representation of your map in a web browser. The performance rendering of map tiles is dependent on the type of geospatial data model (raster or vector) that you are using. + +- **Raster**: Generates map tiles based on a grid of pixels to represent your data. Each cell is a fixed size and contains values for particular map features. On the server-side, each request queries a dataset to retrieve data for each map tile. The grid size of map tiles can often lead to graphic quality issues. + +- **Vector**: Generates map tiles based on pre-defined coordinates to represent your data, similar to how basemap image tiles are rendered. On the client-side, map tiles represent real-world geometries of a map. Depending on the coordinates, vertices are used to connect the data and display points, lines, or polygons for the map tiles. + +**Note:** By default, CARTO uses vector graphics for map rendering. Please [contact us](mailto:support@carto.com) if you need raster rendering enabled as part of your requirements. + +### Mapbox Vector Tiles (MVT) + +[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server. + +CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. + +**Tip:** You can process MVT files with the [`ST_AsMVT` PostGIS function](https://postgis.net/docs/manual-dev/ST_AsMVT.html) with the [Maps API Windshaft renderer](https://github.com/CartoDB/Windshaft/blob/1000x/lib/windshaft/renderers/pg_mvt/renderer.js). diff --git a/docs/static_maps_api.md b/docs/static_maps_api.md index 91db2a6d..34e4e907 100644 --- a/docs/static_maps_api.md +++ b/docs/static_maps_api.md @@ -152,7 +152,8 @@ It is important to note that generated images are cached from the live data refe * Timeout limits for generating static maps are the same across CARTO Builder and CARTO Engine. It is important to ensure timely processing of queries. * If you are publishing your map as a static image with the API, you must manually add [attributions](https://carto.com/attribution) for your static map image. For example, add the following attribution code: -{% highlight javascript %}attribution: '© OpenStreetMap contributors, © CARTO +{% highlight javascript %} +attribution: '© OpenStreetMap contributors, © CARTO {% endhighlight %} ## Examples From 9326217c18e7dbbec84ebb1052a25f1fb9014a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 18 Oct 2017 14:36:04 +0200 Subject: [PATCH 041/167] Upgrade camshaft to version 0.59.4 --- package.json | 2 +- yarn.lock | 122 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 92 insertions(+), 32 deletions(-) diff --git a/package.json b/package.json index 3a52e3d9..d3e488aa 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ ], "dependencies": { "body-parser": "^1.18.2", - "camshaft": "0.59.2", + "camshaft": "0.59.4", "cartodb-psql": "0.10.2", "cartodb-query-tables": "0.3.0", "cartodb-redis": "0.14.0", diff --git a/yarn.lock b/yarn.lock index db232491..ba46c6f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,11 +10,7 @@ mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" - -abbrev@1.0.x: +abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -114,7 +110,11 @@ aws-sign2@~0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f" -aws4@^1.2.1: +aws-sign2@~0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" + +aws4@^1.2.1, aws4@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e" @@ -159,6 +159,18 @@ boom@2.x.x: dependencies: hoek "2.x.x" +boom@4.x.x: + version "4.3.1" + resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31" + dependencies: + hoek "4.x.x" + +boom@5.x.x: + version "5.2.0" + resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02" + dependencies: + hoek "4.x.x" + brace-expansion@^1.1.7: version "1.1.8" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" @@ -199,9 +211,9 @@ camelcase@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a" -camshaft@0.59.2: - version "0.59.2" - resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.2.tgz#8b032771faa1264bd8a81040c6075beb1a32e286" +camshaft@0.59.4: + version "0.59.4" + resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.59.4.tgz#7d4d0b8585fa180b5e750206aa84940870ebabfc" dependencies: async "^1.5.2" bunyan "1.8.1" @@ -210,7 +222,7 @@ camshaft@0.59.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -228,15 +240,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -"carto@github:cartodb/carto#0.15.1-cdb1": +carto@CartoDB/carto#0.15.1-cdb1: version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -344,7 +356,7 @@ combined-stream@^1.0.5, combined-stream@~1.0.5: dependencies: delayed-stream "~1.0.0" -commander@2.9.0, commander@^2.9.0: +commander@2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4" dependencies: @@ -390,6 +402,12 @@ cryptiles@2.x.x: dependencies: boom "2.x.x" +cryptiles@3.x.x: + version "3.1.2" + resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe" + dependencies: + boom "5.x.x" + d3-queue@^2.0.2: version "2.0.3" resolved "https://registry.yarnpkg.com/d3-queue/-/d3-queue-2.0.3.tgz#07fbda3acae5358a9c5299aaf880adf0953ed2c2" @@ -618,7 +636,7 @@ express@~4.16.0: utils-merge "1.0.1" vary "~1.1.2" -extend@~3.0.0: +extend@~3.0.0, extend@~3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444" @@ -626,6 +644,10 @@ extsprintf@1.3.0, extsprintf@^1.2.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" +fast-deep-equal@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff" + fast-levenshtein@~2.0.4: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" @@ -832,6 +854,13 @@ har-validator@~4.2.1: ajv "^4.9.1" har-schema "^1.0.5" +har-validator@~5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd" + dependencies: + ajv "^5.1.0" + har-schema "^2.0.0" + has-ansi@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-2.0.0.tgz#34f5049ce1ecdf2b0649af3ef24e45ed35416d91" @@ -855,6 +884,15 @@ hawk@3.1.3, hawk@~3.1.3: hoek "2.x.x" sntp "1.x.x" +hawk@~6.0.2: + version "6.0.2" + resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038" + dependencies: + boom "4.x.x" + cryptiles "3.x.x" + hoek "4.x.x" + sntp "2.x.x" + hiredis@~0.5.0: version "0.5.0" resolved "https://registry.yarnpkg.com/hiredis/-/hiredis-0.5.0.tgz#db03a98becd7003d13c260043aceecfacdf59b87" @@ -866,6 +904,10 @@ hoek@2.x.x: version "2.16.3" resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed" +hoek@4.x.x: + version "4.2.0" + resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d" + hosted-git-info@^2.1.4: version "2.5.0" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.5.0.tgz#6d60e34b3abbc8313062c3b798ef8d901a07af3c" @@ -897,6 +939,14 @@ http-signature@~1.1.0: jsprim "^1.2.2" sshpk "^1.7.0" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + husl@^6.0.1: version "6.0.6" resolved "https://registry.yarnpkg.com/husl/-/husl-6.0.6.tgz#f71b3e45d2000d6406432a9cc17a4b7e0c5b800d" @@ -1023,6 +1073,10 @@ jshint@~2.9.4: shelljs "0.3.x" strip-json-comments "1.0.x" +json-schema-traverse@^0.3.0: + version "0.3.1" + resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340" + json-schema@0.2.3: version "0.2.3" resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13" @@ -1246,6 +1300,10 @@ mime@~1.2.11: version "1.2.11" resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10" +mime@~1.3.4: + version "1.3.6" + resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" + "minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" @@ -1286,11 +1344,7 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6: - version "2.19.0" - resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.0.tgz#44f675ef6b944942762581b1c179fb679e599d67" - -moment@~2.18.1: +moment@^2.10.6, moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1395,7 +1449,7 @@ number-is-nan@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/number-is-nan/-/number-is-nan-1.0.1.tgz#097b602b53422a522c1afb8790318336941a011d" -oauth-sign@~0.8.1: +oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" @@ -1506,6 +1560,10 @@ performance-now@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-0.2.0.tgz#33ef30c5c77d4ea21c5a53869d91b56d8f2555e5" +performance-now@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" + pg-connection-string@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-0.1.3.tgz#da1847b20940e42ee1492beaf65d49d91b245df7" @@ -1526,7 +1584,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1849,10 +1907,6 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - send@0.16.1: version "0.16.1" resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" @@ -1910,6 +1964,12 @@ sntp@1.x.x: dependencies: hoek "2.x.x" +sntp@2.x.x: + version "2.0.2" + resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.0.2.tgz#5064110f0af85f7cfdb7d6b67a40028ce52b4b2b" + dependencies: + hoek "4.x.x" + source-map@^0.4.4: version "0.4.4" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b" @@ -2021,7 +2081,7 @@ string_decoder@~1.0.3: dependencies: safe-buffer "~5.1.0" -stringstream@~0.0.4: +stringstream@~0.0.4, stringstream@~0.0.5: version "0.0.5" resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878" @@ -2093,7 +2153,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2101,7 +2161,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2215,7 +2275,7 @@ utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" -uuid@^3.0.0: +uuid@^3.0.0, uuid@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" From ddcb812218f92b97677dce4ffacc2e0e9ddb548f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 18 Oct 2017 15:14:03 +0200 Subject: [PATCH 042/167] Update NEWS --- NEWS.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NEWS.md b/NEWS.md index 92ef4bff..62a4251a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ ## 4.0.1 Released 2017-mm-dd +Announcements: + - Upgrades camshaft to [0.59.4](https://github.com/CartoDB/camshaft/releases/tag/0.59.4). - Split and move `req2params` method to multiple middlewares. - Use express error handler middleware to respond in case of something went wrong. - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params. From 1396ca9fe321e5b0690de912cc4f07f65147e4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 18 Oct 2017 15:17:17 +0200 Subject: [PATCH 043/167] Release 4.0.1 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 62a4251a..4b4aeda7 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 4.0.1 -Released 2017-mm-dd +Released 2017-10-18 Announcements: - Upgrades camshaft to [0.59.4](https://github.com/CartoDB/camshaft/releases/tag/0.59.4). From 35c5cd34c2ea3f6b278faba9a5a73dc2299e5e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 18 Oct 2017 15:19:37 +0200 Subject: [PATCH 044/167] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4b4aeda7..5b70378c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 4.0.2 +Released 2017-mm-dd + + ## 4.0.1 Released 2017-10-18 diff --git a/package.json b/package.json index d3e488aa..f4c31609 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.0.1", + "version": "4.0.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From eb6da1398e696e94371a3cd1457a404ab0e49317 Mon Sep 17 00:00:00 2001 From: csobier Date: Wed, 18 Oct 2017 13:34:33 -0400 Subject: [PATCH 045/167] mvt content added to anonymous map chapter, under retrieve resources heading- WIP --- docs/anonymous_maps.md | 204 ++++++++++++++++++++++++++++++++++++--- docs/general_concepts.md | 18 ---- 2 files changed, 191 insertions(+), 31 deletions(-) diff --git a/docs/anonymous_maps.md b/docs/anonymous_maps.md index e8ed43ef..2ff63f85 100644 --- a/docs/anonymous_maps.md +++ b/docs/anonymous_maps.md @@ -70,21 +70,203 @@ curl 'https://{username}.carto.com/api/v1/map' -H 'Content-Type: application/jso } ``` -### Retrieve resources from the layergroup +## Map Tile Rendering + +Map tiles create the graphical representation of your map in a web browser. The performance rendering of map tiles is dependent on the type of geospatial data model (raster or vector) that you are using. + +- **Raster**: Generates map tiles based on a grid of pixels to represent your data. Each cell is a fixed size and contains values for particular map features. On the server-side, each request queries a dataset to retrieve data for each map tile. The grid size of map tiles can often lead to graphic quality issues. + +- **Vector**: Generates map tiles based on pre-defined coordinates to represent your data, similar to how basemap image tiles are rendered. On the client-side, map tiles represent real-world geometries of a map. Depending on the coordinates, vertices are used to connect the data and display points, lines, or polygons for the map tiles. + +## Retrieve resources from the layergroup When you have a layergroup, there are several resources for retrieving layergoup details such as, accessing Mapnik tiles, getting individual layers, accessing defined Attributes, and blending and layer selection. -#### Mapnik tiles +### Mapnik tiles -These tiles will get just the Mapnik layers. To get individual layers, see the following section. +These raster tiles retrieve just the Mapnik layers. See [individual layers](#individual-layers) for details about how to retrieve other layers. ```bash https://{username}.carto.com/api/v1/map/{layergroupid}/{z}/{x}/{y}.png ``` -#### Individual layers +### Mapbox Vector Tiles (MVT) -The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually in different formats depending on the layer type. +[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server. + +CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. However, you can use any implementation tool for processing MVT files. + +The following examples describe how to fetch MVT tiles with a cURL request. + +#### MVT and Windshaft + +CARTO uses Windshaft as the map tiler library to render multilayer maps with the Maps API. You can use Windshaft to request MVT using the same layer type that is used for requesting raster tiles (Mapnik layer). Simply change the file format `.mvt` in the URL. + + +```bash +https://{username}.cartodb.com/api/v1/map/HASH/:layer/{z}/{x}/{y}.mvt +``` + +The following example instantiates an anonymous map with layer options: + +```bash +{ + user_name: 'mycartodbuser', + sublayers: [{ + sql: "SELECT * FROM table_name"; + cartocss: '#layer { marker-fill: #F0F0F0; }' + }], + maps_api_template: 'https://{user}.cartodb.com' // Optional +} +``` + +**Note**: If no layer type is specified, Mapnik tiles are used by default. To access MVT tiles, specify `https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt` as the `maps_api_template` variable. + +**Tip:** If you are using [Named Maps](https://carto.com/docs/carto-engine/maps-api/named-maps/) to instantiate a layer, indicate the MVT file format and layer in the response: + +```bash +https://{username}.cartodb.com/api/v1/map/named/:templateId/:layer/{z}/{x}/{y}.mvt +``` + +For all layers in a Named Map, you must indicate Mapnik as the layer filter: + +```bash +https://{username}.cartodb.com/api/v1/map/named/:templateId/mapnik/{z}/{x}/{y}.mvt +``` + +#### Layergroup Filter for MVT Tiles + +To filter layers using Windshaft, use the following request where layers are numbered: + +```bash +https://{username}.cartodb.com/api/v1/map/HASH/0,1,2/{z}/{x}/{y}.mvt +``` + +To request all layers, remove the layergroup filter parameter: + +```bash +https://{username}.cartodb.com/api/v1/map/HASH/{z}/{x}/{y}.mvt +``` + +To filter a specific layer: + +```bash +https://{username}.cartodb.com/api/v1/map/HASH/2/{z}/{x}/{y}.mvt +``` + +#### Example 1: MVT Tiles with Windshaft, CARTO.js, and MapboxGL + +1. Import the required libraries: + +```bash + + + +``` + +2. Configure Map Client: + +```bash +mapboxgl.accessToken = '{yourMapboxToken}'; +``` + +3. Create Map Object (Mapbox): + +```bash +var map = new mapboxgl.Map({ +container: 'map', +zoom: 1, +minZoom: 0, +maxZoom: 18, +center: [30, 0] +}); +``` + +4. Define Layer Options (CARTO): + +```bash +var layerOptions = { +user_name: "{username}", +sublayers: [{ +sql: "SELECT * FROM {table_name}", +cartocss: "...", + }] +}; +``` + +5. Request Tiles (from CARTO) and Set to Map Object (Mapbox): + +**Note:** By default, [CARTO core functions](https://carto.com/docs/carto-engine/carto-js/core-api/) retrieve URLs for fully rendered tiles. You must replace the default format (.png) with the MVT format (.mvt). + + +```bash +cartodb.Tiles.getTiles(layerOptions, function(result, err) { +var tiles = result.tiles.map(function(tileUrl) { +return tileUrl +.replace('{s}', 'a') +.replace(/\.png/, '.mvt'); +}); +map.setStyle(simpleStyle(tiles)); +}); +``` + +#### Example 2: MVT Libraries with Windshaft and MapboxGL + +When you are not including CARTO.js to implement MVT tiles, you must use the `map.setStyle` parameter to specify vector map rendering. + +1. Import the required libraries: + +```bash + + +``` + +2. Configure Map Client: + +```bash +mapboxgl.accessToken = '{yourMapboxToken}'; +``` + +3. Create Map Object (Mapbox): + +```bash +var map = new mapboxgl.Map({ +container: 'map', +zoom: 1, +minZoom: 0, +maxZoom: 18, +center: [30, 0] +}); +``` + +4. Set the Style + +```bash +map.setStyle({ + "version": 7, + "glyphs": "...", + "constants": {...}, + "sources": { + "cartodb": { + "type": "vector", + "tiles": [ "http://{username}.cartodb.com/api/v1/map/named/templateId/mapnik/{z}/{x}/{y}.mvt" + ], + "maxzoom": 18 + } + }, + "layers": [{...}] +}); +``` + +**Tip:** If you are using MapboxGL, see the following resource for additional information. + +- [MapboxGL API Reference](https://www.mapbox.com/mapbox-gl-js/api/) +- [MapboxGL Style Specifications](https://www.mapbox.com/mapbox-gl-js/style-spec/) +- [Example of MapboxGL Implementation](https://www.mapbox.com/mapbox-gl-js/examples/) + +### Individual layers + +The MapConfig specification holds the layers definition in a 0-based index. Layers can be requested individually, in different formats, depending on the layer type. Individual layers can be accessed using that 0-based index. For UTF grid tiles: @@ -100,19 +282,19 @@ If the MapConfig had a Torque layer at index 1 it could be possible to request i https://{username}.carto.com/api/v1/map/{layergroupid}/1/{z}/{x}/{y}.torque.json ``` -#### Attributes defined in `attributes` section +### Attributes defined in `attributes` section ```bash https://{username}.carto.com/api/v1/map/{layergroupid}/{layer}/attributes/{feature_id} ``` -Which returns JSON with the attributes defined, like: +Which returns JSON with the attributes defined, such as: ```javascript { "c": 1, "d": 2 } ``` -#### Blending and layer selection +### Blending and layer selection ```bash https://{username}.carto.com/api/v1/map/{layergroupid}/{layer_filter}/{z}/{x}/{y}.png @@ -141,10 +323,7 @@ https://{username}.carto.com/api/v1/map/{layergroupid}/0,3,4/{z}/{x}/{y}.png Some notes about filtering: - Invalid index values or out of bounds indexes will end in `Invalid layer filtering` errors. - - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this - may change in the future **it is recommended** to always select the layers in ascending order so you will get a - consistent behavior in the future. - + - Ordering is not considered. So right now filtering layers 0,3,4 is the very same thing as filtering 3,4,0. As this may change in the future, **it is recommended** to always select the layers in ascending order so that you will always get consistent behavior. ## Create JSONP @@ -185,7 +364,6 @@ callback({ }) ``` - ## Remove Anonymous Maps cannot be removed by an API call. They will expire after about five minutes, or sometimes longer. If an Anonymous Map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time, returns to the map and attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem. diff --git a/docs/general_concepts.md b/docs/general_concepts.md index e34bfbd7..ca8c72ec 100644 --- a/docs/general_concepts.md +++ b/docs/general_concepts.md @@ -25,21 +25,3 @@ If you use JSONP, the 200 HTTP code is always returned so the JavaScript client ## CORS Support All the endpoints, which might be accessed using a web browser, add CORS headers and allow OPTIONS method. - -## Map Tile Rendering - -Map tiles create the graphical representation of your map in a web browser. The performance rendering of map tiles is dependent on the type of geospatial data model (raster or vector) that you are using. - -- **Raster**: Generates map tiles based on a grid of pixels to represent your data. Each cell is a fixed size and contains values for particular map features. On the server-side, each request queries a dataset to retrieve data for each map tile. The grid size of map tiles can often lead to graphic quality issues. - -- **Vector**: Generates map tiles based on pre-defined coordinates to represent your data, similar to how basemap image tiles are rendered. On the client-side, map tiles represent real-world geometries of a map. Depending on the coordinates, vertices are used to connect the data and display points, lines, or polygons for the map tiles. - -**Note:** By default, CARTO uses vector graphics for map rendering. Please [contact us](mailto:support@carto.com) if you need raster rendering enabled as part of your requirements. - -### Mapbox Vector Tiles (MVT) - -[Mapbox Vector Tiles (MVT)](https://www.mapbox.com/vector-tiles/specification/) are map tiles that store geographic vector data on the client-side. Browser performance is fast since you can pan and zoom without having to query the server. - -CARTO uses a Web Graphics Library (WebGL) to process MVT files. This is useful since WebGL's are compatible with most web browsers, include support for multiple client-side mapping engines, and do not require additional information from the server; which makes it more efficient for rendering map tiles. - -**Tip:** You can process MVT files with the [`ST_AsMVT` PostGIS function](https://postgis.net/docs/manual-dev/ST_AsMVT.html) with the [Maps API Windshaft renderer](https://github.com/CartoDB/Windshaft/blob/1000x/lib/windshaft/renderers/pg_mvt/renderer.js). From c0df0d12c686fba836f3ea9f109c018bfaf9fd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 20 Oct 2017 17:21:55 +0200 Subject: [PATCH 046/167] ensure dont overwrite data in copy from req.params to res.locals --- lib/cartodb/middleware/context/locals.js | 2 +- lib/cartodb/middleware/user.js | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/cartodb/middleware/context/locals.js b/lib/cartodb/middleware/context/locals.js index bb7d5ee2..eb8fca89 100644 --- a/lib/cartodb/middleware/context/locals.js +++ b/lib/cartodb/middleware/context/locals.js @@ -1,7 +1,7 @@ const _ = require('underscore'); module.exports = function localsMiddleware(req, res, next) { - _.extend(res.locals, req.params); + _.defaults(res.locals, req.params); next(); }; diff --git a/lib/cartodb/middleware/user.js b/lib/cartodb/middleware/user.js index ce3fdd29..adf06203 100644 --- a/lib/cartodb/middleware/user.js +++ b/lib/cartodb/middleware/user.js @@ -4,11 +4,5 @@ var cdbRequest = new CdbRequest(); module.exports = function userMiddleware(req, res, next) { res.locals.user = cdbRequest.userByReq(req); - // avoid a req.params.user equals to undefined - // overwrites res.locals.user - if(!req.params.user) { - delete req.params.user; - } - next(); }; From b04cc9c228831c70c1426cb8a6049755e2a14f2b Mon Sep 17 00:00:00 2001 From: csobier Date: Fri, 20 Oct 2017 12:52:40 -0400 Subject: [PATCH 047/167] Fixed numbering formatting for steps No code changes, just formatting. (The API repos require different format of numbering then that docs. I couldn't see it until the output was live). --- docs/anonymous_maps.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/docs/anonymous_maps.md b/docs/anonymous_maps.md index 2ff63f85..1dc2f5f3 100644 --- a/docs/anonymous_maps.md +++ b/docs/anonymous_maps.md @@ -156,7 +156,7 @@ https://{username}.cartodb.com/api/v1/map/HASH/2/{z}/{x}/{y}.mvt #### Example 1: MVT Tiles with Windshaft, CARTO.js, and MapboxGL -1. Import the required libraries: +1) Import the required libraries: ```bash @@ -164,13 +164,13 @@ https://{username}.cartodb.com/api/v1/map/HASH/2/{z}/{x}/{y}.mvt ``` -2. Configure Map Client: +2) Configure Map Client: ```bash mapboxgl.accessToken = '{yourMapboxToken}'; ``` -3. Create Map Object (Mapbox): +3) Create Map Object (Mapbox): ```bash var map = new mapboxgl.Map({ @@ -182,7 +182,7 @@ center: [30, 0] }); ``` -4. Define Layer Options (CARTO): +4) Define Layer Options (CARTO): ```bash var layerOptions = { @@ -194,7 +194,7 @@ cartocss: "...", }; ``` -5. Request Tiles (from CARTO) and Set to Map Object (Mapbox): +5) Request Tiles (from CARTO) and Set to Map Object (Mapbox): **Note:** By default, [CARTO core functions](https://carto.com/docs/carto-engine/carto-js/core-api/) retrieve URLs for fully rendered tiles. You must replace the default format (.png) with the MVT format (.mvt). @@ -214,20 +214,20 @@ map.setStyle(simpleStyle(tiles)); When you are not including CARTO.js to implement MVT tiles, you must use the `map.setStyle` parameter to specify vector map rendering. -1. Import the required libraries: +1) Import the required libraries: ```bash ``` -2. Configure Map Client: +2) Configure Map Client: ```bash mapboxgl.accessToken = '{yourMapboxToken}'; ``` -3. Create Map Object (Mapbox): +3) Create Map Object (Mapbox): ```bash var map = new mapboxgl.Map({ @@ -239,7 +239,7 @@ center: [30, 0] }); ``` -4. Set the Style +4) Set the Style ```bash map.setStyle({ From b7227e0581486c895ab8da932b7f14db5b7c4c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 27 Oct 2017 11:11:58 +0200 Subject: [PATCH 048/167] Upgrade windshaft to version 4.0.1 --- package.json | 2 +- yarn.lock | 28 ++++++++++------------------ 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/package.json b/package.json index f4c31609..8e4c1d84 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.1", "underscore": "~1.6.0", - "windshaft": "~4.0.0", + "windshaft": "4.0.1", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index ba46c6f4..7b5394aa 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -222,7 +222,7 @@ camshaft@0.59.4: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -248,7 +248,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -746,7 +746,7 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -758,10 +758,6 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -1453,14 +1449,10 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -2153,7 +2145,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2161,7 +2153,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2322,9 +2314,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@~4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.0.tgz#b28ffb3775db3a4f2578e79d73c9b854a1ae5854" +windshaft@4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.1.tgz#16230a0b5b28f08a443491339c37637e3caefa52" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From cd7c604d1070e7aae1ae102d5966b8676640154c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 27 Oct 2017 11:18:10 +0200 Subject: [PATCH 049/167] Update NEWS --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 5b70378c..6312c5bb 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,12 +3,16 @@ ## 4.0.2 Released 2017-mm-dd +Announcements: + - Upgrades windshaft to [4.0.1](https://github.com/CartoDB/windshaft/releases/tag/4.0.1). + ## 4.0.1 Released 2017-10-18 Announcements: - Upgrades camshaft to [0.59.4](https://github.com/CartoDB/camshaft/releases/tag/0.59.4). + - Upgrades windshaft to [4.0.0](https://github.com/CartoDB/windshaft/releases/tag/4.0.0). - Split and move `req2params` method to multiple middlewares. - Use express error handler middleware to respond in case of something went wrong. - Use `res.locals` object to share info between middlewares and leave `req.params` as an object containing properties mapped to the named route params. From b6c405bf68d5b03b95ac3307518f18cf4a526d8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Oct 2017 19:28:40 +0100 Subject: [PATCH 050/167] Remove send method in base controller and remove BaseController class --- lib/cartodb/controllers/analyses.js | 19 ++++++++-------- lib/cartodb/controllers/base.js | 22 ------------------- lib/cartodb/controllers/layergroup.js | 24 ++++++++++++++------- lib/cartodb/controllers/map.js | 19 +++++++--------- lib/cartodb/controllers/named_maps.js | 22 +++++++------------ lib/cartodb/controllers/named_maps_admin.js | 17 +++++++-------- 6 files changed, 49 insertions(+), 74 deletions(-) delete mode 100644 lib/cartodb/controllers/base.js diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 7aabc11b..ffc0bea1 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -3,21 +3,13 @@ var assert = require('assert'); var dot = require('dot'); dot.templateSettings.strip = false; var PSQL = require('cartodb-psql'); - -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); - function AnalysesController(prepareContext) { - BaseController.call(this); this.prepareContext = prepareContext; } -util.inherits(AnalysesController, BaseController); - module.exports = AnalysesController; AnalysesController.prototype.register = function(app) { @@ -30,9 +22,16 @@ AnalysesController.prototype.register = function(app) { ); }; -AnalysesController.prototype.sendResponse = function(req, res, resource) { +AnalysesController.prototype.sendResponse = function(req, res, body) { res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - this.send(req, res, resource, 200); + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } }; AnalysesController.prototype.catalog = function (req, res, next) { diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js deleted file mode 100644 index b9c6ac1e..00000000 --- a/lib/cartodb/controllers/base.js +++ /dev/null @@ -1,22 +0,0 @@ -function BaseController() { -} - -module.exports = BaseController; - -BaseController.prototype.send = function(req, res, body, status, headers) { - if (headers) { - res.set(headers); - } - - res.status(status); - - if (!Buffer.isBuffer(body) && typeof body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(body); - } else { - res.json(body); - } - } else { - res.send(body); - } -}; diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index c5e4c53a..dc6f85ff 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -1,9 +1,6 @@ var assert = require('assert'); var step = require('step'); -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); @@ -30,8 +27,6 @@ var QueryTables = require('cartodb-query-tables'); */ function LayergroupController(prepareContext, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend) { - BaseController.call(this); - this.pgConnection = pgConnection; this.mapStore = mapStore; this.tileBackend = tileBackend; @@ -47,8 +42,6 @@ function LayergroupController(prepareContext, pgConnection, mapStore, tileBacken this.prepareContext = prepareContext; } -util.inherits(LayergroupController, BaseController); - module.exports = LayergroupController; LayergroupController.prototype.register = function(app) { @@ -422,7 +415,22 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h res.set('X-Cache-Channel', affectedTables.getCacheChannel()); self.surrogateKeysCache.tag(res, affectedTables); } - self.send(req, res, body, status, headers); + + if (headers) { + res.set(headers); + } + + res.status(status); + + if (!Buffer.isBuffer(body) && typeof body === 'object') { + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } + } else { + res.send(body); + } } ); diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f2e86d13..d1f67813 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -6,9 +6,6 @@ var QueryTables = require('cartodb-query-tables'); var ResourceLocator = require('../models/resource-locator'); -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); @@ -36,9 +33,6 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter, statsBackend) { - - BaseController.call(this); - this.pgConnection = pgConnection; this.templateMaps = templateMaps; this.mapBackend = mapBackend; @@ -54,11 +48,8 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m this.prepareContext = prepareContext; } -util.inherits(MapController, BaseController); - module.exports = MapController; - MapController.prototype.register = function(app) { app.get( app.base_url_mapconfig, @@ -181,7 +172,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { var datasource = context.datasource || Datasource.EmptyDatasource(); mapConfig = new MapConfig(requestMapConfig, datasource); self.mapBackend.createLayergroup( - mapConfig, + mapConfig, res.locals, new CreateLayergroupMapConfigProvider(mapConfig, res.locals.user, self.userLimitsApi, res.locals), this @@ -294,7 +285,13 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn res.set('X-Layergroup-Id', layergroup.layergroupid); self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); - self.send(req, res, layergroup, 200); + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); + } else { + res.json(layergroup); + } } } ); diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 759ae4b6..55790bbf 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -3,17 +3,12 @@ var assert = require('assert'); var _ = require('underscore'); var NamedMapsCacheEntry = require('../cache/model/named_maps_entry'); -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { - BaseController.call(this); - this.namedMapProviderCache = namedMapProviderCache; this.tileBackend = tileBackend; this.previewBackend = previewBackend; @@ -23,8 +18,6 @@ function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, this.prepareContext = prepareContext; } -util.inherits(NamedMapsController, BaseController); - module.exports = NamedMapsController; NamedMapsController.prototype.register = function(app) { @@ -46,7 +39,7 @@ NamedMapsController.prototype.register = function(app) { ); }; -NamedMapsController.prototype.sendResponse = function(req, res, resource, headers, namedMapProvider) { +NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) { this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName())); res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); @@ -79,7 +72,8 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header self.surrogateKeysCache.tag(res, result); } } - self.send(req, res, resource, 200); + res.status(200); + res.send(body); } ); }; @@ -143,7 +137,7 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { assert.ifError(err); namedMapProvider = _namedMapProvider; - + self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this); }, function prepareImageOptions(err) { @@ -192,10 +186,10 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { }; NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function ( - user, - req, - params, - namedMapProvider, + user, + req, + params, + namedMapProvider, callback ) { var self = this; diff --git a/lib/cartodb/controllers/named_maps_admin.js b/lib/cartodb/controllers/named_maps_admin.js index ae57ada3..0f54ddf3 100644 --- a/lib/cartodb/controllers/named_maps_admin.js +++ b/lib/cartodb/controllers/named_maps_admin.js @@ -2,9 +2,6 @@ var step = require('step'); var assert = require('assert'); var templateName = require('../backends/template_maps').templateName; -var util = require('util'); -var BaseController = require('./base'); - var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); @@ -16,14 +13,10 @@ var userMiddleware = require('../middleware/user'); * @constructor */ function NamedMapsAdminController(authApi, templateMaps) { - BaseController.call(this); - this.authApi = authApi; this.templateMaps = templateMaps; } -util.inherits(NamedMapsAdminController, BaseController); - module.exports = NamedMapsAdminController; NamedMapsAdminController.prototype.register = function (app) { @@ -206,12 +199,18 @@ NamedMapsAdminController.prototype.list = function(req, res, next) { }; function finishFn(controller, req, res, description, status, next) { - return function finish(err, response){ + return function finish(err, body){ if (err) { err.label = description; next(err); } else { - controller.send(req, res, response, status || 200); + res.status(status || 200); + + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } } }; } From fb3afaa6ab5fa5977c9e52f354de132a48f8fcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Oct 2017 19:29:19 +0100 Subject: [PATCH 051/167] Fix jshint max-complexity issue by using extract method --- lib/cartodb/controllers/map.js | 59 ++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index d1f67813..f1bbe55e 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -184,38 +184,49 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { }, function finish(err, layergroup) { if (err) { - if (Number.isFinite(err.layerIndex)) { - var error = new Error(err.message); - error.http_status = err.http_status; + err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - err = error; - } err.label = 'ANONYMOUS LAYERGROUP'; - next(err); + + return next(err); + } + + var analysesResults = context.analysesResults || []; + self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj()); + self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); + addContextMetadata(layergroup, mapConfig.obj(), context); + res.set('X-Layergroup-Id', layergroup.layergroupid); + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); } else { - var analysesResults = context.analysesResults || []; - self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj()); - self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); - addContextMetadata(layergroup, mapConfig.obj(), context); - res.set('X-Layergroup-Id', layergroup.layergroupid); - self.send(req, res, layergroup, 200); + res.json(layergroup); } } ); }; +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} + function addContextMetadata(layergroup, mapConfig, context) { if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { From 3f49743cd062265ca90fa43cb27bcda27b40e736 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 30 Oct 2017 19:30:03 +0100 Subject: [PATCH 052/167] Remove BaseController dependency and remove unused code --- test/acceptance/overviews_queries.js | 9 --------- test/acceptance/ported/attributes.js | 13 ------------- test/acceptance/ported/blend.js | 14 -------------- test/acceptance/ported/blend_filtering.js | 8 -------- test/acceptance/ported/blend_http_fallback.js | 7 ------- test/acceptance/ported/blend_http_timeout.js | 14 -------------- test/acceptance/ported/external_resources.js | 10 ---------- test/acceptance/ported/limits.js | 8 +------- test/acceptance/ported/multilayer.js | 11 ----------- test/acceptance/ported/multilayer_error_cases.js | 12 ------------ test/acceptance/ported/multilayer_interactivity.js | 12 ------------ test/acceptance/ported/raster.js | 13 ------------- test/acceptance/ported/regressions.js | 12 ------------ test/acceptance/ported/retina.js | 10 ---------- test/acceptance/ported/server.js | 10 ---------- test/acceptance/ported/server_gettile.js | 11 ----------- test/acceptance/ported/server_png8_format.js | 10 ---------- test/acceptance/ported/static_maps.js | 8 -------- test/acceptance/ported/torque.js | 11 ----------- test/acceptance/ported/torque_boundaries.js | 12 ------------ test/acceptance/ported/torque_png.js | 14 -------------- test/acceptance/ported/torque_zero_zero.js | 14 -------------- test/acceptance/ported/wrap.js | 14 -------------- 23 files changed, 1 insertion(+), 256 deletions(-) diff --git a/test/acceptance/overviews_queries.js b/test/acceptance/overviews_queries.js index 0434ebfc..ad7599a6 100644 --- a/test/acceptance/overviews_queries.js +++ b/test/acceptance/overviews_queries.js @@ -4,7 +4,6 @@ var assert = require('../support/assert'); var cartodbServer = require('../../lib/cartodb/server'); var ServerOptions = require('./ported/support/ported_server_options'); var testClient = require('./ported/support/test_client'); -var BaseController = require('../../lib/cartodb/controllers/base'); describe('overviews_queries', function() { @@ -13,15 +12,7 @@ describe('overviews_queries', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - after(function() { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/attributes.js b/test/acceptance/ported/attributes.js index 39f3f461..abd4e5ab 100644 --- a/test/acceptance/ported/attributes.js +++ b/test/acceptance/ported/attributes.js @@ -4,13 +4,10 @@ var assert = require('../../support/assert'); var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); - describe('attributes', function() { - var server = cartodbServer(PortedServerOptions); server.setMaxListeners(0); @@ -49,16 +46,6 @@ describe('attributes', function() { testHelper.deleteRedisKeys(keysToDelete, done); }); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - it("can only be fetched from layer having an attributes spec", function(done) { var expected_token; diff --git a/test/acceptance/ported/blend.js b/test/acceptance/ported/blend.js index 1e844d80..52273568 100644 --- a/test/acceptance/ported/blend.js +++ b/test/acceptance/ported/blend.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('blend png renderer', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var IMAGE_TOLERANCE_PER_MIL = 20; function plainTorqueMapConfig(plainColor) { diff --git a/test/acceptance/ported/blend_filtering.js b/test/acceptance/ported/blend_filtering.js index dc8cd470..2ada716e 100644 --- a/test/acceptance/ported/blend_filtering.js +++ b/test/acceptance/ported/blend_filtering.js @@ -5,20 +5,13 @@ var testClient = require('./support/test_client'); var fs = require('fs'); var http = require('http'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('blend layer filtering', function() { var IMG_TOLERANCE_PER_MIL = 20; var httpRendererResourcesServer; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - // Start a server to test external resources httpRendererResourcesServer = http.createServer( function(request, response) { var filename = __dirname + '/../../fixtures/http/light_nolabels-1-0-0.png'; @@ -32,7 +25,6 @@ describe('blend layer filtering', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; httpRendererResourcesServer.close(done); }); diff --git a/test/acceptance/ported/blend_http_fallback.js b/test/acceptance/ported/blend_http_fallback.js index 480dc5c0..f86d3dfc 100644 --- a/test/acceptance/ported/blend_http_fallback.js +++ b/test/acceptance/ported/blend_http_fallback.js @@ -5,19 +5,13 @@ var testClient = require('./support/test_client'); var fs = require('fs'); var http = require('http'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('blend http fallback', function() { var IMG_TOLERANCE_PER_MIL = 20; var httpRendererResourcesServer; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; // Start a server to test external resources httpRendererResourcesServer = http.createServer( function(request, response) { if (request.url.match(/^\/error404\//)) { @@ -39,7 +33,6 @@ describe('blend http fallback', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; httpRendererResourcesServer.close(done); }); diff --git a/test/acceptance/ported/blend_http_timeout.js b/test/acceptance/ported/blend_http_timeout.js index 341f72b3..54481fd0 100644 --- a/test/acceptance/ported/blend_http_timeout.js +++ b/test/acceptance/ported/blend_http_timeout.js @@ -6,21 +6,7 @@ var serverOptions = require('./support/ported_server_options'); var fs = require('fs'); var http = require('http'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe.skip('blend http client timeout', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var mapConfig = { version: '1.3.0', layers: [ diff --git a/test/acceptance/ported/external_resources.js b/test/acceptance/ported/external_resources.js index 11ad5176..0aae14c3 100644 --- a/test/acceptance/ported/external_resources.js +++ b/test/acceptance/ported/external_resources.js @@ -1,16 +1,12 @@ var testHelper = require('../../support/test_helper'); - var assert = require('../../support/assert'); var fs = require('fs'); var PortedServerOptions = require('./support/ported_server_options'); var http = require('http'); var testClient = require('./support/test_client'); - var nock = require('nock'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('external resources', function() { var res_serv; // resources server @@ -19,12 +15,8 @@ describe('external resources', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; - var req2paramsFn; before(function(done) { nock.enableNetConnect('127.0.0.1'); - - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; // Start a server to test external resources res_serv = http.createServer( function(request, response) { ++res_serv_status.numrequests; @@ -44,8 +36,6 @@ describe('external resources', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); // Close the resources server diff --git a/test/acceptance/ported/limits.js b/test/acceptance/ported/limits.js index c111c87c..0a8ee378 100644 --- a/test/acceptance/ported/limits.js +++ b/test/acceptance/ported/limits.js @@ -6,19 +6,14 @@ var assert = require('../../support/assert'); var testClient = require('./support/test_client'); var serverOptions = require('./support/ported_server_options'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe.skip('render limits', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; var limitsConfig; var onTileErrorStrategy; - var req2paramsFn; + before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; limitsConfig = serverOptions.renderer.mapnik.limits; serverOptions.renderer.mapnik.limits = { render: 50, @@ -31,7 +26,6 @@ describe.skip('render limits', function() { }); after(function() { - BaseController.prototype.req2params = req2paramsFn; serverOptions.renderer.mapnik.limits = limitsConfig; serverOptions.renderer.onTileErrorStrategy = onTileErrorStrategy; }); diff --git a/test/acceptance/ported/multilayer.js b/test/acceptance/ported/multilayer.js index 0592c147..0ad9f5de 100644 --- a/test/acceptance/ported/multilayer.js +++ b/test/acceptance/ported/multilayer.js @@ -8,7 +8,6 @@ var mapnik = require('windshaft').mapnik; var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); -var BaseController = require('../../../lib/cartodb/controllers/base'); describe('multilayer', function() { @@ -24,16 +23,6 @@ describe('multilayer', function() { assert.equal(res.headers['access-control-allow-origin'], '*'); } - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - // See https://github.com/Vizzuality/Windshaft/issues/70 it("post layergroup with encoding in content-type", function(done) { var layergroup = { diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index 3408baa2..92ce0b1f 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -7,23 +7,11 @@ var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); var TestClient = require('../../support/test-client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('multilayer error cases', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - // var client = null; afterEach(function(done) { if (this.client) { diff --git a/test/acceptance/ported/multilayer_interactivity.js b/test/acceptance/ported/multilayer_interactivity.js index 7d512670..4542f710 100644 --- a/test/acceptance/ported/multilayer_interactivity.js +++ b/test/acceptance/ported/multilayer_interactivity.js @@ -7,23 +7,11 @@ var getLayerTypeFn = require('windshaft').model.MapConfig.prototype.getType; var PortedServerOptions = require('./support/ported_server_options'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('multilayer interactivity and layers order', function() { var server = cartodbServer(PortedServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - function layerType(layer) { return layer.type || 'undefined'; } diff --git a/test/acceptance/ported/raster.js b/test/acceptance/ported/raster.js index b16dd56d..448661f2 100644 --- a/test/acceptance/ported/raster.js +++ b/test/acceptance/ported/raster.js @@ -4,8 +4,6 @@ var assert = require('../../support/assert'); var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); - -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('raster', function() { @@ -20,18 +18,7 @@ describe('raster', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - it("can render raster for valid mapconfig", function(done) { - var mapconfig = { version: '1.2.0', layers: [ diff --git a/test/acceptance/ported/regressions.js b/test/acceptance/ported/regressions.js index 2a2188c1..dd765b57 100644 --- a/test/acceptance/ported/regressions.js +++ b/test/acceptance/ported/regressions.js @@ -1,22 +1,10 @@ var testHelper = require('../../support/test_helper'); var assert = require('../../support/assert'); -var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('regressions', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/retina.js b/test/acceptance/ported/retina.js index 80c3fcc5..7469214b 100644 --- a/test/acceptance/ported/retina.js +++ b/test/acceptance/ported/retina.js @@ -5,7 +5,6 @@ var mapnik = require('windshaft').mapnik; var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('retina support', function() { @@ -15,15 +14,6 @@ describe('retina support', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); var keysToDelete; beforeEach(function(done) { diff --git a/test/acceptance/ported/server.js b/test/acceptance/ported/server.js index 650cf2c5..b18bae07 100644 --- a/test/acceptance/ported/server.js +++ b/test/acceptance/ported/server.js @@ -5,22 +5,12 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('server', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - after(function() { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/server_gettile.js b/test/acceptance/ported/server_gettile.js index d9a92661..f3c9ba81 100644 --- a/test/acceptance/ported/server_gettile.js +++ b/test/acceptance/ported/server_gettile.js @@ -7,8 +7,6 @@ var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); var testClient = require('./support/test_client'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('server_gettile', function() { var server = cartodbServer(ServerOptions); @@ -16,16 +14,7 @@ describe('server_gettile', function() { var IMAGE_EQUALS_TOLERANCE_PER_MIL = 25; - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); }); diff --git a/test/acceptance/ported/server_png8_format.js b/test/acceptance/ported/server_png8_format.js index 092e9ad7..18151330 100644 --- a/test/acceptance/ported/server_png8_format.js +++ b/test/acceptance/ported/server_png8_format.js @@ -6,13 +6,11 @@ var fs = require('fs'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); var IMAGE_EQUALS_TOLERANCE_PER_MIL = 85; describe('server_png8_format', function() { - var serverOptionsPng32 = ServerOptions; serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore); serverOptionsPng32.grainstore.mapnik_tile_format = 'png32'; @@ -25,13 +23,9 @@ describe('server_png8_format', function() { var serverPng8 = cartodbServer(serverOptionsPng8); serverPng8.setMaxListeners(0); - var layergroupId; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; var testPngFilesDir = __dirname + '/../../results/png'; fs.readdirSync(testPngFilesDir) .filter(function(fileName) { @@ -45,10 +39,6 @@ describe('server_png8_format', function() { done(); }); - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var keysToDelete; beforeEach(function() { keysToDelete = { diff --git a/test/acceptance/ported/static_maps.js b/test/acceptance/ported/static_maps.js index 2a04ca29..1002cdd5 100644 --- a/test/acceptance/ported/static_maps.js +++ b/test/acceptance/ported/static_maps.js @@ -5,9 +5,6 @@ var testClient = require('./support/test_client'); var http = require('http'); var fs = require('fs'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('static_maps', function() { var validUrlTemplate = 'http://127.0.0.1:8033/{s}/{z}/{x}/{y}.png'; @@ -15,11 +12,7 @@ describe('static_maps', function() { var httpRendererResourcesServer; - var req2paramsFn; before(function(done) { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - // Start a server to test external resources httpRendererResourcesServer = http.createServer( function(request, response) { var filename = __dirname + '/../../fixtures/http/basemap.png'; @@ -33,7 +26,6 @@ describe('static_maps', function() { }); after(function(done) { - BaseController.prototype.req2params = req2paramsFn; httpRendererResourcesServer.close(done); }); diff --git a/test/acceptance/ported/torque.js b/test/acceptance/ported/torque.js index c148a521..47ae2eee 100644 --- a/test/acceptance/ported/torque.js +++ b/test/acceptance/ported/torque.js @@ -6,7 +6,6 @@ var step = require('step'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('torque', function() { @@ -14,16 +13,6 @@ describe('torque', function() { var server = cartodbServer(ServerOptions); server.setMaxListeners(0); - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var keysToDelete; beforeEach(function() { keysToDelete = {}; diff --git a/test/acceptance/ported/torque_boundaries.js b/test/acceptance/ported/torque_boundaries.js index 1456c055..a984389d 100644 --- a/test/acceptance/ported/torque_boundaries.js +++ b/test/acceptance/ported/torque_boundaries.js @@ -4,21 +4,9 @@ var assert = require('../../support/assert'); var cartodbServer = require('../../../lib/cartodb/server'); var ServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token'); describe('torque boundary points', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = ServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var layergroupIdToDelete = null; beforeEach(function() { diff --git a/test/acceptance/ported/torque_png.js b/test/acceptance/ported/torque_png.js index 69eaadab..57b3ffa1 100644 --- a/test/acceptance/ported/torque_png.js +++ b/test/acceptance/ported/torque_png.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('torque png renderer', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - var IMAGE_TOLERANCE_PER_MIL = 20; var torquePngPointsMapConfig = { diff --git a/test/acceptance/ported/torque_zero_zero.js b/test/acceptance/ported/torque_zero_zero.js index b0b00081..fe6755ae 100644 --- a/test/acceptance/ported/torque_zero_zero.js +++ b/test/acceptance/ported/torque_zero_zero.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('torque tiles at 0,0 point', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - /* Tiles are represented as in: diff --git a/test/acceptance/ported/wrap.js b/test/acceptance/ported/wrap.js index 64a8e2f4..d79e2b56 100644 --- a/test/acceptance/ported/wrap.js +++ b/test/acceptance/ported/wrap.js @@ -3,21 +3,7 @@ require('../../support/test_helper'); var assert = require('../../support/assert'); var testClient = require('./support/test_client'); -var PortedServerOptions = require('./support/ported_server_options'); -var BaseController = require('../../../lib/cartodb/controllers/base'); - describe('wrap x coordinate', function() { - - var req2paramsFn; - before(function() { - req2paramsFn = BaseController.prototype.req2params; - BaseController.prototype.req2params = PortedServerOptions.req2params; - }); - - after(function() { - BaseController.prototype.req2params = req2paramsFn; - }); - describe('renders correct tile', function() { var IMG_TOLERANCE_PER_MIL = 20; From 235f5e456687bcbf0db4d6de3310b83ca72a4ad2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 11:38:18 +0100 Subject: [PATCH 053/167] Extract cache channel to its own method --- lib/cartodb/controllers/layergroup.js | 71 ++++++++++++++------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index dc6f85ff..d9551c9c 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -385,8 +385,6 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo }; LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) { - var self = this; - req.profiler.done('res'); res.set('Cache-Control', 'public,max-age=31536000'); @@ -401,43 +399,48 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h } res.set('Last-Modified', lastUpdated.toUTCString()); - var dbName = res.locals.dbname; - step( - function getAffectedTables() { - self.getAffectedTables(res.locals.user, dbName, res.locals.token, this); - }, - function sendResponse(err, affectedTables) { - req.profiler.done('affectedTables'); - if (err) { - global.logger.warn('ERROR generating cache channel: ' + err); - } - if (!!affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - self.surrogateKeysCache.tag(res, affectedTables); - } - - if (headers) { - res.set(headers); - } - - res.status(status); - - if (!Buffer.isBuffer(body) && typeof body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(body); - } else { - res.json(body); - } - } else { - res.send(body); - } + this.setCacheChannel(req, res, (err) => { + if (err) { + global.logger.warn('ERROR generating cache channel: ' + err); } - ); + if (headers) { + res.set(headers); + } + + res.status(status); + + if (!Buffer.isBuffer(body) && typeof body === 'object') { + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } + } else { + res.send(body); + } + }); +}; + +LayergroupController.prototype.setCacheChannel = function(req, res, callback) { + const { dbname, user, token } = res.locals; + + this.getAffectedTables(user, dbname, token, (err, affectedTables) => { + req.profiler.done('affectedTables'); + if (err) { + return callback(err); + } + + if (!!affectedTables) { + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + this.surrogateKeysCache.tag(res, affectedTables); + } + + callback(); + }); }; LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { - if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId)); } From 3f2ef63976decb9e828abb23efc296288f75f2b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 11:38:54 +0100 Subject: [PATCH 054/167] Extract cache channel to its own method --- lib/cartodb/controllers/named_maps.js | 85 +++++++++++++++------------ 1 file changed, 49 insertions(+), 36 deletions(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 55790bbf..ffea03a2 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -39,43 +39,56 @@ NamedMapsController.prototype.register = function(app) { ); }; -NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) { +NamedMapsController.prototype.sendResponse = function(req, res, body, headers) { + const { namedMapProvider } = res.locals; + this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName())); res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - var self = this; - - step( - function getAffectedTablesAndLastUpdatedTime() { - namedMapProvider.getAffectedTablesAndLastUpdatedTime(this); - }, - function sendResponse(err, result) { - req.profiler.done('affectedTables'); - if (err) { - global.logger.log('ERROR generating cache channel: ' + err); - } - if (!result || !!result.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - - var lastModifiedDate; - if (Number.isFinite(result.lastUpdatedTime)) { - lastModifiedDate = new Date(result.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); - } - res.set('Last-Modified', lastModifiedDate.toUTCString()); - - res.set('X-Cache-Channel', result.getCacheChannel()); - if (result.tables.length > 0) { - self.surrogateKeysCache.tag(res, result); - } - } - res.status(200); - res.send(body); + this.setCacheChannel(req, res, (err) => { + if (err) { + global.logger.log('ERROR generating cache channel: ' + err); } - ); + + res.status(200); + res.send(body); + }); +}; + +NamedMapsController.prototype.setCacheChannel = function(req, res, callback) { + const { namedMapProvider } = res.locals; + + this.getAffectedTables(namedMapProvider, (err, result) => { + + req.profiler.done('affectedTables'); + if (err) { + global.logger.log('ERROR generating cache channel: ' + err); + } + if (!result || !!result.tables) { + // we increase cache control as we can invalidate it + res.set('Cache-Control', 'public,max-age=31536000'); + + var lastModifiedDate; + if (Number.isFinite(result.lastUpdatedTime)) { + lastModifiedDate = new Date(result.getLastUpdatedAt()); + } else { + lastModifiedDate = new Date(); + } + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + res.set('X-Cache-Channel', result.getCacheChannel()); + if (result.tables.length > 0) { + this.surrogateKeysCache.tag(res, result); + } + } + + callback(); + }); +}; + +NamedMapsController.prototype.getAffectedTables = function(namedMapProvider, callback) { + namedMapProvider.getAffectedTablesAndLastUpdatedTime(callback); }; NamedMapsController.prototype.tile = function(req, res, next) { @@ -97,7 +110,7 @@ NamedMapsController.prototype.tile = function(req, res, next) { }, function getTile(err, _namedMapProvider) { assert.ifError(err); - namedMapProvider = _namedMapProvider; + res.locals.namedMapProvider = namedMapProvider = _namedMapProvider; self.tileBackend.getTile(namedMapProvider, req.params, this); }, function handleImage(err, tile, headers, stats) { @@ -106,7 +119,7 @@ NamedMapsController.prototype.tile = function(req, res, next) { err.label = 'NAMED_MAP_TILE'; next(err); } else { - self.sendResponse(req, res, tile, headers, namedMapProvider); + self.sendResponse(req, res, tile, headers); } } ); @@ -136,7 +149,7 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { function prepareLayerVisibility(err, _namedMapProvider) { assert.ifError(err); - namedMapProvider = _namedMapProvider; + res.locals.namedMapProvider = namedMapProvider = _namedMapProvider; self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this); }, @@ -179,7 +192,7 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { err.label = 'STATIC_VIZ_MAP'; next(err); } else { - self.sendResponse(req, res, image, headers, namedMapProvider); + self.sendResponse(req, res, image, headers); } } ); From 467bee4c911fddb5e780bffc560916fc49469c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 13:13:20 +0100 Subject: [PATCH 055/167] Split afterLayergroupCreate method in multiple "pre-middlewares" --- lib/cartodb/controllers/map.js | 157 +++++++++++++++++++++++---------- 1 file changed, 108 insertions(+), 49 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f1bbe55e..4603a7b3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -312,8 +312,6 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) { var self = this; - var username = res.locals.user; - var tasksleft = 2; // redis key and affectedTables var errors = []; @@ -335,6 +333,7 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) { // Don't wait for the mapview count increment to // take place before proceeding. Error will be logged // asynchronously + var username = res.locals.user; this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) { req.profiler.done('incMapviewCount'); if ( err ) { @@ -343,6 +342,40 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) { done(); }); + res.locals.layergroup = layergroup; + res.locals.mapconfig = mapconfig; + res.locals.analysesResults = analysesResults; + + step( + function getAffectedTables () { + self.getAffectedTables(req, res, this); + }, + function setCacheChannel (err) { + assert.ifError(err); + self.setCacheChannel(req, res, this); + }, + function setLastUpdatedTime (err) { + assert.ifError(err); + self.setLastUpdatedTimeToLayergroup(req, res, this); + }, + function setCacheControl (err) { + assert.ifError(err); + self.setCacheControl(req, res, this); + }, + function setLayerStats (err) { + assert.ifError(err); + self.setLayerStats(req, res, this); + }, + function finish(err) { + done(err); + } + ); +}; + +MapController.prototype.getAffectedTables = function (req, res, callback) { + const self = this; + const { dbname, layergroup, user, mapconfig } = res.locals; + var sql = []; mapconfig.getLayers().forEach(function(layer) { sql.push(layer.options.sql); @@ -353,65 +386,91 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) { } }); - var dbName = res.locals.dbname; - var layergroupId = layergroup.layergroupid; - var dbConnection; - step( function getPgConnection() { - self.pgConnection.getConnection(username, this); + self.pgConnection.getConnection(user, this); }, function getAffectedTablesAndLastUpdatedTime(err, connection) { assert.ifError(err); - dbConnection = connection; - QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this); + QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this); }, - function handleAffectedTablesAndLastUpdatedTime(err, result) { - req.profiler.done('queryTablesAndLastUpdated'); - assert.ifError(err); - // feed affected tables cache so it can be reused from, for instance, layergroup controller - self.layergroupAffectedTables.set(dbName, layergroupId, result); - - var lastUpdateTime = result.getLastUpdatedAt(); - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; - - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); - - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - res.set('Last-Modified', (new Date()).toUTCString()); - res.set('X-Cache-Channel', result.getCacheChannel()); - if (result.tables && result.tables.length > 0) { - self.surrogateKeysCache.tag(res, result); - } + function handleAffectedTablesAndLastUpdatedTime(err, affectedTables) { + if (err) { + return callback(err); } - return null; - }, - function fetchLayersStats(err) { - assert.ifError(err); - var next = this; - self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) { - if (err) { - return next(err); - } - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } - return next(); - }); - }, - function finish(err) { - done(err); + // feed affected tables cache so it can be reused from, for instance, layergroup controller + self.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + + res.locals.affectedTables = affectedTables; + + callback(); } ); }; +MapController.prototype.setCacheChannel = function (req, res, callback) { + const self = this; + const { affectedTables } = res.locals; + + if (req.method === 'GET') { + res.set('Last-Modified', (new Date()).toUTCString()); + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + if (affectedTables.tables && affectedTables.tables.length > 0) { + self.surrogateKeysCache.tag(res, affectedTables); + } + } + + callback(); +}; + +MapController.prototype.setLastUpdatedTimeToLayergroup = function (req, res, callback) { + const { affectedTables, layergroup, analysesResults } = res.locals; + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + callback(); +}; + +MapController.prototype.setCacheControl = function (req, res, callback) { + if (req.method === 'GET') { + var ttl = global.environment.varnish.layergroupTtl || 86400; + res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); + } + + callback(); +}; + +MapController.prototype.setLayerStats = function (req, res, callback) { + const { user, mapconfig, layergroup } = res.locals; + + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return callback(err); + } + + this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + if (err) { + return callback(err); + } + + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + callback(); + }); + }); +}; + function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; From 48172d4dc1475cd67ba7c52824b2225cdd11f27f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 13:36:17 +0100 Subject: [PATCH 056/167] make afterLayergroupCreate to follow the middleware signature --- lib/cartodb/controllers/map.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4603a7b3..51d38d99 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -180,7 +180,11 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, context.analysesResults, this); + res.locals.mapconfig = mapConfig; + res.locals.analysesResults = context.analysesResults; + res.locals.layergroup = layergroup; + + self.afterLayergroupCreate(req, res, this); }, function finish(err, layergroup) { if (err) { @@ -276,9 +280,12 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - self.afterLayergroupCreate(req, res, mapConfig, layergroup, - mapConfigProvider.analysesResults, - this); + + res.locals.mapconfig = mapConfig; + res.locals.analysesResults = mapConfigProvider.analysesResults; + res.locals.layergroup = layergroup; + + self.afterLayergroupCreate(req, res, this); }, function finishTemplateInstantiation(err, layergroup) { if (err) { @@ -308,9 +315,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreate = -function(req, res, mapconfig, layergroup, analysesResults, callback) { +MapController.prototype.afterLayergroupCreate = function (req, res, callback) { var self = this; + const { layergroup, mapconfig, user } = res.locals; var tasksleft = 2; // redis key and affectedTables var errors = []; @@ -333,19 +340,14 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) { // Don't wait for the mapview count increment to // take place before proceeding. Error will be logged // asynchronously - var username = res.locals.user; - this.metadataBackend.incMapviewCount(username, mapconfig.obj().stat_tag, function(err) { + this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, function(err) { req.profiler.done('incMapviewCount'); if ( err ) { - global.logger.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); + global.logger.log("ERROR: failed to increment mapview count for user '" + user + "': " + err); } done(); }); - res.locals.layergroup = layergroup; - res.locals.mapconfig = mapconfig; - res.locals.analysesResults = analysesResults; - step( function getAffectedTables () { self.getAffectedTables(req, res, this); From 3cf4a8f70b0966d47046cda0c21d1af3f90c2bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 13:46:03 +0100 Subject: [PATCH 057/167] Extract layergroup data augmentation to its own "middleware" --- lib/cartodb/controllers/map.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 51d38d99..6ebb3010 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -332,11 +332,6 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { } }; - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - // Don't wait for the mapview count increment to // take place before proceeding. Error will be logged // asynchronously @@ -349,7 +344,11 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { }); step( - function getAffectedTables () { + function () { + self.augmentLayergroupData(req, res, this); + }, + function getAffectedTables (err) { + assert.ifError(err); self.getAffectedTables(req, res, this); }, function setCacheChannel (err) { @@ -374,6 +373,17 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { ); }; +MapController.prototype.augmentLayergroupData = function (req, res, callback) { + const { layergroup } = res.locals; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + callback(); +} + MapController.prototype.getAffectedTables = function (req, res, callback) { const self = this; const { dbname, layergroup, user, mapconfig } = res.locals; From 34e219353cff41fdab306bf1d27515877ea30c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 14:47:29 +0100 Subject: [PATCH 058/167] do not pass layergroup since it's already available in res.locals --- lib/cartodb/controllers/map.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 6ebb3010..b2663299 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -317,7 +317,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn MapController.prototype.afterLayergroupCreate = function (req, res, callback) { var self = this; - const { layergroup, mapconfig, user } = res.locals; + const { mapconfig, user } = res.locals; var tasksleft = 2; // redis key and affectedTables var errors = []; @@ -328,7 +328,7 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { } if ( ! --tasksleft ) { err = errors.length ? new Error(errors.join('\n')) : null; - callback(err, layergroup); + callback(err); } }; @@ -340,7 +340,8 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { if ( err ) { global.logger.log("ERROR: failed to increment mapview count for user '" + user + "': " + err); } - done(); + + done(err); }); step( From fab87e21688ddce977d5584d8d99eace6a75afde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 15:47:59 +0100 Subject: [PATCH 059/167] Get layergroup from locals. It's not provided by previous middleware anymore --- lib/cartodb/controllers/map.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b2663299..0c860f05 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -186,7 +186,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { self.afterLayergroupCreate(req, res, this); }, - function finish(err, layergroup) { + function finish(err) { if (err) { err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; @@ -195,6 +195,8 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { return next(err); } + const { layergroup } = res.locals; + var analysesResults = context.analysesResults || []; self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj()); self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); @@ -287,11 +289,13 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn self.afterLayergroupCreate(req, res, this); }, - function finishTemplateInstantiation(err, layergroup) { + function finishTemplateInstantiation(err) { if (err) { err.label = 'NAMED MAP LAYERGROUP'; next(err); } else { + const { layergroup } = res.locals; + var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; From 12822c4341bd95b8641c9eee27c5046d2da69f39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 15:49:10 +0100 Subject: [PATCH 060/167] Follow node.js convention regarding early returns --- lib/cartodb/controllers/map.js | 40 +++++++++++++++++----------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 0c860f05..94418b77 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -293,27 +293,27 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn if (err) { err.label = 'NAMED MAP LAYERGROUP'; next(err); + } + + const { layergroup } = res.locals; + + var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); + layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; + + var _mapConfig = mapConfig.obj(); + self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig); + self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); + addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context); + + res.set('X-Layergroup-Id', layergroup.layergroupid); + self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); } else { - const { layergroup } = res.locals; - - var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); - layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; - - var _mapConfig = mapConfig.obj(); - self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig); - self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); - addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context); - - res.set('X-Layergroup-Id', layergroup.layergroupid); - self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } + res.json(layergroup); } } ); From d1093686a3a1d9f0bcce35f1205af14ba60a19bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 15:51:42 +0100 Subject: [PATCH 061/167] Avoid to hold info in local variables --- lib/cartodb/controllers/map.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 94418b77..5f4f6e45 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -300,10 +300,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; - var _mapConfig = mapConfig.obj(); - self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig); + self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj()); self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); - addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context); + addContextMetadata(layergroup, mapConfig.obj(), mapConfigProvider.context); res.set('X-Layergroup-Id', layergroup.layergroupid); self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); From a4b2044e10b475a0f86eabdc9aad41acab0ad106 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 15:58:41 +0100 Subject: [PATCH 062/167] missing early return --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 5f4f6e45..26653ad3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -292,7 +292,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn function finishTemplateInstantiation(err) { if (err) { err.label = 'NAMED MAP LAYERGROUP'; - next(err); + return next(err); } const { layergroup } = res.locals; From 5f7d5f6ec8993c4481bcda3ce7aa6a62ce1f60f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 16:01:18 +0100 Subject: [PATCH 063/167] Get analyses results from res.locals --- lib/cartodb/controllers/map.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 26653ad3..80f99d8e 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -189,15 +189,12 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { function finish(err) { if (err) { err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - err.label = 'ANONYMOUS LAYERGROUP'; - return next(err); } - const { layergroup } = res.locals; + const { layergroup, analysesResults = [] } = res.locals; - var analysesResults = context.analysesResults || []; self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj()); self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); addContextMetadata(layergroup, mapConfig.obj(), context); @@ -295,13 +292,13 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn return next(err); } - const { layergroup } = res.locals; + const { layergroup, analysesResults = [] } = res.locals; var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj()); - self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); + self.addAnalysesMetadata(cdbuser, layergroup, analysesResults); addContextMetadata(layergroup, mapConfig.obj(), mapConfigProvider.context); res.set('X-Layergroup-Id', layergroup.layergroupid); From 39eb0f7bec5bc745b8b34002174e3e0bee86d266 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 16:58:00 +0100 Subject: [PATCH 064/167] Avoid regression and update comment --- lib/cartodb/controllers/map.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 80f99d8e..05363a06 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -332,16 +332,15 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { } }; - // Don't wait for the mapview count increment to - // take place before proceeding. Error will be logged - // asynchronously + // Perform mapview count increment in parallel along the rest of after-layergroup-create + // tasks. Error won't blow up, just logged. this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, function(err) { req.profiler.done('incMapviewCount'); if ( err ) { global.logger.log("ERROR: failed to increment mapview count for user '" + user + "': " + err); } - done(err); + done(); }); step( @@ -383,7 +382,7 @@ MapController.prototype.augmentLayergroupData = function (req, res, callback) { _.extend(layergroup, global.environment.serverMetadata); callback(); -} +}; MapController.prototype.getAffectedTables = function (req, res, callback) { const self = this; From bb02494e02b471bb9c5138e986712eb3ae16a048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 17:09:42 +0100 Subject: [PATCH 065/167] Do not perform "increment map view count" in parallel --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 05363a06..4f4213f4 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -333,7 +333,7 @@ MapController.prototype.afterLayergroupCreate = function (req, res, callback) { }; // Perform mapview count increment in parallel along the rest of after-layergroup-create - // tasks. Error won't blow up, just logged. + // tasks. Error won't blow up, just be logged. this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, function(err) { req.profiler.done('incMapviewCount'); if ( err ) { From d85a5d83b79253bdc58ccd15dadee342b9922bac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 17:29:55 +0100 Subject: [PATCH 066/167] Make afterLayergroupCreate function as a 'middleware' builder --- lib/cartodb/controllers/map.js | 102 ++++++++++++++++----------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4f4213f4..de0561cf 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -184,7 +184,9 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { res.locals.analysesResults = context.analysesResults; res.locals.layergroup = layergroup; - self.afterLayergroupCreate(req, res, this); + const afterLayergroupCreate = self.afterLayergroupCreateBuilder(); + + afterLayergroupCreate(req, res, this); }, function finish(err) { if (err) { @@ -284,7 +286,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn res.locals.analysesResults = mapConfigProvider.analysesResults; res.locals.layergroup = layergroup; - self.afterLayergroupCreate(req, res, this); + const afterLayergroupCreate = self.afterLayergroupCreateBuilder(); + + afterLayergroupCreate(req, res, this); }, function finishTemplateInstantiation(err) { if (err) { @@ -315,62 +319,58 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreate = function (req, res, callback) { +MapController.prototype.afterLayergroupCreateBuilder = function () { var self = this; + + return function (req, res, callback) { + step( + function incrementMapViewCount () { + self.incrementMapViewCount(req, res, this); + }, + function augmentLayergroupData (err) { + assert.ifError(err); + self.augmentLayergroupData(req, res, this); + }, + function getAffectedTables (err) { + assert.ifError(err); + self.getAffectedTables(req, res, this); + }, + function setCacheChannel (err) { + assert.ifError(err); + self.setCacheChannel(req, res, this); + }, + function setLastUpdatedTime (err) { + assert.ifError(err); + self.setLastUpdatedTimeToLayergroup(req, res, this); + }, + function setCacheControl (err) { + assert.ifError(err); + self.setCacheControl(req, res, this); + }, + function setLayerStats (err) { + assert.ifError(err); + self.setLayerStats(req, res, this); + }, + function finish(err) { + callback(err); + } + ); + }; +}; + +MapController.prototype.incrementMapViewCount = function (req, res, callback) { const { mapconfig, user } = res.locals; - var tasksleft = 2; // redis key and affectedTables - var errors = []; - - var done = function(err) { - if ( err ) { - errors.push('' + err); - } - if ( ! --tasksleft ) { - err = errors.length ? new Error(errors.join('\n')) : null; - callback(err); - } - }; - - // Perform mapview count increment in parallel along the rest of after-layergroup-create - // tasks. Error won't blow up, just be logged. - this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, function(err) { + // Error won't blow up, just be logged. + this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { req.profiler.done('incMapviewCount'); - if ( err ) { - global.logger.log("ERROR: failed to increment mapview count for user '" + user + "': " + err); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); } - done(); + callback(); }); - - step( - function () { - self.augmentLayergroupData(req, res, this); - }, - function getAffectedTables (err) { - assert.ifError(err); - self.getAffectedTables(req, res, this); - }, - function setCacheChannel (err) { - assert.ifError(err); - self.setCacheChannel(req, res, this); - }, - function setLastUpdatedTime (err) { - assert.ifError(err); - self.setLastUpdatedTimeToLayergroup(req, res, this); - }, - function setCacheControl (err) { - assert.ifError(err); - self.setCacheControl(req, res, this); - }, - function setLayerStats (err) { - assert.ifError(err); - self.setLayerStats(req, res, this); - }, - function finish(err) { - done(err); - } - ); }; MapController.prototype.augmentLayergroupData = function (req, res, callback) { From 99fa66c02691dea9517d7b26ae0ae5edc10869b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 18:02:23 +0100 Subject: [PATCH 067/167] Extract hash template for layergroup id and dataviews/widgets to a middlewares --- lib/cartodb/controllers/map.js | 45 ++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index de0561cf..41575629 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -197,7 +197,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { const { layergroup, analysesResults = [] } = res.locals; - self.addDataviewsAndWidgetsUrls(res.locals.user, layergroup, mapConfig.obj()); self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); addContextMetadata(layergroup, mapConfig.obj(), context); res.set('X-Layergroup-Id', layergroup.layergroupid); @@ -285,8 +284,11 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn res.locals.mapconfig = mapConfig; res.locals.analysesResults = mapConfigProvider.analysesResults; res.locals.layergroup = layergroup; + res.locals.template = mapConfigProvider.template; - const afterLayergroupCreate = self.afterLayergroupCreateBuilder(); + const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ + useTemplateHash: true + }); afterLayergroupCreate(req, res, this); }, @@ -298,10 +300,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn const { layergroup, analysesResults = [] } = res.locals; - var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); - layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; - - self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, mapConfig.obj()); self.addAnalysesMetadata(cdbuser, layergroup, analysesResults); addContextMetadata(layergroup, mapConfig.obj(), mapConfigProvider.context); @@ -319,8 +317,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); }; -MapController.prototype.afterLayergroupCreateBuilder = function () { - var self = this; +MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { + const self = this; + const { useTemplateHash = false } = options; return function (req, res, callback) { step( @@ -351,6 +350,19 @@ MapController.prototype.afterLayergroupCreateBuilder = function () { assert.ifError(err); self.setLayerStats(req, res, this); }, + function augmentLayergroupIdWithTemplateHash (err) { + assert.ifError(err); + + if (!useTemplateHash) { + return null; + } + + self.augmentLayergroupIdWithTemplateHash(req, res, this); + }, + function (err) { + assert.ifError(err); + self.setDataviewsAndWidgetsUrlsToLayergroupMetadata(req, res, this); + }, function finish(err) { callback(err); } @@ -483,6 +495,23 @@ MapController.prototype.setLayerStats = function (req, res, callback) { }); }; +MapController.prototype.augmentLayergroupIdWithTemplateHash = function (req, res, callback) { + const { layergroup, user, template } = res.locals; + + var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + + callback(); +}; + +MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function (req, res, callback) { + const { layergroup, user, mapconfig } = res.locals; + + this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + + callback(); +} + function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; From cdc39c8cae5d3862741c7c7396eb2f98d25169e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 18:25:17 +0100 Subject: [PATCH 068/167] Extract addAnalysesMetadata functionallity to its own middleware --- lib/cartodb/controllers/map.js | 36 +++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 41575629..0163160a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -184,7 +184,9 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { res.locals.analysesResults = context.analysesResults; res.locals.layergroup = layergroup; - const afterLayergroupCreate = self.afterLayergroupCreateBuilder(); + const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ + includeQuery: true + }); afterLayergroupCreate(req, res, this); }, @@ -195,9 +197,8 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { return next(err); } - const { layergroup, analysesResults = [] } = res.locals; + const { layergroup } = res.locals; - self.addAnalysesMetadata(res.locals.user, layergroup, analysesResults, true); addContextMetadata(layergroup, mapConfig.obj(), context); res.set('X-Layergroup-Id', layergroup.layergroupid); @@ -298,9 +299,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn return next(err); } - const { layergroup, analysesResults = [] } = res.locals; + const { layergroup } = res.locals; - self.addAnalysesMetadata(cdbuser, layergroup, analysesResults); addContextMetadata(layergroup, mapConfig.obj(), mapConfigProvider.context); res.set('X-Layergroup-Id', layergroup.layergroupid); @@ -319,7 +319,10 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { const self = this; - const { useTemplateHash = false } = options; + const { + useTemplateHash = false, + includeQuery = false + } = options; return function (req, res, callback) { step( @@ -363,6 +366,13 @@ MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { assert.ifError(err); self.setDataviewsAndWidgetsUrlsToLayergroupMetadata(req, res, this); }, + function setAnalysesMetadataToLayergroup(err) { + assert.ifError(err); + + const setAnalysesMetadataToLayergroup = self.setAnalysesMetadataToLayergroupBuilder(includeQuery); + + setAnalysesMetadataToLayergroup(req, res, this); + }, function finish(err) { callback(err); } @@ -510,7 +520,19 @@ MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = functio this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); callback(); -} +}; + +MapController.prototype.setAnalysesMetadataToLayergroupBuilder = function (includeQuery) { + const self = this; + + return function setAnalysesMetadataToLayergroup (req, res, callback) { + const { layergroup, user, analysesResults = [] } = res.locals; + + self.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + + callback(); + }; +}; function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { From da2228088e16103323cb2f0cbdee005cf4c94f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 18:42:11 +0100 Subject: [PATCH 069/167] Extract context metadata (turbo-carto) functionallity to its own middleware --- lib/cartodb/controllers/map.js | 39 ++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 0163160a..b6348f3f 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -183,6 +183,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { res.locals.mapconfig = mapConfig; res.locals.analysesResults = context.analysesResults; res.locals.layergroup = layergroup; + res.locals.context = context; const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ includeQuery: true @@ -199,7 +200,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { const { layergroup } = res.locals; - addContextMetadata(layergroup, mapConfig.obj(), context); res.set('X-Layergroup-Id', layergroup.layergroupid); res.status(200); @@ -232,17 +232,6 @@ function populateError(err, mapConfig) { return error; } -function addContextMetadata(layergroup, mapConfig, context) { - if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { - layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { - if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { - layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; - } - return layer; - }); - } -} - MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn, next) { var self = this; @@ -286,6 +275,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn res.locals.analysesResults = mapConfigProvider.analysesResults; res.locals.layergroup = layergroup; res.locals.template = mapConfigProvider.template; + res.locals.context = mapConfigProvider.context; const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ useTemplateHash: true @@ -301,8 +291,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn const { layergroup } = res.locals; - addContextMetadata(layergroup, mapConfig.obj(), mapConfigProvider.context); - res.set('X-Layergroup-Id', layergroup.layergroupid); self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); @@ -373,6 +361,10 @@ MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { setAnalysesMetadataToLayergroup(req, res, this); }, + function setTurboCartoMetadataToLayergroup (err) { + assert.ifError(err); + self.setTurboCartoMetadataToLayergroup(req, res, this); + }, function finish(err) { callback(err); } @@ -534,6 +526,25 @@ MapController.prototype.setAnalysesMetadataToLayergroupBuilder = function (inclu }; }; +MapController.prototype.setTurboCartoMetadataToLayergroup = function (req, res, callback) { + const { layergroup, mapconfig, context } = res.locals; + + addContextMetadata(layergroup, mapconfig.obj(), context); + + callback(); +}; + +function addContextMetadata(layergroup, mapConfig, context) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { + layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; + } + return layer; + }); + } +} + function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; From e06f8fe25ec137596bc166e2865f70684f1aa347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 18:54:32 +0100 Subject: [PATCH 070/167] Set layergroup-id header in the right middleware --- lib/cartodb/controllers/map.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b6348f3f..1859dd07 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -200,8 +200,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { const { layergroup } = res.locals; - res.set('X-Layergroup-Id', layergroup.layergroupid); - res.status(200); if (req.query && req.query.callback) { @@ -291,7 +289,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn const { layergroup } = res.locals; - res.set('X-Layergroup-Id', layergroup.layergroupid); self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); res.status(200); @@ -341,14 +338,12 @@ MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { assert.ifError(err); self.setLayerStats(req, res, this); }, - function augmentLayergroupIdWithTemplateHash (err) { + function setLayergroupIdHeader (err) { assert.ifError(err); - if (!useTemplateHash) { - return null; - } + const setLayergroupIdHeader = self.setLayergroupIdHeaderBuilder(useTemplateHash); - self.augmentLayergroupIdWithTemplateHash(req, res, this); + setLayergroupIdHeader(req, res, this); }, function (err) { assert.ifError(err); @@ -497,13 +492,20 @@ MapController.prototype.setLayerStats = function (req, res, callback) { }); }; -MapController.prototype.augmentLayergroupIdWithTemplateHash = function (req, res, callback) { - const { layergroup, user, template } = res.locals; +MapController.prototype.setLayergroupIdHeaderBuilder = function (useTemplateHash) { + const self = this; + return function setLayergroupIdHeader(req, res, callback) { + const { layergroup, user, template } = res.locals; - var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8); - layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + if (useTemplateHash) { + var templateHash = self.templateMaps.fingerPrint(template).substring(0, 8); + layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + } - callback(); + res.set('X-Layergroup-Id', layergroup.layergroupid); + + callback(); + }; }; MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function (req, res, callback) { From e0ffeb0adc0c5ea6b248667b3c41b0dfed2b8af9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 19:49:12 +0100 Subject: [PATCH 071/167] extract surrogate key functionality to its own middleware --- lib/cartodb/controllers/map.js | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1859dd07..ac874e59 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -274,6 +274,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn res.locals.layergroup = layergroup; res.locals.template = mapConfigProvider.template; res.locals.context = mapConfigProvider.context; + res.locals.templateName = mapConfigProvider.getTemplateName(); const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ useTemplateHash: true @@ -289,8 +290,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn const { layergroup } = res.locals; - self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); - res.status(200); if (req.query && req.query.callback) { @@ -360,6 +359,10 @@ MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { assert.ifError(err); self.setTurboCartoMetadataToLayergroup(req, res, this); }, + function setSurrogateKeyHeader (err) { + assert.ifError(err); + self.setSurrogateKeyHeader(req, res, this); + }, function finish(err) { callback(err); } @@ -431,15 +434,11 @@ MapController.prototype.getAffectedTables = function (req, res, callback) { }; MapController.prototype.setCacheChannel = function (req, res, callback) { - const self = this; const { affectedTables } = res.locals; if (req.method === 'GET') { res.set('Last-Modified', (new Date()).toUTCString()); res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - if (affectedTables.tables && affectedTables.tables.length > 0) { - self.surrogateKeysCache.tag(res, affectedTables); - } } callback(); @@ -547,6 +546,20 @@ function addContextMetadata(layergroup, mapConfig, context) { } } +MapController.prototype.setSurrogateKeyHeader = function (req, res, callback) { + const { affectedTables, user, templateName } = res.locals; + + if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { + this.surrogateKeysCache.tag(res, affectedTables); + } + + if (templateName) { + this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); + } + + callback(); +}; + function getLastUpdatedTime(analysesResults, lastUpdateTime) { if (!Array.isArray(analysesResults)) { return lastUpdateTime; From 3d15551cb5831a17afa6372bc388fed803e0bba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 20:10:37 +0100 Subject: [PATCH 072/167] Minor style umprovements --- lib/cartodb/controllers/map.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ac874e59..054a901e 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -180,6 +180,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); + res.locals.mapconfig = mapConfig; res.locals.analysesResults = context.analysesResults; res.locals.layergroup = layergroup; @@ -273,8 +274,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn res.locals.analysesResults = mapConfigProvider.analysesResults; res.locals.layergroup = layergroup; res.locals.template = mapConfigProvider.template; - res.locals.context = mapConfigProvider.context; res.locals.templateName = mapConfigProvider.getTemplateName(); + res.locals.context = mapConfigProvider.context; const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ useTemplateHash: true From 6bbaeaa2860337782e6be604fc2d00c02c62da86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 31 Oct 2017 20:49:26 +0100 Subject: [PATCH 073/167] Create a custom error middleware to augment error info --- lib/cartodb/controllers/map.js | 85 ++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 30 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 054a901e..82a7732b 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -56,28 +56,42 @@ MapController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.createGet.bind(this) + this.createGet.bind(this), + mapErrorMiddleware({ + label: 'ANONYMOUS LAYERGROUP', + augmentError: true + }) ); app.post( app.base_url_mapconfig, cors(), userMiddleware, this.prepareContext, - this.createPost.bind(this) + this.createPost.bind(this), + mapErrorMiddleware({ + label: 'ANONYMOUS LAYERGROUP', + augmentError: true + }) ); app.get( app.base_url_templated + '/:template_id/jsonp', cors(), userMiddleware, this.prepareContext, - this.jsonp.bind(this) + this.jsonp.bind(this), + mapErrorMiddleware({ + label: 'NAMED MAP LAYERGROUP' + }) ); app.post( app.base_url_templated + '/:template_id', cors(), userMiddleware, this.prepareContext, - this.instantiate.bind(this) + this.instantiate.bind(this), + mapErrorMiddleware({ + label: 'NAMED MAP LAYERGROUP' + }) ); app.options(app.base_url_mapconfig, cors('Content-Type')); }; @@ -170,7 +184,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { function createLayergroup(err, requestMapConfig) { assert.ifError(err); var datasource = context.datasource || Datasource.EmptyDatasource(); - mapConfig = new MapConfig(requestMapConfig, datasource); + res.locals.mapconfig = mapConfig = new MapConfig(requestMapConfig, datasource); self.mapBackend.createLayergroup( mapConfig, res.locals, @@ -181,7 +195,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - res.locals.mapconfig = mapConfig; res.locals.analysesResults = context.analysesResults; res.locals.layergroup = layergroup; res.locals.context = context; @@ -194,8 +207,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { }, function finish(err) { if (err) { - err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err; - err.label = 'ANONYMOUS LAYERGROUP'; return next(err); } @@ -212,25 +223,6 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { ); }; -function populateError(err, mapConfig) { - var error = new Error(err.message); - error.http_status = err.http_status; - - if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { - error.http_status = 400; - } - - error.type = 'layer'; - error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; - error.layer = { - id: mapConfig.getLayerId(err.layerIndex), - index: err.layerIndex, - type: mapConfig.layerType(err.layerIndex) - }; - - return error; -} - MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn, next) { var self = this; @@ -260,7 +252,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function createLayergroup(err, mapConfig_, rendererParams) { assert.ifError(err); - mapConfig = mapConfig_; + res.locals.mapconfig = mapConfig = mapConfig_; self.mapBackend.createLayergroup( mapConfig, rendererParams, new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams), @@ -270,7 +262,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - res.locals.mapconfig = mapConfig; res.locals.analysesResults = mapConfigProvider.analysesResults; res.locals.layergroup = layergroup; res.locals.template = mapConfigProvider.template; @@ -285,7 +276,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn }, function finishTemplateInstantiation(err) { if (err) { - err.label = 'NAMED MAP LAYERGROUP'; return next(err); } @@ -640,3 +630,38 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig }.bind(this)); } }; + +function mapErrorMiddleware (options) { + const { augmentError = false, label = 'MAPS CONTROLLER' } = options; + + return function mapError (err, req, res, next) { + const { mapconfig } = res.locals; + + if (augmentError) { + err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; + } + + err.label = label; + + next(err); + }; +} + +function populateError(err, mapConfig) { + var error = new Error(err.message); + error.http_status = err.http_status; + + if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) { + error.http_status = 400; + } + + error.type = 'layer'; + error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined; + error.layer = { + id: mapConfig.getLayerId(err.layerIndex), + index: err.layerIndex, + type: mapConfig.layerType(err.layerIndex) + }; + + return error; +} From 8ed5df00727724abe1201d4f498eff18ee100bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 1 Nov 2017 17:57:35 +0100 Subject: [PATCH 074/167] Move prepeareConfigFn to a middleware --- lib/cartodb/controllers/map.js | 115 +++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 49 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 82a7732b..4566cdb3 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -56,6 +56,7 @@ MapController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, + createGetPrepareConfig, this.createGet.bind(this), mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -67,6 +68,7 @@ MapController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, + createPostPrepareConfig, this.createPost.bind(this), mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -78,6 +80,7 @@ MapController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, + prepareJsonTemplateParams, this.jsonp.bind(this), mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -88,6 +91,7 @@ MapController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, + prepareTemplateParams, this.instantiate.bind(this), mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -96,62 +100,80 @@ MapController.prototype.register = function(app) { app.options(app.base_url_mapconfig, cors('Content-Type')); }; +function createGetPrepareConfig (req, res, next) { + const { config } = res.locals; + + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } + + try { + req.body = JSON.parse(config); + } catch (err) { + return next(err); + } + + return next(); +} + +function createPostPrepareConfig(req, res, next) { + if (!req.is('application/json')) { + return next(new Error('layergroup POST data must be of type application/json')); + } + + next(); +} + +function prepareTemplateParams(req, res, next) { + if (!req.is('application/json')) { + return next(new Error('Template POST data must be of type application/json')); + } + + return next(); +} + +function prepareJsonTemplateParams(req, res, next) { + const { callback, config } = req.query; + + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); + } + + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } + } + + return next(); +} + MapController.prototype.createGet = function(req, res, next){ req.profiler.start('windshaft.createmap_get'); - this.create(req, res, function createGet$prepareConfig(req, config) { - if ( ! config ) { - throw new Error('layergroup GET needs a "config" parameter'); - } - return JSON.parse(config); - }, next); + this.create(req, res, next); }; MapController.prototype.createPost = function(req, res, next) { req.profiler.start('windshaft.createmap_post'); - this.create(req, res, function createPost$prepareConfig(req) { - if (!req.is('application/json')) { - throw new Error('layergroup POST data must be of type application/json'); - } - return req.body; - }, next); + this.create(req, res, next); }; MapController.prototype.instantiate = function(req, res, next) { req.profiler.start('windshaft-cartodb.instance_template_post'); - this.instantiateTemplate(req, res, function prepareTemplateParams(callback) { - if (!req.is('application/json')) { - return callback(new Error('Template POST data must be of type application/json')); - } - return callback(null, req.body); - }, next); + this.instantiateTemplate(req, res, next); }; MapController.prototype.jsonp = function(req, res, next) { req.profiler.start('windshaft-cartodb.instance_template_get'); - - this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) { - var err = null; - if ( req.query.callback === undefined || req.query.callback.length === 0) { - err = new Error('callback parameter should be present and be a function name'); - } - - var templateParams = {}; - if (req.query.config) { - try { - templateParams = JSON.parse(req.query.config); - } catch(e) { - err = new Error('Invalid config parameter, should be a valid JSON'); - } - } - - return callback(err, templateParams); - }, next); + this.instantiateTemplate(req, res, next); }; -MapController.prototype.create = function(req, res, prepareConfigFn, next) { +MapController.prototype.create = function(req, res, next) { var self = this; var mapConfig; @@ -159,12 +181,9 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { var context = {}; step( - function prepareConfig () { - const requestMapConfig = prepareConfigFn(req, res.locals.config); - return requestMapConfig; - }, - function prepareAdapterMapConfig(err, requestMapConfig) { - assert.ifError(err); + function prepareAdapterMapConfig() { + const requestMapConfig = req.body; + context.analysisConfiguration = { user: res.locals.user, db: { @@ -223,7 +242,7 @@ MapController.prototype.create = function(req, res, prepareConfigFn, next) { ); }; -MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn, next) { +MapController.prototype.instantiateTemplate = function(req, res, next) { var self = this; var cdbuser = res.locals.user; @@ -231,11 +250,9 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn var mapConfigProvider; var mapConfig; step( - function getTemplateParams() { - prepareParamsFn(this); - }, - function getTemplate(err, templateParams) { - assert.ifError(err); + function getTemplate() { + const templateParams = req.body; + mapConfigProvider = new NamedMapMapConfigProvider( self.templateMaps, self.pgConnection, From aeb9585708fd60eef65d03e9744b0324062f747c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 1 Nov 2017 19:02:07 +0100 Subject: [PATCH 075/167] extract prepare mapconfig and get template to their respective middlewares --- lib/cartodb/controllers/map.js | 141 ++++++++++++++++++++------------- 1 file changed, 85 insertions(+), 56 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4566cdb3..96844ecf 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -57,6 +57,7 @@ MapController.prototype.register = function(app) { userMiddleware, this.prepareContext, createGetPrepareConfig, + this.prepareAdapterMapConfig.bind(this), this.createGet.bind(this), mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -69,6 +70,7 @@ MapController.prototype.register = function(app) { userMiddleware, this.prepareContext, createPostPrepareConfig, + this.prepareAdapterMapConfig.bind(this), this.createPost.bind(this), mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -81,6 +83,7 @@ MapController.prototype.register = function(app) { userMiddleware, this.prepareContext, prepareJsonTemplateParams, + this.getTemplate.bind(this), this.jsonp.bind(this), mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -92,6 +95,7 @@ MapController.prototype.register = function(app) { userMiddleware, this.prepareContext, prepareTemplateParams, + this.getTemplate.bind(this), this.instantiate.bind(this), mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -173,47 +177,61 @@ MapController.prototype.jsonp = function(req, res, next) { this.instantiateTemplate(req, res, next); }; +MapController.prototype.prepareAdapterMapConfig = function (req, res, next) { + const requestMapConfig = req.body; + const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; + + const context = { + analysisConfiguration: { + user, + db: { + host: dbhost, + port: dbport, + dbname: dbname, + user: dbuser, + pass: dbpassword + }, + batch: { + username: user, + apiKey: api_key + } + } + }; + + this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { + if (err) { + return next(err); + } + + req.body = requestMapConfig; + res.locals.context = context; + + next(); + }); +}; + MapController.prototype.create = function(req, res, next) { var self = this; - var mapConfig; - - var context = {}; - step( - function prepareAdapterMapConfig() { + function createLayergroup() { const requestMapConfig = req.body; + const { context } = res.locals; - context.analysisConfiguration = { - user: res.locals.user, - db: { - host: res.locals.dbhost, - port: res.locals.dbport, - dbname: res.locals.dbname, - user: res.locals.dbuser, - pass: res.locals.dbpassword - }, - batch: { - username: res.locals.user, - apiKey: res.locals.api_key - } - }; - self.mapConfigAdapter.getMapConfig(res.locals.user, requestMapConfig, res.locals, context, this); - }, - function createLayergroup(err, requestMapConfig) { - assert.ifError(err); - var datasource = context.datasource || Datasource.EmptyDatasource(); - res.locals.mapconfig = mapConfig = new MapConfig(requestMapConfig, datasource); + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapconfig = res.locals.mapconfig = new MapConfig(requestMapConfig, datasource); self.mapBackend.createLayergroup( - mapConfig, + mapconfig, res.locals, - new CreateLayergroupMapConfigProvider(mapConfig, res.locals.user, self.userLimitsApi, res.locals), + new CreateLayergroupMapConfigProvider(mapconfig, res.locals.user, self.userLimitsApi, res.locals), this ); }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); + const { context } = res.locals; + res.locals.analysesResults = context.analysesResults; res.locals.layergroup = layergroup; res.locals.context = context; @@ -242,48 +260,59 @@ MapController.prototype.create = function(req, res, next) { ); }; +MapController.prototype.getTemplate = function(req, res, next) { + const templateParams = req.body; + const { user } = res.locals; + + const mapConfigProvider = res.locals.mapconfigProvider = new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsApi, + this.mapConfigAdapter, + user, + req.params.template_id, + templateParams, + res.locals.auth_token, + res.locals + ); + + mapConfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + if (err) { + return next(err); + } + + res.locals.mapconfig = mapconfig; + res.locals.rendererParams = rendererParams; + + next(); + }); +}; + MapController.prototype.instantiateTemplate = function(req, res, next) { var self = this; - var cdbuser = res.locals.user; - - var mapConfigProvider; - var mapConfig; step( - function getTemplate() { - const templateParams = req.body; + function createLayergroup() { + const { user, mapconfig, rendererParams } = res.locals; - mapConfigProvider = new NamedMapMapConfigProvider( - self.templateMaps, - self.pgConnection, - self.metadataBackend, - self.userLimitsApi, - self.mapConfigAdapter, - cdbuser, - req.params.template_id, - templateParams, - res.locals.auth_token, - res.locals - ); - mapConfigProvider.getMapConfig(this); - }, - function createLayergroup(err, mapConfig_, rendererParams) { - assert.ifError(err); - res.locals.mapconfig = mapConfig = mapConfig_; self.mapBackend.createLayergroup( - mapConfig, rendererParams, - new CreateLayergroupMapConfigProvider(mapConfig, cdbuser, self.userLimitsApi, rendererParams), + mapconfig, rendererParams, + new CreateLayergroupMapConfigProvider(mapconfig, user, self.userLimitsApi, rendererParams), this ); }, function afterLayergroupCreate(err, layergroup) { assert.ifError(err); - res.locals.analysesResults = mapConfigProvider.analysesResults; res.locals.layergroup = layergroup; - res.locals.template = mapConfigProvider.template; - res.locals.templateName = mapConfigProvider.getTemplateName(); - res.locals.context = mapConfigProvider.context; + + const { mapconfigProvider } = res.locals; + + res.locals.analysesResults = mapconfigProvider.analysesResults; + res.locals.template = mapconfigProvider.template; + res.locals.templateName = mapconfigProvider.getTemplateName(); + res.locals.context = mapconfigProvider.context; const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ useTemplateHash: true From 125587522f73f8a4ec331d497450c467031266d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 1 Nov 2017 19:27:01 +0100 Subject: [PATCH 076/167] Create middleware for layergroup creation --- lib/cartodb/controllers/map.js | 52 +++++++++++++++++++--------------- 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 96844ecf..342fdeb2 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -84,6 +84,7 @@ MapController.prototype.register = function(app) { this.prepareContext, prepareJsonTemplateParams, this.getTemplate.bind(this), + this.createLayergroupFromTemplate.bind(this), this.jsonp.bind(this), mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -96,6 +97,7 @@ MapController.prototype.register = function(app) { this.prepareContext, prepareTemplateParams, this.getTemplate.bind(this), + this.createLayergroupFromTemplate.bind(this), this.instantiate.bind(this), mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -264,7 +266,7 @@ MapController.prototype.getTemplate = function(req, res, next) { const templateParams = req.body; const { user } = res.locals; - const mapConfigProvider = res.locals.mapconfigProvider = new NamedMapMapConfigProvider( + const mapconfigProvider = new NamedMapMapConfigProvider( this.templateMaps, this.pgConnection, this.metadataBackend, @@ -277,13 +279,37 @@ MapController.prototype.getTemplate = function(req, res, next) { res.locals ); - mapConfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { if (err) { return next(err); } res.locals.mapconfig = mapconfig; res.locals.rendererParams = rendererParams; + res.locals.mapconfigProvider = mapconfigProvider; + + next(); + }); +}; + +MapController.prototype.createLayergroupFromTemplate = function(req, res, next) { + const { user, mapconfig, rendererParams } = res.locals; + const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); + + this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + if (err) { + return next(err); + } + + res.locals.layergroup = layergroup; + + // TODO: Do not provide shortcuts + const { mapconfigProvider } = res.locals; + + res.locals.analysesResults = mapconfigProvider.analysesResults; + res.locals.template = mapconfigProvider.template; + res.locals.templateName = mapconfigProvider.getTemplateName(); + res.locals.context = mapconfigProvider.context; next(); }); @@ -293,27 +319,7 @@ MapController.prototype.instantiateTemplate = function(req, res, next) { var self = this; step( - function createLayergroup() { - const { user, mapconfig, rendererParams } = res.locals; - - self.mapBackend.createLayergroup( - mapconfig, rendererParams, - new CreateLayergroupMapConfigProvider(mapconfig, user, self.userLimitsApi, rendererParams), - this - ); - }, - function afterLayergroupCreate(err, layergroup) { - assert.ifError(err); - - res.locals.layergroup = layergroup; - - const { mapconfigProvider } = res.locals; - - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; - + function afterLayergroupCreate() { const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ useTemplateHash: true }); From e6bec5ccb025024e6c1d5f65fe863e8d086805ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 1 Nov 2017 19:28:32 +0100 Subject: [PATCH 077/167] Make style linter happy --- lib/cartodb/controllers/map.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 342fdeb2..4bd4457b 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -294,7 +294,8 @@ MapController.prototype.getTemplate = function(req, res, next) { MapController.prototype.createLayergroupFromTemplate = function(req, res, next) { const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); + const mapconfigProvider = + new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { if (err) { From 46c76d6a4c1ffde26c85243453bb5c1b278674f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 1 Nov 2017 19:57:20 +0100 Subject: [PATCH 078/167] Create middleware for layergroup creation (anonymous map) --- lib/cartodb/controllers/map.js | 46 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4bd4457b..2d65b801 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -58,6 +58,7 @@ MapController.prototype.register = function(app) { this.prepareContext, createGetPrepareConfig, this.prepareAdapterMapConfig.bind(this), + this.createLayergroup.bind(this), this.createGet.bind(this), mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -71,6 +72,7 @@ MapController.prototype.register = function(app) { this.prepareContext, createPostPrepareConfig, this.prepareAdapterMapConfig.bind(this), + this.createLayergroup.bind(this), this.createPost.bind(this), mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -212,32 +214,32 @@ MapController.prototype.prepareAdapterMapConfig = function (req, res, next) { }); }; +MapController.prototype.createLayergroup = function(req, res, next) { + const requestMapConfig = req.body; + const { context, user } = res.locals; + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapconfig = new MapConfig(requestMapConfig, datasource); + const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); + + res.locals.mapconfig = mapconfig; + res.locals.analysesResults = context.analysesResults; + + this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + if (err) { + return next(err); + } + + res.locals.layergroup = layergroup; + + next(); + }); +}; + MapController.prototype.create = function(req, res, next) { var self = this; step( - function createLayergroup() { - const requestMapConfig = req.body; - const { context } = res.locals; - - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = res.locals.mapconfig = new MapConfig(requestMapConfig, datasource); - self.mapBackend.createLayergroup( - mapconfig, - res.locals, - new CreateLayergroupMapConfigProvider(mapconfig, res.locals.user, self.userLimitsApi, res.locals), - this - ); - }, - function afterLayergroupCreate(err, layergroup) { - assert.ifError(err); - - const { context } = res.locals; - - res.locals.analysesResults = context.analysesResults; - res.locals.layergroup = layergroup; - res.locals.context = context; - + function afterLayergroupCreate() { const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ includeQuery: true }); From c8000e5cf8a2fa2bd6af71c0fc4a9ba67dc3bcb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 1 Nov 2017 20:06:32 +0100 Subject: [PATCH 079/167] Make a middleware to respond layergroup --- lib/cartodb/controllers/map.js | 38 ++++++++++++++++------------------ 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 2d65b801..901cc886 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -60,6 +60,7 @@ MapController.prototype.register = function(app) { this.prepareAdapterMapConfig.bind(this), this.createLayergroup.bind(this), this.createGet.bind(this), + respond, mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', augmentError: true @@ -74,6 +75,7 @@ MapController.prototype.register = function(app) { this.prepareAdapterMapConfig.bind(this), this.createLayergroup.bind(this), this.createPost.bind(this), + respond, mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', augmentError: true @@ -88,6 +90,7 @@ MapController.prototype.register = function(app) { this.getTemplate.bind(this), this.createLayergroupFromTemplate.bind(this), this.jsonp.bind(this), + respond, mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' }) @@ -101,6 +104,7 @@ MapController.prototype.register = function(app) { this.getTemplate.bind(this), this.createLayergroupFromTemplate.bind(this), this.instantiate.bind(this), + respond, mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' }) @@ -250,16 +254,7 @@ MapController.prototype.create = function(req, res, next) { if (err) { return next(err); } - - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } + next(); } ); }; @@ -333,20 +328,23 @@ MapController.prototype.instantiateTemplate = function(req, res, next) { if (err) { return next(err); } - - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } + next(); } ); }; +function respond (req, res) { + const { layergroup } = res.locals; + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); + } else { + res.json(layergroup); + } +} + MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { const self = this; const { From eb5bf52bd9be1f5b97aca748f574cb10462cf2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 10:22:30 +0100 Subject: [PATCH 080/167] Move profiler start to the right place --- lib/cartodb/controllers/map.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 901cc886..a50bc72e 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -113,6 +113,8 @@ MapController.prototype.register = function(app) { }; function createGetPrepareConfig (req, res, next) { + req.profiler.start('windshaft.createmap_get'); + const { config } = res.locals; if (!config) { @@ -129,6 +131,8 @@ function createGetPrepareConfig (req, res, next) { } function createPostPrepareConfig(req, res, next) { + req.profiler.start('windshaft.createmap_post'); + if (!req.is('application/json')) { return next(new Error('layergroup POST data must be of type application/json')); } @@ -137,6 +141,8 @@ function createPostPrepareConfig(req, res, next) { } function prepareTemplateParams(req, res, next) { + req.profiler.start('windshaft-cartodb.instance_template_post'); + if (!req.is('application/json')) { return next(new Error('Template POST data must be of type application/json')); } @@ -145,6 +151,8 @@ function prepareTemplateParams(req, res, next) { } function prepareJsonTemplateParams(req, res, next) { + req.profiler.start('windshaft-cartodb.instance_template_get'); + const { callback, config } = req.query; if (callback === undefined || callback.length === 0) { @@ -163,25 +171,18 @@ function prepareJsonTemplateParams(req, res, next) { } MapController.prototype.createGet = function(req, res, next){ - req.profiler.start('windshaft.createmap_get'); - this.create(req, res, next); }; MapController.prototype.createPost = function(req, res, next) { - req.profiler.start('windshaft.createmap_post'); - this.create(req, res, next); }; MapController.prototype.instantiate = function(req, res, next) { - req.profiler.start('windshaft-cartodb.instance_template_post'); - this.instantiateTemplate(req, res, next); }; MapController.prototype.jsonp = function(req, res, next) { - req.profiler.start('windshaft-cartodb.instance_template_get'); this.instantiateTemplate(req, res, next); }; From d2b5eaa8c334923c6bd0442573b94790ae145eb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 10:28:33 +0100 Subject: [PATCH 081/167] Do not proxy create and intantiate middlewares --- lib/cartodb/controllers/map.js | 24 ++++-------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index a50bc72e..72d597a1 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -59,7 +59,7 @@ MapController.prototype.register = function(app) { createGetPrepareConfig, this.prepareAdapterMapConfig.bind(this), this.createLayergroup.bind(this), - this.createGet.bind(this), + this.create.bind(this), respond, mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -74,7 +74,7 @@ MapController.prototype.register = function(app) { createPostPrepareConfig, this.prepareAdapterMapConfig.bind(this), this.createLayergroup.bind(this), - this.createPost.bind(this), + this.create.bind(this), respond, mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -89,7 +89,7 @@ MapController.prototype.register = function(app) { prepareJsonTemplateParams, this.getTemplate.bind(this), this.createLayergroupFromTemplate.bind(this), - this.jsonp.bind(this), + this.instantiateTemplate.bind(this), respond, mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -103,7 +103,7 @@ MapController.prototype.register = function(app) { prepareTemplateParams, this.getTemplate.bind(this), this.createLayergroupFromTemplate.bind(this), - this.instantiate.bind(this), + this.instantiateTemplate.bind(this), respond, mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -170,22 +170,6 @@ function prepareJsonTemplateParams(req, res, next) { return next(); } -MapController.prototype.createGet = function(req, res, next){ - this.create(req, res, next); -}; - -MapController.prototype.createPost = function(req, res, next) { - this.create(req, res, next); -}; - -MapController.prototype.instantiate = function(req, res, next) { - this.instantiateTemplate(req, res, next); -}; - -MapController.prototype.jsonp = function(req, res, next) { - this.instantiateTemplate(req, res, next); -}; - MapController.prototype.prepareAdapterMapConfig = function (req, res, next) { const requestMapConfig = req.body; const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; From 658763da8c7d7d246cd775b1d7404e43940c1ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 10:33:39 +0100 Subject: [PATCH 082/167] Build after layergroup create while registering routes --- lib/cartodb/controllers/map.js | 56 ++++++++-------------------------- 1 file changed, 12 insertions(+), 44 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 72d597a1..384678cb 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -59,7 +59,9 @@ MapController.prototype.register = function(app) { createGetPrepareConfig, this.prepareAdapterMapConfig.bind(this), this.createLayergroup.bind(this), - this.create.bind(this), + this.afterLayergroupCreateBuilder({ + includeQuery: true + }), respond, mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -74,7 +76,9 @@ MapController.prototype.register = function(app) { createPostPrepareConfig, this.prepareAdapterMapConfig.bind(this), this.createLayergroup.bind(this), - this.create.bind(this), + this.afterLayergroupCreateBuilder({ + includeQuery: true + }), respond, mapErrorMiddleware({ label: 'ANONYMOUS LAYERGROUP', @@ -89,7 +93,9 @@ MapController.prototype.register = function(app) { prepareJsonTemplateParams, this.getTemplate.bind(this), this.createLayergroupFromTemplate.bind(this), - this.instantiateTemplate.bind(this), + this.afterLayergroupCreateBuilder({ + useTemplateHash: true + }), respond, mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -103,7 +109,9 @@ MapController.prototype.register = function(app) { prepareTemplateParams, this.getTemplate.bind(this), this.createLayergroupFromTemplate.bind(this), - this.instantiateTemplate.bind(this), + this.afterLayergroupCreateBuilder({ + useTemplateHash: true + }), respond, mapErrorMiddleware({ label: 'NAMED MAP LAYERGROUP' @@ -224,26 +232,6 @@ MapController.prototype.createLayergroup = function(req, res, next) { }); }; -MapController.prototype.create = function(req, res, next) { - var self = this; - - step( - function afterLayergroupCreate() { - const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ - includeQuery: true - }); - - afterLayergroupCreate(req, res, this); - }, - function finish(err) { - if (err) { - return next(err); - } - next(); - } - ); -}; - MapController.prototype.getTemplate = function(req, res, next) { const templateParams = req.body; const { user } = res.locals; @@ -298,26 +286,6 @@ MapController.prototype.createLayergroupFromTemplate = function(req, res, next) }); }; -MapController.prototype.instantiateTemplate = function(req, res, next) { - var self = this; - - step( - function afterLayergroupCreate() { - const afterLayergroupCreate = self.afterLayergroupCreateBuilder({ - useTemplateHash: true - }); - - afterLayergroupCreate(req, res, this); - }, - function finishTemplateInstantiation(err) { - if (err) { - return next(err); - } - next(); - } - ); -}; - function respond (req, res) { const { layergroup } = res.locals; From 93bd2c9e509e69db8aa1f5b591057e6afa730aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 10:43:22 +0100 Subject: [PATCH 083/167] Build afterLayergroupCreate middleware as an array of middlewares instead of preforming all them as one middleware --- lib/cartodb/controllers/map.js | 75 +++++++--------------------------- 1 file changed, 14 insertions(+), 61 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 384678cb..7478ce49 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -299,72 +299,25 @@ function respond (req, res) { } MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { - const self = this; const { useTemplateHash = false, includeQuery = false } = options; - return function (req, res, callback) { - step( - function incrementMapViewCount () { - self.incrementMapViewCount(req, res, this); - }, - function augmentLayergroupData (err) { - assert.ifError(err); - self.augmentLayergroupData(req, res, this); - }, - function getAffectedTables (err) { - assert.ifError(err); - self.getAffectedTables(req, res, this); - }, - function setCacheChannel (err) { - assert.ifError(err); - self.setCacheChannel(req, res, this); - }, - function setLastUpdatedTime (err) { - assert.ifError(err); - self.setLastUpdatedTimeToLayergroup(req, res, this); - }, - function setCacheControl (err) { - assert.ifError(err); - self.setCacheControl(req, res, this); - }, - function setLayerStats (err) { - assert.ifError(err); - self.setLayerStats(req, res, this); - }, - function setLayergroupIdHeader (err) { - assert.ifError(err); - - const setLayergroupIdHeader = self.setLayergroupIdHeaderBuilder(useTemplateHash); - - setLayergroupIdHeader(req, res, this); - }, - function (err) { - assert.ifError(err); - self.setDataviewsAndWidgetsUrlsToLayergroupMetadata(req, res, this); - }, - function setAnalysesMetadataToLayergroup(err) { - assert.ifError(err); - - const setAnalysesMetadataToLayergroup = self.setAnalysesMetadataToLayergroupBuilder(includeQuery); - - setAnalysesMetadataToLayergroup(req, res, this); - }, - function setTurboCartoMetadataToLayergroup (err) { - assert.ifError(err); - self.setTurboCartoMetadataToLayergroup(req, res, this); - }, - function setSurrogateKeyHeader (err) { - assert.ifError(err); - self.setSurrogateKeyHeader(req, res, this); - }, - function finish(err) { - callback(err); - } - ); - }; + return [ + this.incrementMapViewCount.bind(this), + this.augmentLayergroupData.bind(this), + this.getAffectedTables.bind(this), + this.setCacheChannel.bind(this), + this.setLastUpdatedTimeToLayergroup.bind(this), + this.setCacheControl.bind(this), + this.setLayerStats.bind(this), + this.setLayergroupIdHeaderBuilder(useTemplateHash), + this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), + this.setAnalysesMetadataToLayergroupBuilder(includeQuery), + this.setTurboCartoMetadataToLayergroup.bind(this), + this.setSurrogateKeyHeader.bind(this) + ]; }; MapController.prototype.incrementMapViewCount = function (req, res, callback) { From b11b872b755ce4204befbd7da1ac593ceaa808a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 11:29:43 +0100 Subject: [PATCH 084/167] Remove step requirement --- lib/cartodb/controllers/map.js | 46 +++++++++++++++------------------- 1 file changed, 20 insertions(+), 26 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 7478ce49..372dd567 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -1,6 +1,4 @@ var _ = require('underscore'); -var assert = require('assert'); -var step = require('step'); var windshaft = require('windshaft'); var QueryTables = require('cartodb-query-tables'); @@ -346,41 +344,37 @@ MapController.prototype.augmentLayergroupData = function (req, res, callback) { callback(); }; -MapController.prototype.getAffectedTables = function (req, res, callback) { - const self = this; +MapController.prototype.getAffectedTables = function (req, res, next) { const { dbname, layergroup, user, mapconfig } = res.locals; - var sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); } - }); - step( - function getPgConnection() { - self.pgConnection.getConnection(user, this); - }, - function getAffectedTablesAndLastUpdatedTime(err, connection) { - assert.ifError(err); - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this); - }, - function handleAffectedTablesAndLastUpdatedTime(err, affectedTables) { + const sql = []; + mapconfig.getLayers().forEach(function(layer) { + sql.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(function(table) { + sql.push('SELECT * FROM ' + table + ' LIMIT 0'); + }); + } + }); + + QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { if (err) { - return callback(err); + return next(err); } // feed affected tables cache so it can be reused from, for instance, layergroup controller - self.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); res.locals.affectedTables = affectedTables; - callback(); - } - ); + next(); + }); + }); }; MapController.prototype.setCacheChannel = function (req, res, callback) { From 1d08734721c3125559103fe702d665735be23273 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 18:28:37 +0100 Subject: [PATCH 085/167] Rename middleware --- lib/cartodb/controllers/map.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 372dd567..4d98b2b7 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -90,7 +90,7 @@ MapController.prototype.register = function(app) { this.prepareContext, prepareJsonTemplateParams, this.getTemplate.bind(this), - this.createLayergroupFromTemplate.bind(this), + this.instantiateLayergroup.bind(this), this.afterLayergroupCreateBuilder({ useTemplateHash: true }), @@ -106,7 +106,7 @@ MapController.prototype.register = function(app) { this.prepareContext, prepareTemplateParams, this.getTemplate.bind(this), - this.createLayergroupFromTemplate.bind(this), + this.instantiateLayergroup.bind(this), this.afterLayergroupCreateBuilder({ useTemplateHash: true }), @@ -260,7 +260,7 @@ MapController.prototype.getTemplate = function(req, res, next) { }); }; -MapController.prototype.createLayergroupFromTemplate = function(req, res, next) { +MapController.prototype.instantiateLayergroup = function(req, res, next) { const { user, mapconfig, rendererParams } = res.locals; const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); From 08b91f935d52a9eb9501339f35b44a6ec6825647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 18:38:34 +0100 Subject: [PATCH 086/167] Rename error middleware --- lib/cartodb/controllers/map.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4d98b2b7..a9f2f327 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -61,9 +61,9 @@ MapController.prototype.register = function(app) { includeQuery: true }), respond, - mapErrorMiddleware({ + augmentError({ label: 'ANONYMOUS LAYERGROUP', - augmentError: true + addContext: true }) ); app.post( @@ -78,9 +78,9 @@ MapController.prototype.register = function(app) { includeQuery: true }), respond, - mapErrorMiddleware({ + augmentError({ label: 'ANONYMOUS LAYERGROUP', - augmentError: true + addContext: true }) ); app.get( @@ -95,7 +95,7 @@ MapController.prototype.register = function(app) { useTemplateHash: true }), respond, - mapErrorMiddleware({ + augmentError({ label: 'NAMED MAP LAYERGROUP' }) ); @@ -111,7 +111,7 @@ MapController.prototype.register = function(app) { useTemplateHash: true }), respond, - mapErrorMiddleware({ + augmentError({ label: 'NAMED MAP LAYERGROUP' }) ); @@ -584,8 +584,8 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig } }; -function mapErrorMiddleware (options) { - const { augmentError = false, label = 'MAPS CONTROLLER' } = options; +function augmentError (options) { + const { addContext = false, label = 'MAPS CONTROLLER' } = options; return function mapError (err, req, res, next) { const { mapconfig } = res.locals; From 3e7106002dff0adb285c2ed8c11ea32107287297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 18:39:46 +0100 Subject: [PATCH 087/167] Rename response middleware --- lib/cartodb/controllers/map.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index a9f2f327..b153b607 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -60,7 +60,7 @@ MapController.prototype.register = function(app) { this.afterLayergroupCreateBuilder({ includeQuery: true }), - respond, + sendResponse, augmentError({ label: 'ANONYMOUS LAYERGROUP', addContext: true @@ -77,7 +77,7 @@ MapController.prototype.register = function(app) { this.afterLayergroupCreateBuilder({ includeQuery: true }), - respond, + sendResponse, augmentError({ label: 'ANONYMOUS LAYERGROUP', addContext: true @@ -94,7 +94,7 @@ MapController.prototype.register = function(app) { this.afterLayergroupCreateBuilder({ useTemplateHash: true }), - respond, + sendResponse, augmentError({ label: 'NAMED MAP LAYERGROUP' }) @@ -110,7 +110,7 @@ MapController.prototype.register = function(app) { this.afterLayergroupCreateBuilder({ useTemplateHash: true }), - respond, + sendResponse, augmentError({ label: 'NAMED MAP LAYERGROUP' }) @@ -284,7 +284,7 @@ MapController.prototype.instantiateLayergroup = function(req, res, next) { }); }; -function respond (req, res) { +function sendResponse (req, res) { const { layergroup } = res.locals; res.status(200); From 4607e4a12deb72c701013c64bc8b6c2748fc90f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 19:03:20 +0100 Subject: [PATCH 088/167] Compose create layergroup middleware --- lib/cartodb/controllers/map.js | 79 +++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index b153b607..362281ca 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -49,40 +49,15 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m module.exports = MapController; MapController.prototype.register = function(app) { - app.get( - app.base_url_mapconfig, - cors(), - userMiddleware, - this.prepareContext, - createGetPrepareConfig, - this.prepareAdapterMapConfig.bind(this), - this.createLayergroup.bind(this), - this.afterLayergroupCreateBuilder({ - includeQuery: true - }), - sendResponse, - augmentError({ - label: 'ANONYMOUS LAYERGROUP', - addContext: true - }) - ); - app.post( - app.base_url_mapconfig, - cors(), - userMiddleware, - this.prepareContext, - createPostPrepareConfig, - this.prepareAdapterMapConfig.bind(this), - this.createLayergroup.bind(this), - this.afterLayergroupCreateBuilder({ - includeQuery: true - }), - sendResponse, - augmentError({ - label: 'ANONYMOUS LAYERGROUP', - addContext: true - }) - ); + app.get(app.base_url_mapconfig, this.composeCreateLayergroupMiddleware({ + includeQuery: true, + parseConfigQueryParam: true + })); + + app.post(app.base_url_mapconfig, this.composeCreateLayergroupMiddleware({ + includeQuery: true + })); + app.get( app.base_url_templated + '/:template_id/jsonp', cors(), @@ -118,8 +93,42 @@ MapController.prototype.register = function(app) { app.options(app.base_url_mapconfig, cors('Content-Type')); }; +MapController.prototype.composeCreateLayergroupMiddleware = function (options) { + const { + parseConfigQueryParam = false, + useTemplateHash = false, + includeQuery = false + } = options; + + return [ + cors(), + userMiddleware, + this.prepareContext, + parseConfigQueryParam ? createGetPrepareConfig : createPostPrepareConfig, + this.prepareAdapterMapConfig.bind(this), + this.createLayergroup.bind(this), + this.incrementMapViewCount.bind(this), + this.augmentLayergroupData.bind(this), + this.getAffectedTables.bind(this), + this.setCacheChannel.bind(this), + this.setLastUpdatedTimeToLayergroup.bind(this), + this.setCacheControl.bind(this), + this.setLayerStats.bind(this), + this.setLayergroupIdHeaderBuilder(useTemplateHash), + this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), + this.setAnalysesMetadataToLayergroupBuilder(includeQuery), + this.setTurboCartoMetadataToLayergroup.bind(this), + this.setSurrogateKeyHeader.bind(this), + sendResponse, + augmentError({ + label: 'ANONYMOUS LAYERGROUP', + addContext: true + }) + ]; +}; + function createGetPrepareConfig (req, res, next) { - req.profiler.start('windshaft.createmap_get'); + req.profiler.start(`windshaft.createmap_${req.method.toLowecase}`); const { config } = res.locals; From 717332d9416214b6dfc4802662c08da92fe9ff23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 2 Nov 2017 19:24:33 +0100 Subject: [PATCH 089/167] Compose instantiate layergroup middleware --- lib/cartodb/controllers/map.js | 95 ++++++++++++++++++++-------------- 1 file changed, 55 insertions(+), 40 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 362281ca..9bb23dac 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -50,46 +50,29 @@ module.exports = MapController; MapController.prototype.register = function(app) { app.get(app.base_url_mapconfig, this.composeCreateLayergroupMiddleware({ + parseConfigQueryParam: true, includeQuery: true, - parseConfigQueryParam: true + label: 'ANONYMOUS LAYERGROUP', + addContext: true })); app.post(app.base_url_mapconfig, this.composeCreateLayergroupMiddleware({ - includeQuery: true + includeQuery: true, + label: 'ANONYMOUS LAYERGROUP', + addContext: true + })); + + app.get(app.base_url_templated + '/:template_id/jsonp', this.composeInstantiateLayergroupMiddleware({ + parseConfigQueryParam: true, + useTemplateHash: true, + label: 'NAMED MAP LAYERGROUP' + })); + + app.post(app.base_url_templated + '/:template_id', this.composeInstantiateLayergroupMiddleware({ + useTemplateHash: true, + label: 'NAMED MAP LAYERGROUP' })); - app.get( - app.base_url_templated + '/:template_id/jsonp', - cors(), - userMiddleware, - this.prepareContext, - prepareJsonTemplateParams, - this.getTemplate.bind(this), - this.instantiateLayergroup.bind(this), - this.afterLayergroupCreateBuilder({ - useTemplateHash: true - }), - sendResponse, - augmentError({ - label: 'NAMED MAP LAYERGROUP' - }) - ); - app.post( - app.base_url_templated + '/:template_id', - cors(), - userMiddleware, - this.prepareContext, - prepareTemplateParams, - this.getTemplate.bind(this), - this.instantiateLayergroup.bind(this), - this.afterLayergroupCreateBuilder({ - useTemplateHash: true - }), - sendResponse, - augmentError({ - label: 'NAMED MAP LAYERGROUP' - }) - ); app.options(app.base_url_mapconfig, cors('Content-Type')); }; @@ -97,7 +80,9 @@ MapController.prototype.composeCreateLayergroupMiddleware = function (options) { const { parseConfigQueryParam = false, useTemplateHash = false, - includeQuery = false + includeQuery = false, + label, + addContext = false } = options; return [ @@ -120,15 +105,45 @@ MapController.prototype.composeCreateLayergroupMiddleware = function (options) { this.setTurboCartoMetadataToLayergroup.bind(this), this.setSurrogateKeyHeader.bind(this), sendResponse, - augmentError({ - label: 'ANONYMOUS LAYERGROUP', - addContext: true - }) + augmentError({ label, addContext }) + ]; +}; + +MapController.prototype.composeInstantiateLayergroupMiddleware = function (options) { + const { + parseConfigQueryParam = false, + useTemplateHash = false, + includeQuery = false, + label, + addContext = false + } = options; + + return [ + cors(), + userMiddleware, + this.prepareContext, + parseConfigQueryParam ? prepareJsonTemplateParams : prepareTemplateParams, + this.getTemplate.bind(this), + this.instantiateLayergroup.bind(this), + this.incrementMapViewCount.bind(this), + this.augmentLayergroupData.bind(this), + this.getAffectedTables.bind(this), + this.setCacheChannel.bind(this), + this.setLastUpdatedTimeToLayergroup.bind(this), + this.setCacheControl.bind(this), + this.setLayerStats.bind(this), + this.setLayergroupIdHeaderBuilder(useTemplateHash), + this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), + this.setAnalysesMetadataToLayergroupBuilder(includeQuery), + this.setTurboCartoMetadataToLayergroup.bind(this), + this.setSurrogateKeyHeader.bind(this), + sendResponse, + augmentError({ label, addContext }) ]; }; function createGetPrepareConfig (req, res, next) { - req.profiler.start(`windshaft.createmap_${req.method.toLowecase}`); + req.profiler.start(`windshaft.createmap_get`); const { config } = res.locals; From 2854d0252cb780f8fe4c2109f065467ec37317f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 08:48:13 +0100 Subject: [PATCH 090/167] Fix typo --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 9bb23dac..96a66e37 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -614,7 +614,7 @@ function augmentError (options) { return function mapError (err, req, res, next) { const { mapconfig } = res.locals; - if (augmentError) { + if (addContext) { err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err; } From cb167313d22dca88042fdbe466f920305ad2578e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 09:37:01 +0100 Subject: [PATCH 091/167] Unify middleware builder functions --- lib/cartodb/controllers/map.js | 149 +++++++++++---------------------- 1 file changed, 47 insertions(+), 102 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 96a66e37..ab65b604 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -49,49 +49,34 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m module.exports = MapController; MapController.prototype.register = function(app) { - app.get(app.base_url_mapconfig, this.composeCreateLayergroupMiddleware({ - parseConfigQueryParam: true, - includeQuery: true, - label: 'ANONYMOUS LAYERGROUP', - addContext: true + app.get(app.base_url_mapconfig, this.composeCreateMapMiddleware()); + + app.post(app.base_url_mapconfig, this.composeCreateMapMiddleware()); + + app.get(app.base_url_templated + '/:template_id/jsonp', this.composeCreateMapMiddleware({ + useTemplate: true })); - app.post(app.base_url_mapconfig, this.composeCreateLayergroupMiddleware({ - includeQuery: true, - label: 'ANONYMOUS LAYERGROUP', - addContext: true - })); - - app.get(app.base_url_templated + '/:template_id/jsonp', this.composeInstantiateLayergroupMiddleware({ - parseConfigQueryParam: true, - useTemplateHash: true, - label: 'NAMED MAP LAYERGROUP' - })); - - app.post(app.base_url_templated + '/:template_id', this.composeInstantiateLayergroupMiddleware({ - useTemplateHash: true, - label: 'NAMED MAP LAYERGROUP' + app.post(app.base_url_templated + '/:template_id', this.composeCreateMapMiddleware({ + useTemplate: true })); app.options(app.base_url_mapconfig, cors('Content-Type')); }; -MapController.prototype.composeCreateLayergroupMiddleware = function (options) { - const { - parseConfigQueryParam = false, - useTemplateHash = false, - includeQuery = false, - label, - addContext = false - } = options; +MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = false } = {}) { + const useTemplateHash = useTemplate; + const includeQuery = !useTemplate; + const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; + const addContext = !useTemplate; return [ cors(), userMiddleware, this.prepareContext, - parseConfigQueryParam ? createGetPrepareConfig : createPostPrepareConfig, - this.prepareAdapterMapConfig.bind(this), - this.createLayergroup.bind(this), + useTemplate ? checkIntantiteLayergroup : checkCreateLayergroup, + useTemplate ? this.getTemplate.bind(this) : this.prepareAdapterMapConfig.bind(this), + useTemplate ? this.instantiateLayergroup.bind(this) : this.createLayergroup.bind(this), this.incrementMapViewCount.bind(this), this.augmentLayergroupData.bind(this), this.getAffectedTables.bind(this), @@ -109,91 +94,51 @@ MapController.prototype.composeCreateLayergroupMiddleware = function (options) { ]; }; -MapController.prototype.composeInstantiateLayergroupMiddleware = function (options) { - const { - parseConfigQueryParam = false, - useTemplateHash = false, - includeQuery = false, - label, - addContext = false - } = options; +function checkCreateLayergroup (req, res, next) { + req.profiler.start(`windshaft.createmap_${req.method.toLowerCase()}`); - return [ - cors(), - userMiddleware, - this.prepareContext, - parseConfigQueryParam ? prepareJsonTemplateParams : prepareTemplateParams, - this.getTemplate.bind(this), - this.instantiateLayergroup.bind(this), - this.incrementMapViewCount.bind(this), - this.augmentLayergroupData.bind(this), - this.getAffectedTables.bind(this), - this.setCacheChannel.bind(this), - this.setLastUpdatedTimeToLayergroup.bind(this), - this.setCacheControl.bind(this), - this.setLayerStats.bind(this), - this.setLayergroupIdHeaderBuilder(useTemplateHash), - this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), - this.setAnalysesMetadataToLayergroupBuilder(includeQuery), - this.setTurboCartoMetadataToLayergroup.bind(this), - this.setSurrogateKeyHeader.bind(this), - sendResponse, - augmentError({ label, addContext }) - ]; -}; - -function createGetPrepareConfig (req, res, next) { - req.profiler.start(`windshaft.createmap_get`); - - const { config } = res.locals; - - if (!config) { - return next(new Error('layergroup GET needs a "config" parameter')); - } - - try { - req.body = JSON.parse(config); - } catch (err) { - return next(err); - } - - return next(); -} - -function createPostPrepareConfig(req, res, next) { - req.profiler.start('windshaft.createmap_post'); - - if (!req.is('application/json')) { + if (req.method === 'POST' && !req.is('application/json')) { return next(new Error('layergroup POST data must be of type application/json')); } - next(); -} + if (req.method === 'GET') { + const { config } = res.locals; -function prepareTemplateParams(req, res, next) { - req.profiler.start('windshaft-cartodb.instance_template_post'); + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } - if (!req.is('application/json')) { - return next(new Error('Template POST data must be of type application/json')); + try { + req.body = JSON.parse(config); + } catch (err) { + return next(err); + } } return next(); } -function prepareJsonTemplateParams(req, res, next) { - req.profiler.start('windshaft-cartodb.instance_template_get'); +function checkIntantiteLayergroup(req, res, next) { + // jshint maxcomplexity: 7 + req.profiler.start(`windshaft-cartodb.instance_template_${req.method.toLowerCase()}`); - const { callback, config } = req.query; - - if (callback === undefined || callback.length === 0) { - return next(new Error('callback parameter should be present and be a function name')); + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('Template POST data must be of type application/json')); } - if (config) { - try { - req.body = JSON.parse(config); - } catch(e) { - return next(new Error('Invalid config parameter, should be a valid JSON')); + if (req.method === 'GET') { + const { callback, config } = req.query; + + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); + } + + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } } } From 677f6caab80b9e8fc1b3504f6803f61602d73691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 09:38:36 +0100 Subject: [PATCH 092/167] remove funtion --- lib/cartodb/controllers/map.js | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ab65b604..7a1dc6be 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -265,28 +265,6 @@ function sendResponse (req, res) { } } -MapController.prototype.afterLayergroupCreateBuilder = function (options = {}) { - const { - useTemplateHash = false, - includeQuery = false - } = options; - - return [ - this.incrementMapViewCount.bind(this), - this.augmentLayergroupData.bind(this), - this.getAffectedTables.bind(this), - this.setCacheChannel.bind(this), - this.setLastUpdatedTimeToLayergroup.bind(this), - this.setCacheControl.bind(this), - this.setLayerStats.bind(this), - this.setLayergroupIdHeaderBuilder(useTemplateHash), - this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), - this.setAnalysesMetadataToLayergroupBuilder(includeQuery), - this.setTurboCartoMetadataToLayergroup.bind(this), - this.setSurrogateKeyHeader.bind(this) - ]; -}; - MapController.prototype.incrementMapViewCount = function (req, res, callback) { const { mapconfig, user } = res.locals; From 65e8609fec41586d895050a46e25a67a1cb31a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 09:47:46 +0100 Subject: [PATCH 093/167] Do not bind context if not needed --- lib/cartodb/controllers/map.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 7a1dc6be..2d26cf5d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -78,16 +78,16 @@ MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = f useTemplate ? this.getTemplate.bind(this) : this.prepareAdapterMapConfig.bind(this), useTemplate ? this.instantiateLayergroup.bind(this) : this.createLayergroup.bind(this), this.incrementMapViewCount.bind(this), - this.augmentLayergroupData.bind(this), + this.augmentLayergroupData, this.getAffectedTables.bind(this), - this.setCacheChannel.bind(this), - this.setLastUpdatedTimeToLayergroup.bind(this), - this.setCacheControl.bind(this), + this.setCacheChannel, + this.setLastUpdatedTimeToLayergroup, + this.setCacheControl, this.setLayerStats.bind(this), this.setLayergroupIdHeaderBuilder(useTemplateHash), this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), this.setAnalysesMetadataToLayergroupBuilder(includeQuery), - this.setTurboCartoMetadataToLayergroup.bind(this), + this.setTurboCartoMetadataToLayergroup, this.setSurrogateKeyHeader.bind(this), sendResponse, augmentError({ label, addContext }) From 6acb873d959d0f5d9aae3c91147e8fc3cb64e2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 15:06:15 +0100 Subject: [PATCH 094/167] Enforce all middlewares to follow the same constructor pattern --- lib/cartodb/controllers/map.js | 594 +++++++++++++++++---------------- 1 file changed, 313 insertions(+), 281 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 2d26cf5d..ea443253 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -74,356 +74,386 @@ MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = f cors(), userMiddleware, this.prepareContext, - useTemplate ? checkIntantiteLayergroup : checkCreateLayergroup, - useTemplate ? this.getTemplate.bind(this) : this.prepareAdapterMapConfig.bind(this), - useTemplate ? this.instantiateLayergroup.bind(this) : this.createLayergroup.bind(this), - this.incrementMapViewCount.bind(this), - this.augmentLayergroupData, - this.getAffectedTables.bind(this), - this.setCacheChannel, - this.setLastUpdatedTimeToLayergroup, - this.setCacheControl, - this.setLayerStats.bind(this), + useTemplate ? checkIntantiteLayergroup() : checkCreateLayergroup(), + useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(), + useTemplate ? this.instantiateLayergroup() : this.createLayergroup(), + this.incrementMapViewCount(), + this.augmentLayergroupData(), + this.getAffectedTables(), + this.setCacheChannel(), + this.setLastUpdatedTimeToLayergroup(), + this.setCacheControl(), + this.setLayerStats(), this.setLayergroupIdHeaderBuilder(useTemplateHash), - this.setDataviewsAndWidgetsUrlsToLayergroupMetadata.bind(this), - this.setAnalysesMetadataToLayergroupBuilder(includeQuery), - this.setTurboCartoMetadataToLayergroup, - this.setSurrogateKeyHeader.bind(this), - sendResponse, + this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(), + this.setAnalysesMetadataToLayergroup(includeQuery), + this.setTurboCartoMetadataToLayergroup(), + this.setSurrogateKeyHeader(), + sendResponse(), augmentError({ label, addContext }) ]; }; -function checkCreateLayergroup (req, res, next) { - req.profiler.start(`windshaft.createmap_${req.method.toLowerCase()}`); - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('layergroup POST data must be of type application/json')); - } +function checkCreateLayergroup () { + return function checkCreateLayergroupMiddleware (req, res, next) { + req.profiler.start(`windshaft.createmap_${req.method.toLowerCase()}`); - if (req.method === 'GET') { - const { config } = res.locals; - - if (!config) { - return next(new Error('layergroup GET needs a "config" parameter')); + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('layergroup POST data must be of type application/json')); } - try { - req.body = JSON.parse(config); - } catch (err) { - return next(err); - } - } + if (req.method === 'GET') { + const { config } = res.locals; - return next(); -} + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } -function checkIntantiteLayergroup(req, res, next) { - // jshint maxcomplexity: 7 - req.profiler.start(`windshaft-cartodb.instance_template_${req.method.toLowerCase()}`); - - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('Template POST data must be of type application/json')); - } - - if (req.method === 'GET') { - const { callback, config } = req.query; - - if (callback === undefined || callback.length === 0) { - return next(new Error('callback parameter should be present and be a function name')); - } - - if (config) { try { req.body = JSON.parse(config); - } catch(e) { - return next(new Error('Invalid config parameter, should be a valid JSON')); + } catch (err) { + return next(err); } } - } - return next(); -} - -MapController.prototype.prepareAdapterMapConfig = function (req, res, next) { - const requestMapConfig = req.body; - const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; - - const context = { - analysisConfiguration: { - user, - db: { - host: dbhost, - port: dbport, - dbname: dbname, - user: dbuser, - pass: dbpassword - }, - batch: { - username: user, - apiKey: api_key - } - } + return next(); }; - - this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { - if (err) { - return next(err); - } - - req.body = requestMapConfig; - res.locals.context = context; - - next(); - }); -}; - -MapController.prototype.createLayergroup = function(req, res, next) { - const requestMapConfig = req.body; - const { context, user } = res.locals; - const datasource = context.datasource || Datasource.EmptyDatasource(); - const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); - - res.locals.mapconfig = mapconfig; - res.locals.analysesResults = context.analysesResults; - - this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { - if (err) { - return next(err); - } - - res.locals.layergroup = layergroup; - - next(); - }); -}; - -MapController.prototype.getTemplate = function(req, res, next) { - const templateParams = req.body; - const { user } = res.locals; - - const mapconfigProvider = new NamedMapMapConfigProvider( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsApi, - this.mapConfigAdapter, - user, - req.params.template_id, - templateParams, - res.locals.auth_token, - res.locals - ); - - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { - if (err) { - return next(err); - } - - res.locals.mapconfig = mapconfig; - res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; - - next(); - }); -}; - -MapController.prototype.instantiateLayergroup = function(req, res, next) { - const { user, mapconfig, rendererParams } = res.locals; - const mapconfigProvider = - new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); - - this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { - if (err) { - return next(err); - } - - res.locals.layergroup = layergroup; - - // TODO: Do not provide shortcuts - const { mapconfigProvider } = res.locals; - - res.locals.analysesResults = mapconfigProvider.analysesResults; - res.locals.template = mapconfigProvider.template; - res.locals.templateName = mapconfigProvider.getTemplateName(); - res.locals.context = mapconfigProvider.context; - - next(); - }); -}; - -function sendResponse (req, res) { - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } } -MapController.prototype.incrementMapViewCount = function (req, res, callback) { - const { mapconfig, user } = res.locals; +function checkIntantiteLayergroup () { + return function checkIntantiteLayergroupMiddleware(req, res, next) { + // jshint maxcomplexity: 7 + req.profiler.start(`windshaft-cartodb.instance_template_${req.method.toLowerCase()}`); - // Error won't blow up, just be logged. - this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { - req.profiler.done('incMapviewCount'); - - if (err) { - global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + if (req.method === 'POST' && !req.is('application/json')) { + return next(new Error('Template POST data must be of type application/json')); } - callback(); - }); -}; + if (req.method === 'GET') { + const { callback, config } = req.query; -MapController.prototype.augmentLayergroupData = function (req, res, callback) { - const { layergroup } = res.locals; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - _.extend(layergroup, global.environment.serverMetadata); - - callback(); -}; - -MapController.prototype.getAffectedTables = function (req, res, next) { - const { dbname, layergroup, user, mapconfig } = res.locals; - - this.pgConnection.getConnection(user, (err, connection) => { - if (err) { - return next(err); - } - - const sql = []; - mapconfig.getLayers().forEach(function(layer) { - sql.push(layer.options.sql); - if (layer.options.affected_tables) { - layer.options.affected_tables.map(function(table) { - sql.push('SELECT * FROM ' + table + ' LIMIT 0'); - }); + if (callback === undefined || callback.length === 0) { + return next(new Error('callback parameter should be present and be a function name')); } - }); - QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { + if (config) { + try { + req.body = JSON.parse(config); + } catch(e) { + return next(new Error('Invalid config parameter, should be a valid JSON')); + } + } + } + + return next(); + }; +} + +MapController.prototype.prepareAdapterMapConfig = function () { + return function prepareAdapterMapConfigMiddleware(req, res, next) { + const requestMapConfig = req.body; + const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals; + + const context = { + analysisConfiguration: { + user, + db: { + host: dbhost, + port: dbport, + dbname: dbname, + user: dbuser, + pass: dbpassword + }, + batch: { + username: user, + apiKey: api_key + } + } + }; + + this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { if (err) { return next(err); } - // feed affected tables cache so it can be reused from, for instance, layergroup controller - this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); - - res.locals.affectedTables = affectedTables; + req.body = requestMapConfig; + res.locals.context = context; next(); }); - }); + }.bind(this); }; -MapController.prototype.setCacheChannel = function (req, res, callback) { - const { affectedTables } = res.locals; +MapController.prototype.createLayergroup = function () { + return function createLayergroupMiddleware (req, res, next) { + const requestMapConfig = req.body; + const { context, user } = res.locals; + const datasource = context.datasource || Datasource.EmptyDatasource(); + const mapconfig = new MapConfig(requestMapConfig, datasource); + const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); - if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - } + res.locals.mapconfig = mapconfig; + res.locals.analysesResults = context.analysesResults; - callback(); + this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + if (err) { + return next(err); + } + + res.locals.layergroup = layergroup; + + next(); + }); + }.bind(this); }; -MapController.prototype.setLastUpdatedTimeToLayergroup = function (req, res, callback) { - const { affectedTables, layergroup, analysesResults } = res.locals; +MapController.prototype.getTemplate = function () { + return function getTemplateMiddleware (req, res, next) { + const templateParams = req.body; + const { user } = res.locals; - var lastUpdateTime = affectedTables.getLastUpdatedAt(); + const mapconfigProvider = new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsApi, + this.mapConfigAdapter, + user, + req.params.template_id, + templateParams, + res.locals.auth_token, + res.locals + ); - lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + if (err) { + return next(err); + } - // last update for layergroup cache buster - layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; - layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + res.locals.mapconfig = mapconfig; + res.locals.rendererParams = rendererParams; + res.locals.mapconfigProvider = mapconfigProvider; - callback(); + next(); + }); + }.bind(this); }; -MapController.prototype.setCacheControl = function (req, res, callback) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - } +MapController.prototype.instantiateLayergroup = function () { + return function instantiateLayergroupMiddleware (req, res, next) { + const { user, mapconfig, rendererParams } = res.locals; + const mapconfigProvider = + new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); - callback(); + this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + if (err) { + return next(err); + } + + res.locals.layergroup = layergroup; + + // TODO: Do not provide shortcuts + const { mapconfigProvider } = res.locals; + + res.locals.analysesResults = mapconfigProvider.analysesResults; + res.locals.template = mapconfigProvider.template; + res.locals.templateName = mapconfigProvider.getTemplateName(); + res.locals.context = mapconfigProvider.context; + + next(); + }); + }.bind(this); }; -MapController.prototype.setLayerStats = function (req, res, callback) { - const { user, mapconfig, layergroup } = res.locals; +function sendResponse() { + return function sendResponseMiddleware (req, res) { + const { layergroup } = res.locals; - this.pgConnection.getConnection(user, (err, connection) => { - if (err) { - return callback(err); + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); + } else { + res.json(layergroup); + } + }; +} + +MapController.prototype.incrementMapViewCount = function () { + return function incrementMapViewCountMiddleware(req, res, next) { + const { mapconfig, user } = res.locals; + + // Error won't blow up, just be logged. + this.metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => { + req.profiler.done('incMapviewCount'); + + if (err) { + global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`); + } + + next(); + }); + }.bind(this); +}; + +MapController.prototype.augmentLayergroupData = function () { + return function augmentLayergroupDataMiddleware (req, res, next) { + const { layergroup } = res.locals; + + // include in layergroup response the variables in serverMedata + // those variables are useful to send to the client information + // about how to reach this server or information about it + _.extend(layergroup, global.environment.serverMetadata); + + next(); + }; +}; + +MapController.prototype.getAffectedTables = function () { + return function getAffectedTablesMiddleware (req, res, next) { + const { dbname, layergroup, user, mapconfig } = res.locals; + + this.pgConnection.getConnection(user, (err, connection) => { + if (err) { + return next(err); + } + + const sql = []; + mapconfig.getLayers().forEach(function(layer) { + sql.push(layer.options.sql); + if (layer.options.affected_tables) { + layer.options.affected_tables.map(function(table) { + sql.push('SELECT * FROM ' + table + ' LIMIT 0'); + }); + } + }); + + QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { + if (err) { + return next(err); + } + + // feed affected tables cache so it can be reused from, for instance, layergroup controller + this.layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables); + + res.locals.affectedTables = affectedTables; + + next(); + }); + }); + }.bind(this); +}; + +MapController.prototype.setCacheChannel = function () { + return function setCacheChannelMiddleware (req, res, next) { + const { affectedTables } = res.locals; + + if (req.method === 'GET') { + res.set('Last-Modified', (new Date()).toUTCString()); + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); } - this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + next(); + }; +}; + +MapController.prototype.setLastUpdatedTimeToLayergroup = function () { + return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { + const { affectedTables, layergroup, analysesResults } = res.locals; + + var lastUpdateTime = affectedTables.getLastUpdatedAt(); + + lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime; + + // last update for layergroup cache buster + layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; + layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + + next(); + }; +}; + +MapController.prototype.setCacheControl = function () { + return function setCacheControlMiddleware (req, res, next) { + if (req.method === 'GET') { + var ttl = global.environment.varnish.layergroupTtl || 86400; + res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); + } + + next(); + }; +}; + +MapController.prototype.setLayerStats = function () { + return function setLayerStatsMiddleware(req, res, next) { + const { user, mapconfig, layergroup } = res.locals; + + this.pgConnection.getConnection(user, (err, connection) => { if (err) { - return callback(err); + return next(err); } - if (layersStats.length > 0) { - layergroup.metadata.layers.forEach(function (layer, index) { - layer.meta.stats = layersStats[index]; - }); - } + this.statsBackend.getStats(mapconfig, connection, function(err, layersStats) { + if (err) { + return next(err); + } - callback(); + if (layersStats.length > 0) { + layergroup.metadata.layers.forEach(function (layer, index) { + layer.meta.stats = layersStats[index]; + }); + } + + next(); + }); }); - }); + }.bind(this); }; MapController.prototype.setLayergroupIdHeaderBuilder = function (useTemplateHash) { - const self = this; - return function setLayergroupIdHeader(req, res, callback) { + return function setLayergroupIdHeaderMiddleware (req, res, next) { const { layergroup, user, template } = res.locals; if (useTemplateHash) { - var templateHash = self.templateMaps.fingerPrint(template).substring(0, 8); + var templateHash = this.templateMaps.fingerPrint(template).substring(0, 8); layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; } res.set('X-Layergroup-Id', layergroup.layergroupid); - callback(); - }; + next(); + }.bind(); }; -MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function (req, res, callback) { - const { layergroup, user, mapconfig } = res.locals; +MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () { + return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) { + const { layergroup, user, mapconfig } = res.locals; - this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); + this.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj()); - callback(); + next(); + }.bind(this); }; -MapController.prototype.setAnalysesMetadataToLayergroupBuilder = function (includeQuery) { - const self = this; - - return function setAnalysesMetadataToLayergroup (req, res, callback) { +MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) { + return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { const { layergroup, user, analysesResults = [] } = res.locals; - self.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - callback(); - }; + next(); + }.bind(this); }; -MapController.prototype.setTurboCartoMetadataToLayergroup = function (req, res, callback) { - const { layergroup, mapconfig, context } = res.locals; +MapController.prototype.setTurboCartoMetadataToLayergroup = function () { + return function (req, res, next) { + const { layergroup, mapconfig, context } = res.locals; - addContextMetadata(layergroup, mapconfig.obj(), context); + addContextMetadata(layergroup, mapconfig.obj(), context); - callback(); + next(); + }; }; function addContextMetadata(layergroup, mapConfig, context) { @@ -437,18 +467,20 @@ function addContextMetadata(layergroup, mapConfig, context) { } } -MapController.prototype.setSurrogateKeyHeader = function (req, res, callback) { - const { affectedTables, user, templateName } = res.locals; +MapController.prototype.setSurrogateKeyHeader = function () { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { affectedTables, user, templateName } = res.locals; - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - this.surrogateKeysCache.tag(res, affectedTables); - } + if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { + this.surrogateKeysCache.tag(res, affectedTables); + } - if (templateName) { - this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } + if (templateName) { + this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); + } - callback(); + next(); + }.bind(this); }; function getLastUpdatedTime(analysesResults, lastUpdateTime) { @@ -534,7 +566,7 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig function augmentError (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; - return function mapError (err, req, res, next) { + return function augmentErrorMiddleware (err, req, res, next) { const { mapconfig } = res.locals; if (addContext) { From 05ccf2063403dba3efd8472c747bb5bfa16e31a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 15:12:18 +0100 Subject: [PATCH 095/167] Rename function --- lib/cartodb/controllers/map.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ea443253..cae3ff9a 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -84,7 +84,7 @@ MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = f this.setLastUpdatedTimeToLayergroup(), this.setCacheControl(), this.setLayerStats(), - this.setLayergroupIdHeaderBuilder(useTemplateHash), + this.setLayergroupIdHeader(useTemplateHash), this.setDataviewsAndWidgetsUrlsToLayergroupMetadata(), this.setAnalysesMetadataToLayergroup(includeQuery), this.setTurboCartoMetadataToLayergroup(), @@ -411,7 +411,7 @@ MapController.prototype.setLayerStats = function () { }.bind(this); }; -MapController.prototype.setLayergroupIdHeaderBuilder = function (useTemplateHash) { +MapController.prototype.setLayergroupIdHeader = function (useTemplateHash) { return function setLayergroupIdHeaderMiddleware (req, res, next) { const { layergroup, user, template } = res.locals; @@ -423,7 +423,7 @@ MapController.prototype.setLayergroupIdHeaderBuilder = function (useTemplateHash res.set('X-Layergroup-Id', layergroup.layergroupid); next(); - }.bind(); + }.bind(this); }; MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = function () { From 46289f27df6745c6f0cf8e270cd7597b5278fee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 3 Nov 2017 15:26:25 +0100 Subject: [PATCH 096/167] Remove TODO --- lib/cartodb/controllers/map.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index cae3ff9a..ff88b453 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -253,7 +253,6 @@ MapController.prototype.instantiateLayergroup = function () { res.locals.layergroup = layergroup; - // TODO: Do not provide shortcuts const { mapconfigProvider } = res.locals; res.locals.analysesResults = mapconfigProvider.analysesResults; From ac153232d0a6a79dddc33b11c7129282d7e05779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Fri, 3 Nov 2017 17:21:01 +0100 Subject: [PATCH 097/167] mvt timeout error creation script --- assets/render-timeout-fallback.mvt | 1 + scripts/mvt-timeout-error.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 assets/render-timeout-fallback.mvt create mode 100755 scripts/mvt-timeout-error.py diff --git a/assets/render-timeout-fallback.mvt b/assets/render-timeout-fallback.mvt new file mode 100644 index 00000000..1ebfd46d --- /dev/null +++ b/assets/render-timeout-fallback.mvt @@ -0,0 +1 @@ +'\x1aZ\n\x14errorTileSquareLayer\x12\x1b\x12\x06\x00\x00\x01\x01\x02\x02\x18\x03"\x0f\t\x00\x80@\x1a\x00\xff?\x80@\x00\x00\x80@\x0f\x1a\x03foo\x1a\x03uid\x1a\x03cat"\x05\n\x03bar"\x02 {"\x06\n\x04flew(\x80 x\x01\x1aD\n\x03air\x12\x15\x12\x06\x00\x00\x01\x01\x02\x02\x18\x02"\t\t\xbe\x02\xb6\x03\n\x81\x1b\x00\x1a\x03foo\x1a\x03uid\x1a\x03cat"\x05\n\x03bar"\x03 \xd2\t"\x06\n\x04flew(\x80 x\x01' \ No newline at end of file diff --git a/scripts/mvt-timeout-error.py b/scripts/mvt-timeout-error.py new file mode 100755 index 00000000..d8028997 --- /dev/null +++ b/scripts/mvt-timeout-error.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python + +import mapbox_vector_tile + +tile = mapbox_vector_tile.encode([ + { + "name": "errorTileSquareLayer", + "features": [{ + "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))", + "properties":{ + "uid":123, + "foo":"bar", + "cat":"flew" + } + }] + }, + { + "name": "air", + "features": [{ + "geometry":"LINESTRING(159 3877, -1570 3877)", + "properties":{ + "uid":1234, + "foo":"bar", + "cat":"flew" + } + }] + } +]) + +with open('./assets/render-timeout-fallback.mvt', 'w+') as f: + f.write(repr(tile)) \ No newline at end of file From f9ba3c41d306fc89c04ec6057d3d0f15edcf1f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sun, 5 Nov 2017 18:55:23 +0100 Subject: [PATCH 098/167] Create new middlewares to init profiling and another to check JSON content-type --- lib/cartodb/controllers/map.js | 74 ++++++++++--------- .../ported/multilayer_error_cases.js | 2 +- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index ff88b453..93a917b8 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -49,22 +49,18 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m module.exports = MapController; MapController.prototype.register = function(app) { - app.get(app.base_url_mapconfig, this.composeCreateMapMiddleware()); - - app.post(app.base_url_mapconfig, this.composeCreateMapMiddleware()); - - app.get(app.base_url_templated + '/:template_id/jsonp', this.composeCreateMapMiddleware({ - useTemplate: true - })); - - app.post(app.base_url_templated + '/:template_id', this.composeCreateMapMiddleware({ - useTemplate: true - })); + const { base_url_mapconfig, base_url_templated } = app; + const useTemplate = true; + app.get(base_url_mapconfig, this.composeCreateMapMiddleware()); + app.post(base_url_mapconfig, this.composeCreateMapMiddleware()); + app.get(`${base_url_templated}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate)); + app.post(`${base_url_templated}/:template_id`, this.composeCreateMapMiddleware(useTemplate)); app.options(app.base_url_mapconfig, cors('Content-Type')); }; -MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = false } = {}) { +MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) { + const isTemplateInstantiation = useTemplate; const useTemplateHash = useTemplate; const includeQuery = !useTemplate; const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP'; @@ -74,7 +70,9 @@ MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = f cors(), userMiddleware, this.prepareContext, - useTemplate ? checkIntantiteLayergroup() : checkCreateLayergroup(), + this.initProfiler(isTemplateInstantiation), + this.checkJsonContentType(), + useTemplate ? this.checkInstantiteLayergroup() : this.checkCreateLayergroup(), useTemplate ? this.getTemplate() : this.prepareAdapterMapConfig(), useTemplate ? this.instantiateLayergroup() : this.createLayergroup(), this.incrementMapViewCount(), @@ -89,20 +87,32 @@ MapController.prototype.composeCreateMapMiddleware = function ({ useTemplate = f this.setAnalysesMetadataToLayergroup(includeQuery), this.setTurboCartoMetadataToLayergroup(), this.setSurrogateKeyHeader(), - sendResponse(), - augmentError({ label, addContext }) + this.sendResponse(), + this.augmentError({ label, addContext }) ]; }; +MapController.prototype.initProfiler = function (isTemplateInstantiation) { + const operation = isTemplateInstantiation ? 'instance_template' : 'createmap'; -function checkCreateLayergroup () { - return function checkCreateLayergroupMiddleware (req, res, next) { - req.profiler.start(`windshaft.createmap_${req.method.toLowerCase()}`); + return function initProfilerMiddleware (req, res, next) { + req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + next(); + }; +}; +MapController.prototype.checkJsonContentType = function () { + return function checkJsonContentTypeMiddleware(req, res, next) { if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('layergroup POST data must be of type application/json')); + return next(new Error('POST data must be of type application/json')); } + next(); + }; +}; + +MapController.prototype.checkCreateLayergroup = function () { + return function checkCreateLayergroupMiddleware (req, res, next) { if (req.method === 'GET') { const { config } = res.locals; @@ -119,17 +129,10 @@ function checkCreateLayergroup () { return next(); }; -} - -function checkIntantiteLayergroup () { - return function checkIntantiteLayergroupMiddleware(req, res, next) { - // jshint maxcomplexity: 7 - req.profiler.start(`windshaft-cartodb.instance_template_${req.method.toLowerCase()}`); - - if (req.method === 'POST' && !req.is('application/json')) { - return next(new Error('Template POST data must be of type application/json')); - } +}; +MapController.prototype.checkInstantiteLayergroup = function () { + return function checkInstantiteLayergroupMiddleware(req, res, next) { if (req.method === 'GET') { const { callback, config } = req.query; @@ -148,7 +151,7 @@ function checkIntantiteLayergroup () { return next(); }; -} +}; MapController.prototype.prepareAdapterMapConfig = function () { return function prepareAdapterMapConfigMiddleware(req, res, next) { @@ -191,7 +194,8 @@ MapController.prototype.createLayergroup = function () { const { context, user } = res.locals; const datasource = context.datasource || Datasource.EmptyDatasource(); const mapconfig = new MapConfig(requestMapConfig, datasource); - const mapconfigProvider = new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); + const mapconfigProvider = + new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, res.locals); res.locals.mapconfig = mapconfig; res.locals.analysesResults = context.analysesResults; @@ -265,7 +269,7 @@ MapController.prototype.instantiateLayergroup = function () { }.bind(this); }; -function sendResponse() { +MapController.prototype.sendResponse = function () { return function sendResponseMiddleware (req, res) { const { layergroup } = res.locals; @@ -277,7 +281,7 @@ function sendResponse() { res.json(layergroup); } }; -} +}; MapController.prototype.incrementMapViewCount = function () { return function incrementMapViewCountMiddleware(req, res, next) { @@ -562,7 +566,7 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig } }; -function augmentError (options) { +MapController.prototype.augmentError = function (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; return function augmentErrorMiddleware (err, req, res, next) { @@ -576,7 +580,7 @@ function augmentError (options) { next(err); }; -} +}; function populateError(err, mapConfig) { var error = new Error(err.message); diff --git a/test/acceptance/ported/multilayer_error_cases.js b/test/acceptance/ported/multilayer_error_cases.js index 92ce0b1f..9efd7c7b 100644 --- a/test/acceptance/ported/multilayer_error_cases.js +++ b/test/acceptance/ported/multilayer_error_cases.js @@ -28,7 +28,7 @@ describe('multilayer error cases', function() { }, {}, function(res) { assert.equal(res.statusCode, 400, res.body); var parsedBody = JSON.parse(res.body); - assert.deepEqual(parsedBody.errors, ["layergroup POST data must be of type application/json"]); + assert.deepEqual(parsedBody.errors, ["POST data must be of type application/json"]); done(); }); }); From 693a2e7beefe6ca92b0b95b3a4699f61a1ffa0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sun, 5 Nov 2017 19:13:56 +0100 Subject: [PATCH 099/167] Order middlewares --- lib/cartodb/controllers/map.js | 312 ++++++++++++++++----------------- 1 file changed, 156 insertions(+), 156 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 93a917b8..1510dfc2 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -111,26 +111,6 @@ MapController.prototype.checkJsonContentType = function () { }; }; -MapController.prototype.checkCreateLayergroup = function () { - return function checkCreateLayergroupMiddleware (req, res, next) { - if (req.method === 'GET') { - const { config } = res.locals; - - if (!config) { - return next(new Error('layergroup GET needs a "config" parameter')); - } - - try { - req.body = JSON.parse(config); - } catch (err) { - return next(err); - } - } - - return next(); - }; -}; - MapController.prototype.checkInstantiteLayergroup = function () { return function checkInstantiteLayergroupMiddleware(req, res, next) { if (req.method === 'GET') { @@ -153,6 +133,58 @@ MapController.prototype.checkInstantiteLayergroup = function () { }; }; +MapController.prototype.checkCreateLayergroup = function () { + return function checkCreateLayergroupMiddleware (req, res, next) { + if (req.method === 'GET') { + const { config } = res.locals; + + if (!config) { + return next(new Error('layergroup GET needs a "config" parameter')); + } + + try { + req.body = JSON.parse(config); + } catch (err) { + return next(err); + } + } + + return next(); + }; +}; + +MapController.prototype.getTemplate = function () { + return function getTemplateMiddleware (req, res, next) { + const templateParams = req.body; + const { user } = res.locals; + + const mapconfigProvider = new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + this.metadataBackend, + this.userLimitsApi, + this.mapConfigAdapter, + user, + req.params.template_id, + templateParams, + res.locals.auth_token, + res.locals + ); + + mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + if (err) { + return next(err); + } + + res.locals.mapconfig = mapconfig; + res.locals.rendererParams = rendererParams; + res.locals.mapconfigProvider = mapconfigProvider; + + next(); + }); + }.bind(this); +}; + MapController.prototype.prepareAdapterMapConfig = function () { return function prepareAdapterMapConfigMiddleware(req, res, next) { const requestMapConfig = req.body; @@ -212,38 +244,6 @@ MapController.prototype.createLayergroup = function () { }.bind(this); }; -MapController.prototype.getTemplate = function () { - return function getTemplateMiddleware (req, res, next) { - const templateParams = req.body; - const { user } = res.locals; - - const mapconfigProvider = new NamedMapMapConfigProvider( - this.templateMaps, - this.pgConnection, - this.metadataBackend, - this.userLimitsApi, - this.mapConfigAdapter, - user, - req.params.template_id, - templateParams, - res.locals.auth_token, - res.locals - ); - - mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { - if (err) { - return next(err); - } - - res.locals.mapconfig = mapconfig; - res.locals.rendererParams = rendererParams; - res.locals.mapconfigProvider = mapconfigProvider; - - next(); - }); - }.bind(this); -}; - MapController.prototype.instantiateLayergroup = function () { return function instantiateLayergroupMiddleware (req, res, next) { const { user, mapconfig, rendererParams } = res.locals; @@ -269,20 +269,6 @@ MapController.prototype.instantiateLayergroup = function () { }.bind(this); }; -MapController.prototype.sendResponse = function () { - return function sendResponseMiddleware (req, res) { - const { layergroup } = res.locals; - - res.status(200); - - if (req.query && req.query.callback) { - res.jsonp(layergroup); - } else { - res.json(layergroup); - } - }; -}; - MapController.prototype.incrementMapViewCount = function () { return function incrementMapViewCountMiddleware(req, res, next) { const { mapconfig, user } = res.locals; @@ -377,6 +363,19 @@ MapController.prototype.setLastUpdatedTimeToLayergroup = function () { }; }; +function getLastUpdatedTime(analysesResults, lastUpdateTime) { + if (!Array.isArray(analysesResults)) { + return lastUpdateTime; + } + return analysesResults.reduce(function(lastUpdateTime, analysis) { + return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { + var nodeUpdatedAtDate = node.getUpdatedAt(); + var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; + return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; + }, lastUpdateTime); + }, lastUpdateTime); +} + MapController.prototype.setCacheControl = function () { return function setCacheControlMiddleware (req, res, next) { if (req.method === 'GET') { @@ -439,96 +438,6 @@ MapController.prototype.setDataviewsAndWidgetsUrlsToLayergroupMetadata = functio }.bind(this); }; -MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) { - return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { - const { layergroup, user, analysesResults = [] } = res.locals; - - this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); - - next(); - }.bind(this); -}; - -MapController.prototype.setTurboCartoMetadataToLayergroup = function () { - return function (req, res, next) { - const { layergroup, mapconfig, context } = res.locals; - - addContextMetadata(layergroup, mapconfig.obj(), context); - - next(); - }; -}; - -function addContextMetadata(layergroup, mapConfig, context) { - if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { - layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { - if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { - layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; - } - return layer; - }); - } -} - -MapController.prototype.setSurrogateKeyHeader = function () { - return function setSurrogateKeyHeaderMiddleware(req, res, next) { - const { affectedTables, user, templateName } = res.locals; - - if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { - this.surrogateKeysCache.tag(res, affectedTables); - } - - if (templateName) { - this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); - } - - next(); - }.bind(this); -}; - -function getLastUpdatedTime(analysesResults, lastUpdateTime) { - if (!Array.isArray(analysesResults)) { - return lastUpdateTime; - } - return analysesResults.reduce(function(lastUpdateTime, analysis) { - return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) { - var nodeUpdatedAtDate = node.getUpdatedAt(); - var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0; - return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime; - }, lastUpdateTime); - }, lastUpdateTime); -} - -MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) { - includeQuery = includeQuery || false; - analysesResults = analysesResults || []; - layergroup.metadata.analyses = []; - - analysesResults.forEach(function(analysis) { - var nodes = analysis.getNodes(); - layergroup.metadata.analyses.push({ - nodes: nodes.reduce(function(nodesIdMap, node) { - if (node.params.id) { - var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); - var nodeRepr = { - status: node.getStatus(), - url: this.resourceLocator.getUrls(username, nodeResource) - }; - if (includeQuery) { - nodeRepr.query = node.getQuery(); - } - if (node.getStatus() === 'failed') { - nodeRepr.error_message = node.getErrorMessage(); - } - nodesIdMap[node.params.id] = nodeRepr; - } - - return nodesIdMap; - }.bind(this), {}) - }); - }.bind(this)); -}; - // TODO this should take into account several URL patterns MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) { this.addDataviewsUrls(username, layergroup, mapConfig); @@ -566,6 +475,97 @@ MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig } }; +MapController.prototype.setAnalysesMetadataToLayergroup = function (includeQuery) { + return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) { + const { layergroup, user, analysesResults = [] } = res.locals; + + this.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery); + + next(); + }.bind(this); +}; + +MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) { + includeQuery = includeQuery || false; + analysesResults = analysesResults || []; + layergroup.metadata.analyses = []; + + analysesResults.forEach(function(analysis) { + var nodes = analysis.getNodes(); + layergroup.metadata.analyses.push({ + nodes: nodes.reduce(function(nodesIdMap, node) { + if (node.params.id) { + var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); + var nodeRepr = { + status: node.getStatus(), + url: this.resourceLocator.getUrls(username, nodeResource) + }; + if (includeQuery) { + nodeRepr.query = node.getQuery(); + } + if (node.getStatus() === 'failed') { + nodeRepr.error_message = node.getErrorMessage(); + } + nodesIdMap[node.params.id] = nodeRepr; + } + + return nodesIdMap; + }.bind(this), {}) + }); + }.bind(this)); +}; + +MapController.prototype.setTurboCartoMetadataToLayergroup = function () { + return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) { + const { layergroup, mapconfig, context } = res.locals; + + addContextMetadata(layergroup, mapconfig.obj(), context); + + next(); + }; +}; + +function addContextMetadata(layergroup, mapConfig, context) { + if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { + layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { + if (context.turboCarto && Array.isArray(context.turboCarto.layers)) { + layer.meta.cartocss_meta = context.turboCarto.layers[layerIndex]; + } + return layer; + }); + } +} + +MapController.prototype.setSurrogateKeyHeader = function () { + return function setSurrogateKeyHeaderMiddleware(req, res, next) { + const { affectedTables, user, templateName } = res.locals; + + if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) { + this.surrogateKeysCache.tag(res, affectedTables); + } + + if (templateName) { + this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName)); + } + + next(); + }.bind(this); +}; + +MapController.prototype.sendResponse = function () { + return function sendResponseMiddleware (req, res) { + const { layergroup } = res.locals; + + res.status(200); + + if (req.query && req.query.callback) { + res.jsonp(layergroup); + } else { + res.json(layergroup); + } + }; +}; + MapController.prototype.augmentError = function (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; From c48e89826d117225050f6f9dafb8cfc29ab45d1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 7 Nov 2017 09:50:52 +0100 Subject: [PATCH 100/167] Split middleware to follow SRP --- lib/cartodb/controllers/map.js | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 1510dfc2..4c2d6764 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -79,6 +79,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal this.augmentLayergroupData(), this.getAffectedTables(), this.setCacheChannel(), + this.setLastModified(), this.setLastUpdatedTimeToLayergroup(), this.setCacheControl(), this.setLayerStats(), @@ -339,7 +340,6 @@ MapController.prototype.setCacheChannel = function () { const { affectedTables } = res.locals; if (req.method === 'GET') { - res.set('Last-Modified', (new Date()).toUTCString()); res.set('X-Cache-Channel', affectedTables.getCacheChannel()); } @@ -347,6 +347,16 @@ MapController.prototype.setCacheChannel = function () { }; }; +MapController.prototype.setLastModified = function () { + return function setLastModifiedMiddleware (req, res, next) { + if (req.method === 'GET') { + res.set('Last-Modified', (new Date()).toUTCString()); + } + + next(); + }; +}; + MapController.prototype.setLastUpdatedTimeToLayergroup = function () { return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) { const { affectedTables, layergroup, analysesResults } = res.locals; From 33fbff5011b1e1e9a0b8263fb484c0703793f437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 11:06:53 +0100 Subject: [PATCH 101/167] some improvements mvt timeout error script --- assets/render-timeout-fallback.mvt | Bin 394 -> 180 bytes scripts/mvt-timeout-error.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assets/render-timeout-fallback.mvt b/assets/render-timeout-fallback.mvt index 1ebfd46de7ed738db8b9d37413b4ac9528df4557..ee74f5ac795a10a915d3dc315be31cdcc1ae359f 100644 GIT binary patch literal 180 zcmb1&;u1+MD#|Yk$;?R&E-XzfO7%&sOf3?U7Gh&yU}R)sl3-Ti=VWMbkYf06-{8Oi zWbsQer{(8MF_&hhNHHfTmMF1uF()M!DKRNjE3t90q~)ZRYcwcSFiM4Si6YD{DatHJ gEru8_3OAfdiIa05(>7+VMrj5d#xg5h;zThQ0Orpu1poj5 literal 394 zcmb79u@1r@6#SK@OC6!KO*b7J-E=Z64+}|?#zxXcj6d%^D4U6Kc-(u(Cwf^AzDAjZqSHZ$tt5~O_gu0EsuG9O3r?){;sBn~;M{&=vO>JZ zc3M$xo^!j7LvC-hux85^g)-!Ms*5*67sk>si=fLa> Date: Tue, 7 Nov 2017 11:07:38 +0100 Subject: [PATCH 102/167] verctor error middleware --- lib/cartodb/controllers/layergroup.js | 10 ++++++--- lib/cartodb/middleware/vector-error.js | 30 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 lib/cartodb/middleware/vector-error.js diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index c5e4c53a..573c53ee 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -7,6 +7,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); +var vectorError = require('../middleware/vector-error'); var DataviewBackend = require('../backends/dataview'); var AnalysisStatusBackend = require('../backends/analysis-status'); @@ -57,7 +58,8 @@ LayergroupController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.tile.bind(this) + this.tile.bind(this), + vectorError() ); app.get( @@ -65,7 +67,8 @@ LayergroupController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.tile.bind(this) + this.tile.bind(this), + vectorError() ); app.get( @@ -74,7 +77,8 @@ LayergroupController.prototype.register = function(app) { userMiddleware, validateLayerRouteMiddleware, this.prepareContext, - this.layer.bind(this) + this.layer.bind(this), + vectorError() ); app.get( diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js new file mode 100644 index 00000000..3da6ff3b --- /dev/null +++ b/lib/cartodb/middleware/vector-error.js @@ -0,0 +1,30 @@ +const fs = require('fs'); + +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt', {encoding: null}); + +module.exports = function vectorError() { + return function vectorErrorMiddleware(err, req, res, next) { + if(req.params.format === 'mvt') { + + if (isTimeoutError(err)) { + res.set('Content-Type', 'application/x-protobuf'); + return res.status(429).send(timeoutErrorVectorTile); + } + } + + next(err); + } +} + + +function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; +} + +function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); +} + +function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); +} \ No newline at end of file From cda9a09b8eda51e9e00455a9e6572be896bed830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 11:08:01 +0100 Subject: [PATCH 103/167] vector timeout error tests --- test/acceptance/user-database-timeout-limit.js | 17 +++++++++++------ test/acceptance/user-render-timeout-limit.js | 17 +++++++---------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 42c468fd..82934e65 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -411,7 +411,7 @@ describe('user database timeout limit', function () { this.testClient.setUserDatabaseTimeoutLimit(0, done); }); - it('"mvt" fails due to statement timeout', function (done) { + it('"mvt" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'mvt', @@ -419,18 +419,23 @@ describe('user database timeout limit', function () { response: { status: 429, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'application/x-protobuf' } } }; - this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => { + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); - - assert.deepEqual(timeoutError, DATASOURCE_TIMEOUT_ERROR); - + + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + done(); }); + }); }); }); diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 0bb45c2b..5942d045 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -232,7 +232,7 @@ describe('user render timeout limit', function () { response: { status: 429, headers: { - 'Content-Type': 'application/json; charset=utf-8' + 'Content-Type': 'application/x-protobuf' } } }; @@ -240,15 +240,12 @@ describe('user render timeout limit', function () { this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { assert.ifError(err); - assert.deepEqual(tile, { - errors: ['You are over platform\'s limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'datasource', - message: 'You are over platform\'s limits. Please contact us to know more details' - }] - }); - + var tileJSON = tile.toJSON(); + assert.equal(Array.isArray(tileJSON), true); + assert.equal(tileJSON.length, 2); + assert.equal(tileJSON[0].name, 'errorTileSquareLayer'); + assert.equal(tileJSON[1].name, 'errorTileStripesLayer'); + done(); }); }); From 1ea4fc50c94d2912ac3f78d142c1f0214225c7be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 11:17:32 +0100 Subject: [PATCH 104/167] make jshint happy --- lib/cartodb/middleware/vector-error.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index 3da6ff3b..df9f170d 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -1,6 +1,9 @@ const fs = require('fs'); -const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt', {encoding: null}); +const timeoutErrorVectorTile = fs.readFileSync( + __dirname + '/../../../assets/render-timeout-fallback.mvt', + { encoding: null } +); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { @@ -13,8 +16,8 @@ module.exports = function vectorError() { } next(err); - } -} + }; +}; function isRenderTimeoutError (err) { From 50ecdb5feef135718ab17bd53cfb1845877d8fcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 7 Nov 2017 12:51:48 +0100 Subject: [PATCH 105/167] Add test to ensure that categories param is used to compose the aggregation dataview --- test/acceptance/dataviews/aggregation.js | 77 ++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index d8d03177..4a274d80 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -324,3 +324,80 @@ describe('aggregation-dataview: special float values', function() { }); }); }); + +describe('categories param', function () { + afterEach(function(done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + const mapConfig = { + version: '1.5.0', + layers: [ + { + type: "cartodb", + options: { + source: { + "id": "a0" + }, + cartocss: "#points { marker-width: 10; marker-fill: red; }", + cartocss_version: "2.3.0" + } + } + ], + dataviews: { + categories: { + source: { + id: 'a0' + }, + type: 'aggregation', + options: { + column: 'cat', + aggregation: 'sum', + aggregationColumn: 'val' + } + } + }, + analyses: [ + { + id: "a0", + type: "source", + params: { + query: ` + SELECT + null::geometry the_geom_webmercator, + CASE + WHEN x % 4 = 0 THEN 1 + WHEN x % 4 = 1 THEN 2 + WHEN x % 4 = 2 THEN 3 + ELSE 4 + END AS val, + CASE + WHEN x % 4 = 0 THEN 'category_1' + WHEN x % 4 = 1 THEN 'category_2' + WHEN x % 4 = 2 THEN 'category_3' + ELSE 'category_4' + END AS cat + FROM generate_series(1, 1000) x + ` + } + } + ] + }; + + it('should accept cartegories param to customize aggregation dataview', function (done) { + this.testClient = new TestClient(mapConfig, 1234); + const params = { + categories: 2 + }; + + this.testClient.getDataview('categories', params, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categoriesCount, 2); + done(); + }); + }); +}); From 4fc90db495b9191aa339905f2d109aae00548975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 15:57:33 +0100 Subject: [PATCH 106/167] timeout error vector with lines --- assets/render-timeout-fallback.mvt | Bin 180 -> 1113 bytes scripts/mvt-timeout-error.py | 40 +++++++++++++++++------------ 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/assets/render-timeout-fallback.mvt b/assets/render-timeout-fallback.mvt index ee74f5ac795a10a915d3dc315be31cdcc1ae359f..9ef336aeecc0d02037512d329f7115dd05e63ef9 100644 GIT binary patch literal 1113 zcmYM!!EO>!6vpu>MLP490!}H8BT^g@5fKp)5fOL*clrWd%%Vw?>ZpnC&cpbg$NAcR zad-du<=ouOx#yI8q}ZT(Z+C>4=a$ZYWjw(*bRDw!=_!`A*mTqCo^ zE7;24i$gM7yo4?Now$70ZHpJMg})Wo$!zf)HupE;2AM6M!Djwi+#<8ZQ`ppBiDNQb zJb_L8rMOLIi^s6BzYurGZ1D&-^5^1&%oYz}Lw_bt$!zfeHt?t7E}1RPV3|J=_sDE< zAJ+GCai7c<_h3DLEY8SmaTnJ0N8$mQEly#nKNJtiY;gih{DF8xW{W$pj-QFgWVW~s zYx}8qLS~C&SnT)3Q!-oJg0;LUo{`z&2CU&H;yIZuuEXknEMAb=V!+^$cu8i9Ls;lX z;uV=KuEA=4PrN3x#Z_3<55*fYTU>!v{6M@Vv+L!?hBy0M{sHlESw~S2luH!cQF%hS Q?sxatz8v&jaPv+41C0vn&j0`b literal 180 zcmb1&;u1+MD#|Yk$;?R&E-XzfO7%&sOf3?U7Gh&yU}R)sl3-Ti=VWMbkYf06-{8Oi zWbsQer{(8MF_&hhNHHfTmMF1uF()M!DKRNjE3t90q~)ZRYcwcSFiM4Si6YD{DatHJ gEru8_3OAfdiIa05(>7+VMrj5d#xg5h;zThQ0Orpu1poj5 diff --git a/scripts/mvt-timeout-error.py b/scripts/mvt-timeout-error.py index a6b764b6..1b2e66ee 100755 --- a/scripts/mvt-timeout-error.py +++ b/scripts/mvt-timeout-error.py @@ -2,30 +2,36 @@ import mapbox_vector_tile +lines_list = [] + +# main diagonal line +lines_list.append({ "geometry":"LINESTRING (0 0, 4096 4096)"}) + +# diagonal lines +for i in range(4096/32, 4096, 4096/32): + start = i + end = 4096 - i + + lines_list.append({ "geometry":"LINESTRING (0 " + str(start) + ", " + str(end) + " 4096)" }) + lines_list.append({ "geometry":"LINESTRING (" + str(start) + " 0, 4096 " + str(end) + ")" }) + +# box lines +lines_list.append({ "geometry":"LINESTRING (0 0, 0 4096)"}) +lines_list.append({ "geometry":"LINESTRING (0 4096, 4096 4096)"}) +lines_list.append({ "geometry":"LINESTRING (4096 4096, 4096 0)"}) +lines_list.append({ "geometry":"LINESTRING (4096 0, 0 0)"}) + + tile = mapbox_vector_tile.encode([ { "name": "errorTileSquareLayer", - "features": [{ - "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))", - "properties":{ - "uid":123, - "foo":"bar", - "cat":"flew" - } - }] + "features": [{ "geometry":"POLYGON ((0 0, 0 4096, 4096 4096, 4096 0, 0 0))" }] }, { "name": "errorTileStripesLayer", - "features": [{ - "geometry":"LINESTRING(159 3877, -1570 3877)", - "properties":{ - "uid":1234, - "foo":"bar", - "cat":"flew" - } - }] + "features": lines_list } ]) -with open('./assets/render-timeout-fallback.mvt', 'w') as f: +with open('./assets/render-timeout-fallback.mvt', 'w+') as f: f.write(tile) \ No newline at end of file From 970310bf7f4a3cffdeb257053886319064859479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 16:01:08 +0100 Subject: [PATCH 107/167] add vectorError middleware to named map endpoint --- lib/cartodb/controllers/named_maps.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 759ae4b6..442db927 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -9,6 +9,7 @@ var BaseController = require('./base'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); var allowQueryParams = require('../middleware/allow-query-params'); +var vectorError = require('../middleware/vector-error'); function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend, surrogateKeysCache, tablesExtentApi, metadataBackend) { @@ -33,7 +34,8 @@ NamedMapsController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.tile.bind(this) + this.tile.bind(this), + vectorError() ); app.get( @@ -108,6 +110,7 @@ NamedMapsController.prototype.tile = function(req, res, next) { }, function handleImage(err, tile, headers, stats) { req.profiler.add(stats); + if (err) { err.label = 'NAMED_MAP_TILE'; next(err); From 0bf36fa058bf362fba4176e93f46ae19562d80f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Tue, 7 Nov 2017 16:02:37 +0100 Subject: [PATCH 108/167] remove unneeded encoding control --- lib/cartodb/middleware/vector-error.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index df9f170d..c4674cfd 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -1,9 +1,6 @@ const fs = require('fs'); -const timeoutErrorVectorTile = fs.readFileSync( - __dirname + '/../../../assets/render-timeout-fallback.mvt', - { encoding: null } -); +const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt'); module.exports = function vectorError() { return function vectorErrorMiddleware(err, req, res, next) { From 743bb0723b88b2b2ac43e4af22cfebbd8b111593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 7 Nov 2017 16:14:47 +0100 Subject: [PATCH 109/167] Add query param to define the number of categories to be ranked --- lib/cartodb/backends/dataview.js | 4 +- lib/cartodb/controllers/layergroup.js | 3 +- lib/cartodb/models/dataview/aggregation.js | 6 ++- test/acceptance/dataviews/aggregation.js | 45 +++++++++++++++++----- test/support/test-client.js | 2 +- 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/lib/cartodb/backends/dataview.js b/lib/cartodb/backends/dataview.js index b6037ae6..9df22868 100644 --- a/lib/cartodb/backends/dataview.js +++ b/lib/cartodb/backends/dataview.js @@ -24,7 +24,7 @@ module.exports = DataviewBackend; DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, params, callback) { - var dataviewName = params.dataviewName; + var dataviewName = params.dataviewName; step( function getMapConfig() { mapConfigProvider.getMapConfig(this); @@ -94,7 +94,7 @@ function getQueryRewriteData(mapConfig, dataviewDefinition, params) { } function getOverrideParams(params, ownFilter) { - var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset'), + var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins', 'offset', 'categories'), function castNumbers(overrides, val, k) { if (!Number.isFinite(+val)) { throw new Error('Invalid number format for parameter \'' + k + '\''); diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index dc6f85ff..458951c2 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -109,7 +109,8 @@ LayergroupController.prototype.register = function(app) { 'bins', // number 'aggregation', //string 'offset', // number - 'q' // widgets search + 'q', // widgets search + 'categories', // number ]; app.get( diff --git a/lib/cartodb/models/dataview/aggregation.js b/lib/cartodb/models/dataview/aggregation.js index b4c59af2..40774353 100644 --- a/lib/cartodb/models/dataview/aggregation.js +++ b/lib/cartodb/models/dataview/aggregation.js @@ -245,6 +245,10 @@ module.exports = class Aggregation extends BaseDataview { return null; } + const limit = Number.isFinite(override.categories) && override.categories > 0 ? + override.categories : + CATEGORIES_LIMIT; + const aggregationSql = aggregationDataviewQueryTpl({ override: override, query: this.query, @@ -256,7 +260,7 @@ module.exports = class Aggregation extends BaseDataview { aggregationColumn: this.aggregationColumn || 1 }), isFloatColumn: this._isFloatColumn, - limit: CATEGORIES_LIMIT + limit }); debug(aggregationSql); diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 4a274d80..45189213 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -325,7 +325,7 @@ describe('aggregation-dataview: special float values', function() { }); }); -describe('categories param', function () { +describe('aggregation dataview tuned by categories query param', function () { afterEach(function(done) { if (this.testClient) { this.testClient.drain(done); @@ -388,16 +388,41 @@ describe('categories param', function () { ] }; - it('should accept cartegories param to customize aggregation dataview', function (done) { - this.testClient = new TestClient(mapConfig, 1234); - const params = { - categories: 2 - }; + var scenarios = [ + { + params: { own_filter: 0, categories: -1 }, + categoriesExpected: 4 + }, + { + params: { own_filter: 0, categories: 0 }, + categoriesExpected: 4 + }, + { + params: { own_filter: 0, categories: 1 }, + categoriesExpected: 1 + }, + { + params: { own_filter: 0, categories: 2 }, + categoriesExpected: 2 + }, + { + params: { own_filter: 0, categories: 4 }, + categoriesExpected: 4 + }, + { + params: { own_filter: 0, categories: 5 }, + categoriesExpected: 4 + } + ]; - this.testClient.getDataview('categories', params, (err, dataview) => { - assert.ifError(err); - assert.equal(dataview.categoriesCount, 2); - done(); + scenarios.forEach(function (scenario) { + it(`should handle cartegories to customize aggregations: ${JSON.stringify(scenario.params)}`, function (done) { + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('categories', scenario.params, (err, dataview) => { + assert.ifError(err); + assert.equal(dataview.categories.length, scenario.categoriesExpected); + done(); + }); }); }); }); diff --git a/test/support/test-client.js b/test/support/test-client.js index cd7b4f60..f31f9db4 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -415,7 +415,7 @@ TestClient.prototype.getDataview = function(dataviewName, params, callback) { own_filter: params.hasOwnProperty('own_filter') ? params.own_filter : 1 }; - ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset'].forEach(function(extraParam) { + ['bbox', 'bins', 'start', 'end', 'aggregation', 'offset', 'categories'].forEach(function(extraParam) { if (params.hasOwnProperty(extraParam)) { urlParams[extraParam] = params[extraParam]; } From 9149f72f42d5405c6a96c8df3187e825a2536a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 7 Nov 2017 16:21:55 +0100 Subject: [PATCH 110/167] Update NEWS --- NEWS.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 6312c5bb..78a1be0a 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,11 @@ # Changelog -## 4.0.2 +## 4.1.0 Released 2017-mm-dd Announcements: - Upgrades windshaft to [4.0.1](https://github.com/CartoDB/windshaft/releases/tag/4.0.1). + - Add `categories` query param to define the number of cateries to be ranked for aggregation dataviews. ## 4.0.1 From fc9dce0cca7699b532b8f2a8243627a2a953556a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 7 Nov 2017 16:22:36 +0100 Subject: [PATCH 111/167] Fix typo --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 78a1be0a..409b5ebc 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,7 +5,7 @@ Released 2017-mm-dd Announcements: - Upgrades windshaft to [4.0.1](https://github.com/CartoDB/windshaft/releases/tag/4.0.1). - - Add `categories` query param to define the number of cateries to be ranked for aggregation dataviews. + - Add `categories` query param to define the number of categories to be ranked for aggregation dataviews. ## 4.0.1 From 2aee357006b3ccec62e1f5fbdfb3ac818d99c5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 7 Nov 2017 16:28:37 +0100 Subject: [PATCH 112/167] Improve test structure --- test/acceptance/dataviews/aggregation.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/acceptance/dataviews/aggregation.js b/test/acceptance/dataviews/aggregation.js index 45189213..e29d9ff4 100644 --- a/test/acceptance/dataviews/aggregation.js +++ b/test/acceptance/dataviews/aggregation.js @@ -326,14 +326,6 @@ describe('aggregation-dataview: special float values', function() { }); describe('aggregation dataview tuned by categories query param', function () { - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - const mapConfig = { version: '1.5.0', layers: [ @@ -388,6 +380,14 @@ describe('aggregation dataview tuned by categories query param', function () { ] }; + beforeEach(function () { + this.testClient = new TestClient(mapConfig, 1234); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + var scenarios = [ { params: { own_filter: 0, categories: -1 }, @@ -417,7 +417,6 @@ describe('aggregation dataview tuned by categories query param', function () { scenarios.forEach(function (scenario) { it(`should handle cartegories to customize aggregations: ${JSON.stringify(scenario.params)}`, function (done) { - this.testClient = new TestClient(mapConfig, 1234); this.testClient.getDataview('categories', scenario.params, (err, dataview) => { assert.ifError(err); assert.equal(dataview.categories.length, scenario.categoriesExpected); From 56d33b7f5b36336f4e09b931b94a2bdd179ee47f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 8 Nov 2017 10:52:25 +0100 Subject: [PATCH 113/167] Add profiling --- lib/cartodb/controllers/map.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4c2d6764..936c558e 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -173,6 +173,7 @@ MapController.prototype.getTemplate = function () { ); mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { + req.profiler.done('getMapConfig'); if (err) { return next(err); } @@ -209,6 +210,7 @@ MapController.prototype.prepareAdapterMapConfig = function () { }; this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { + req.profiler.done('getMapConfig'); if (err) { return next(err); } @@ -234,6 +236,7 @@ MapController.prototype.createLayergroup = function () { res.locals.analysesResults = context.analysesResults; this.mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); if (err) { return next(err); } @@ -252,6 +255,7 @@ MapController.prototype.instantiateLayergroup = function () { new CreateLayergroupMapConfigProvider(mapconfig, user, this.userLimitsApi, rendererParams); this.mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => { + req.profiler.done('createLayergroup'); if (err) { return next(err); } @@ -320,6 +324,7 @@ MapController.prototype.getAffectedTables = function () { }); QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => { + req.profiler.done('getAffectedTablesFromQuery'); if (err) { return next(err); } @@ -564,6 +569,7 @@ MapController.prototype.setSurrogateKeyHeader = function () { MapController.prototype.sendResponse = function () { return function sendResponseMiddleware (req, res) { + req.profiler.done('res'); const { layergroup } = res.locals; res.status(200); @@ -580,6 +586,7 @@ MapController.prototype.augmentError = function (options) { const { addContext = false, label = 'MAPS CONTROLLER' } = options; return function augmentErrorMiddleware (err, req, res, next) { + req.profiler.done('error'); const { mapconfig } = res.locals; if (addContext) { From aff5fcda63bc6bd6b152d439cf7562428a6f725f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 8 Nov 2017 14:08:27 +0100 Subject: [PATCH 114/167] Add namespace for getMapConfig step while profiling --- lib/cartodb/controllers/map.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 936c558e..72938009 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -173,7 +173,7 @@ MapController.prototype.getTemplate = function () { ); mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => { - req.profiler.done('getMapConfig'); + req.profiler.done('named.getMapConfig'); if (err) { return next(err); } @@ -210,7 +210,7 @@ MapController.prototype.prepareAdapterMapConfig = function () { }; this.mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => { - req.profiler.done('getMapConfig'); + req.profiler.done('anonymous.getMapConfig'); if (err) { return next(err); } From 6f59c61c8b8f8888dd81979e41808f5c398ad72c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 8 Nov 2017 14:27:35 +0100 Subject: [PATCH 115/167] Add more steps while profiling --- lib/cartodb/controllers/map.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 72938009..6b50fc47 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -98,6 +98,7 @@ MapController.prototype.initProfiler = function (isTemplateInstantiation) { return function initProfilerMiddleware (req, res, next) { req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`); + req.profiler.done(`${operation}.initProfilerMiddleware`); next(); }; }; @@ -108,6 +109,8 @@ MapController.prototype.checkJsonContentType = function () { return next(new Error('POST data must be of type application/json')); } + req.profiler.done('checkJsonContentTypeMiddleware'); + next(); }; }; @@ -130,6 +133,8 @@ MapController.prototype.checkInstantiteLayergroup = function () { } } + req.profiler.done('checkInstantiteLayergroup'); + return next(); }; }; @@ -150,6 +155,7 @@ MapController.prototype.checkCreateLayergroup = function () { } } + req.profiler.done('checkCreateLayergroup'); return next(); }; }; From 590233e3ee32a1cee507f8adabdcd334e2c26c9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 13 Nov 2017 12:15:12 +0100 Subject: [PATCH 116/167] Inline method to set cache-cannel, rollback extract setCacheChannel method --- lib/cartodb/controllers/layergroup.js | 62 +++++++++++++-------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 98616484..208ac027 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -386,6 +386,8 @@ LayergroupController.prototype.staticMap = function(req, res, width, height, zoo }; LayergroupController.prototype.sendResponse = function(req, res, body, status, headers) { + var self = this; + req.profiler.done('res'); res.set('Cache-Control', 'public,max-age=31536000'); @@ -400,48 +402,42 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h } res.set('Last-Modified', lastUpdated.toUTCString()); - this.setCacheChannel(req, res, (err) => { - if (err) { - global.logger.warn('ERROR generating cache channel: ' + err); - } + var dbName = res.locals.dbname; + step( + function getAffectedTables() { + self.getAffectedTables(res.locals.user, dbName, res.locals.token, this); + }, + function sendResponse(err, affectedTables) { + req.profiler.done('affectedTables'); + if (err) { + global.logger.warn('ERROR generating cache channel: ' + err); + } + if (!!affectedTables) { + res.set('X-Cache-Channel', affectedTables.getCacheChannel()); + self.surrogateKeysCache.tag(res, affectedTables); + } - if (headers) { - res.set(headers); - } + if (headers) { + res.set(headers); + } - res.status(status); + res.status(status); - if (!Buffer.isBuffer(body) && typeof body === 'object') { - if (req.query && req.query.callback) { - res.jsonp(body); + if (!Buffer.isBuffer(body) && typeof body === 'object') { + if (req.query && req.query.callback) { + res.jsonp(body); + } else { + res.json(body); + } } else { - res.json(body); + res.send(body); } - } else { - res.send(body); } - }); -}; - -LayergroupController.prototype.setCacheChannel = function(req, res, callback) { - const { dbname, user, token } = res.locals; - - this.getAffectedTables(user, dbname, token, (err, affectedTables) => { - req.profiler.done('affectedTables'); - if (err) { - return callback(err); - } - - if (!!affectedTables) { - res.set('X-Cache-Channel', affectedTables.getCacheChannel()); - this.surrogateKeysCache.tag(res, affectedTables); - } - - callback(); - }); + ); }; LayergroupController.prototype.getAffectedTables = function(user, dbName, layergroupId, callback) { + if (this.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId)) { return callback(null, this.layergroupAffectedTables.get(dbName, layergroupId)); } From 38ca5db51b0157f3af812f7b15b8940f6b34b838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 13 Nov 2017 12:35:46 +0100 Subject: [PATCH 117/167] Inline method to set cache-cannel, rollback extract setCacheChannel method for named maps controller --- lib/cartodb/controllers/named_maps.js | 77 +++++++++++---------------- 1 file changed, 32 insertions(+), 45 deletions(-) diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index ffea03a2..55790bbf 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -39,56 +39,43 @@ NamedMapsController.prototype.register = function(app) { ); }; -NamedMapsController.prototype.sendResponse = function(req, res, body, headers) { - const { namedMapProvider } = res.locals; - +NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) { this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName())); res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png'); res.set('Cache-Control', 'public,max-age=7200,must-revalidate'); - this.setCacheChannel(req, res, (err) => { - if (err) { - global.logger.log('ERROR generating cache channel: ' + err); - } + var self = this; - res.status(200); - res.send(body); - }); -}; - -NamedMapsController.prototype.setCacheChannel = function(req, res, callback) { - const { namedMapProvider } = res.locals; - - this.getAffectedTables(namedMapProvider, (err, result) => { - - req.profiler.done('affectedTables'); - if (err) { - global.logger.log('ERROR generating cache channel: ' + err); - } - if (!result || !!result.tables) { - // we increase cache control as we can invalidate it - res.set('Cache-Control', 'public,max-age=31536000'); - - var lastModifiedDate; - if (Number.isFinite(result.lastUpdatedTime)) { - lastModifiedDate = new Date(result.getLastUpdatedAt()); - } else { - lastModifiedDate = new Date(); + step( + function getAffectedTablesAndLastUpdatedTime() { + namedMapProvider.getAffectedTablesAndLastUpdatedTime(this); + }, + function sendResponse(err, result) { + req.profiler.done('affectedTables'); + if (err) { + global.logger.log('ERROR generating cache channel: ' + err); } - res.set('Last-Modified', lastModifiedDate.toUTCString()); + if (!result || !!result.tables) { + // we increase cache control as we can invalidate it + res.set('Cache-Control', 'public,max-age=31536000'); - res.set('X-Cache-Channel', result.getCacheChannel()); - if (result.tables.length > 0) { - this.surrogateKeysCache.tag(res, result); + var lastModifiedDate; + if (Number.isFinite(result.lastUpdatedTime)) { + lastModifiedDate = new Date(result.getLastUpdatedAt()); + } else { + lastModifiedDate = new Date(); + } + res.set('Last-Modified', lastModifiedDate.toUTCString()); + + res.set('X-Cache-Channel', result.getCacheChannel()); + if (result.tables.length > 0) { + self.surrogateKeysCache.tag(res, result); + } } + res.status(200); + res.send(body); } - - callback(); - }); -}; - -NamedMapsController.prototype.getAffectedTables = function(namedMapProvider, callback) { - namedMapProvider.getAffectedTablesAndLastUpdatedTime(callback); + ); }; NamedMapsController.prototype.tile = function(req, res, next) { @@ -110,7 +97,7 @@ NamedMapsController.prototype.tile = function(req, res, next) { }, function getTile(err, _namedMapProvider) { assert.ifError(err); - res.locals.namedMapProvider = namedMapProvider = _namedMapProvider; + namedMapProvider = _namedMapProvider; self.tileBackend.getTile(namedMapProvider, req.params, this); }, function handleImage(err, tile, headers, stats) { @@ -119,7 +106,7 @@ NamedMapsController.prototype.tile = function(req, res, next) { err.label = 'NAMED_MAP_TILE'; next(err); } else { - self.sendResponse(req, res, tile, headers); + self.sendResponse(req, res, tile, headers, namedMapProvider); } } ); @@ -149,7 +136,7 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { function prepareLayerVisibility(err, _namedMapProvider) { assert.ifError(err); - res.locals.namedMapProvider = namedMapProvider = _namedMapProvider; + namedMapProvider = _namedMapProvider; self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this); }, @@ -192,7 +179,7 @@ NamedMapsController.prototype.staticMap = function(req, res, next) { err.label = 'STATIC_VIZ_MAP'; next(err); } else { - self.sendResponse(req, res, image, headers); + self.sendResponse(req, res, image, headers, namedMapProvider); } } ); From 172b3ece7128f73399d0ddffb3f32c646836365b Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 13 Nov 2017 14:36:14 +0100 Subject: [PATCH 118/167] Test: Strip PARALLEL labels for PostgreSQL releases before 9.6 --- test/support/prepare_db.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index dcf1cba7..41f0fb8c 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -13,6 +13,7 @@ PREPARE_REDIS=yes PREPARE_PGSQL=yes DOWNLOAD_SQL_FILES=yes +PG_PARALLEL=$(pg_config --version | (awk '{$2*=1000; if ($2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0)) while [ -n "$1" ]; do if test "$1" = "--skip-pg"; then @@ -92,6 +93,11 @@ if test x"$PREPARE_PGSQL" = xyes; then ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}" for i in ${ALL_SQL_SCRIPTS} do + # Strip PARALLEL labels for PostgreSQL releases before 9.6 + if [ $PG_PARALLEL -eq 0 ]; then + sed -e 's/PARALLEL \= [A-Z]*,/''/g' \ + -e 's/PARALLEL [A-Z]*/''/g' -i sql/$i.sql + fi cat sql/${i}.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | sed "s/:PUBLICUSER/${PUBLICUSER}/" | From bdf4827300c44581e908b69fbe5b2a458627ff0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 14 Nov 2017 11:28:06 +0100 Subject: [PATCH 119/167] Fix bad call to next middleware callback --- lib/cartodb/controllers/analyses.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index ffc0bea1..e97804f9 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -84,7 +84,7 @@ AnalysesController.prototype.catalog = function (req, res, next) { err.http_status = 401; } - next(req, res, err); + next(err); } else { self.sendResponse(req, res, { catalog: catalogWithTables }); } From 27521964c7cb826233418b4304ecbb85c2240adf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 14 Nov 2017 12:01:21 +0100 Subject: [PATCH 120/167] Remove schema from analyses catalog query perfoming that query in the same way that we do for pooling the node status in analyses backend --- lib/cartodb/controllers/analyses.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index e97804f9..f183e834 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -93,7 +93,7 @@ AnalysesController.prototype.catalog = function (req, res, next) { }; var catalogQueryTpl = dot.template( - 'SELECT analysis_def->>\'type\' as type, * FROM cartodb.cdb_analysis_catalog WHERE username = \'{{=it._username}}\'' + 'SELECT analysis_def->>\'type\' as type, * FROM cdb_analysis_catalog WHERE username = \'{{=it._username}}\'' ); var tablesQueryTpl = dot.template([ From 520e84e46b37b7bddff0c3dc1995926a638efc69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 14 Nov 2017 13:46:47 +0100 Subject: [PATCH 121/167] Add acceptance test for analyses controller --- .../analysis/analyses-controller.js | 114 ++++++++++++++++++ test/support/test-client.js | 38 ++++++ 2 files changed, 152 insertions(+) create mode 100644 test/acceptance/analysis/analyses-controller.js diff --git a/test/acceptance/analysis/analyses-controller.js b/test/acceptance/analysis/analyses-controller.js new file mode 100644 index 00000000..2591f0e1 --- /dev/null +++ b/test/acceptance/analysis/analyses-controller.js @@ -0,0 +1,114 @@ +require('../../support/test_helper'); + +var assert = require('../../support/assert'); +var TestClient = require('../../support/test-client'); + + +describe('analyses controller', function () { + const mapConfig = { + version: '1.5.0', + layers: + [{ + type: 'cartodb', + options: + { + source: { id: 'a1' }, + cartocss: TestClient.CARTOCSS.POLYGONS, + cartocss_version: '2.3.0' + } + }], + dataviews: {}, + analyses: + [{ + id: 'a1', + type: 'buffer', + params: { + source: { + type: 'source', + params: { + query: 'select * from analysis_banks limit 1' + } + }, + radius: 250 + } + }] + }; + + beforeEach(function () { + this.testClient = new TestClient(mapConfig, 1234); + }); + + it('should get an array of analyses from catalog', function (done) { + this.testClient.getAnalysesCatalog({}, (err, result) => { + if (err) { + return done(err); + } + + assert.ok(Array.isArray(result.catalog)); + done(); + }); + }); + + it('should support jsonp responses', function (done) { + this.testClient.getAnalysesCatalog({ jsonp: 'jsonp_test' }, (err, result) => { + if (err) { + return done(err); + } + + assert.ok(result); + + let didRunJsonCallback = false; + // jshint ignore:start + function jsonp_test(body) { + assert.ok(Array.isArray(body.catalog)); + didRunJsonCallback = true; + } + + eval(result); + // jshint ignore:end + + assert.ok(didRunJsonCallback); + + done(); + }); + }); + + it('should respond "unauthorized" when missing api_key', function (done) { + const apiKey = this.testClient.apiKey; + this.testClient.apiKey = null; + + this.testClient.getAnalysesCatalog({ status: 401 }, (err, result) => { + if (err) { + return done(err); + } + + assert.deepEqual(result.errors[0], 'Unauthorized'); + this.testClient.apiKey = apiKey; + done(); + }); + }); + + it('should get an array of analyses from catalog', function (done) { + this.testClient.getTile(0, 0, 0, (err) => { + if (err) { + return done(err); + } + + this.testClient.getAnalysesCatalog({}, (err, result) => { + if (err) { + return done(err); + } + + assert.ok(Array.isArray(result.catalog)); + assert.ok(result.catalog.length >= 2); // buffer & source at least + + result.catalog + .filter(analysis => analysis.node_id === '0a215e1f3405381cf0ea6b3b0deb6fdcfdc2fcaa') + .forEach(analysis => assert.equal(analysis.type, 'buffer')); + + this.testClient.drain(done); + }); + + }); + }); +}); \ No newline at end of file diff --git a/test/support/test-client.js b/test/support/test-client.js index f31f9db4..3188d0fc 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1174,4 +1174,42 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb ); }; +TestClient.prototype.getAnalysesCatalog = function (params, callback) { + var url = '/api/v1/map/analyses/catalog'; + if (this.apiKey) { + url += '?' + qs.stringify({api_key: this.apiKey}); + } + + if (params.jsonp) { + url += '&' + qs.stringify({callback: params.jsonp}); + } + + assert.response(this.server, + { + url: url, + method: 'GET', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + } + }, + { + status: params.status || 200, + headers: { + 'Content-Type': params.jsonp ? + 'text/javascript; charset=utf-8' : + 'application/json; charset=utf-8' + } + }, + function(res, err) { + if (err) { + return callback(err); + } + + var parsedBody = params.jsonp ? res.body : JSON.parse(res.body); + + return callback(null, parsedBody); + } + ); +}; From a5347c27e317efb38662a5bb084b5531adc3343d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 14 Nov 2017 13:49:12 +0100 Subject: [PATCH 122/167] Move sendResponse method to a middleware --- lib/cartodb/controllers/analyses.js | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index f183e834..52c2976e 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -18,24 +18,26 @@ AnalysesController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.catalog.bind(this) + this.catalog.bind(this), + this.sendResponse() ); }; -AnalysesController.prototype.sendResponse = function(req, res, body) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); +AnalysesController.prototype.sendResponse = function() { + return function sendResponseMiddleware (req, res) { + res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - res.status(200); + res.status(200); - if (req.query && req.query.callback) { - res.jsonp(body); - } else { - res.json(body); - } + if (req.query && req.query.callback) { + res.jsonp(res.body); + } else { + res.json(res.body); + } + }; }; AnalysesController.prototype.catalog = function (req, res, next) { - var self = this; var username = res.locals.user; step( @@ -77,7 +79,7 @@ AnalysesController.prototype.catalog = function (req, res, next) { return -1; }); }, - function sendResponse(err, catalogWithTables) { + function prepareResponse(err, catalogWithTables) { if (err) { if (err.message.match(/permission\sdenied/)) { err = new Error('Unauthorized'); @@ -86,7 +88,8 @@ AnalysesController.prototype.catalog = function (req, res, next) { next(err); } else { - self.sendResponse(req, res, { catalog: catalogWithTables }); + res.body = { catalog: catalogWithTables }; + next(); } } ); From d5c552a03a8e91ffb053578b0b9455d9486091e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 14 Nov 2017 13:53:42 +0100 Subject: [PATCH 123/167] Move functionality that sets cache control header to a middleware --- lib/cartodb/controllers/analyses.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 52c2976e..5d6ac238 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -19,14 +19,20 @@ AnalysesController.prototype.register = function(app) { userMiddleware, this.prepareContext, this.catalog.bind(this), + this.setCacheControlHeader(), this.sendResponse() ); }; +AnalysesController.prototype.setCacheControlHeader = function () { + return function setCacheControlHeaderMiddleware (req, res, next) { + res.set('Cache-Control', 'public,max-age=10,must-revalidate'); + next(); + }; +}; + AnalysesController.prototype.sendResponse = function() { return function sendResponseMiddleware (req, res) { - res.set('Cache-Control', 'public,max-age=10,must-revalidate'); - res.status(200); if (req.query && req.query.callback) { From 6aaa5f99e253d551e1b2bbbfeac9d4243dfc9df2 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 14 Nov 2017 16:01:57 +0100 Subject: [PATCH 124/167] Test: PARALLEL compatibility (OSX) --- test/support/prepare_db.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 41f0fb8c..b31b2c9b 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -95,8 +95,10 @@ if test x"$PREPARE_PGSQL" = xyes; then do # Strip PARALLEL labels for PostgreSQL releases before 9.6 if [ $PG_PARALLEL -eq 0 ]; then + TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXXXX) sed -e 's/PARALLEL \= [A-Z]*,/''/g' \ - -e 's/PARALLEL [A-Z]*/''/g' -i sql/$i.sql + -e 's/PARALLEL [A-Z]*/''/g' sql/$i.sql > $TMPFILE + mv $TMPFILE sql/$i.sql fi cat sql/${i}.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" | From 298882f4100c891cf9236835a78b49aed7245766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 15 Nov 2017 15:07:30 +0100 Subject: [PATCH 125/167] remove with spaces and adding line as EOF --- lib/cartodb/middleware/vector-error.js | 2 +- test/acceptance/user-database-timeout-limit.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/middleware/vector-error.js b/lib/cartodb/middleware/vector-error.js index c4674cfd..f42f1c87 100644 --- a/lib/cartodb/middleware/vector-error.js +++ b/lib/cartodb/middleware/vector-error.js @@ -27,4 +27,4 @@ function isDatasourceTimeoutError (err) { function isTimeoutError (err) { return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); -} \ No newline at end of file +} diff --git a/test/acceptance/user-database-timeout-limit.js b/test/acceptance/user-database-timeout-limit.js index 82934e65..9893f00a 100644 --- a/test/acceptance/user-database-timeout-limit.js +++ b/test/acceptance/user-database-timeout-limit.js @@ -411,7 +411,7 @@ describe('user database timeout limit', function () { this.testClient.setUserDatabaseTimeoutLimit(0, done); }); - it('"mvt" fails due to statement timeout', function (done) { + it('"mvt" fails due to statement timeout', function (done) { const params = { layergroupid: this.layergroupid, format: 'mvt', From 667b2a9cb139875b3ff9c287aece81eb8c341105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Mart=C3=ADn?= Date: Wed, 15 Nov 2017 16:56:21 +0100 Subject: [PATCH 126/167] removing underscore dependency in locals middleware --- lib/cartodb/middleware/context/locals.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/cartodb/middleware/context/locals.js b/lib/cartodb/middleware/context/locals.js index eb8fca89..0fdcce50 100644 --- a/lib/cartodb/middleware/context/locals.js +++ b/lib/cartodb/middleware/context/locals.js @@ -1,8 +1,6 @@ -const _ = require('underscore'); - module.exports = function localsMiddleware(req, res, next) { - _.defaults(res.locals, req.params); - + // save req.params in res.locals + res.locals = Object.assign(req.params || {}, res.locals); + next(); }; - From a585ba54807138efea792a70171a7e8b5051a14b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 17 Nov 2017 18:07:19 +0100 Subject: [PATCH 127/167] Use early retutn node pattern --- lib/cartodb/controllers/analyses.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 5d6ac238..a977c854 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -92,11 +92,11 @@ AnalysesController.prototype.catalog = function (req, res, next) { err.http_status = 401; } - next(err); - } else { - res.body = { catalog: catalogWithTables }; - next(); + return next(err); } + + res.body = { catalog: catalogWithTables }; + next(); } ); }; From 5d6ccc07fd2398b31bb3924830a846c232947ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 17 Nov 2017 18:25:13 +0100 Subject: [PATCH 128/167] Move functionality that prepares catalog to be used as response --- lib/cartodb/controllers/analyses.js | 76 ++++++++++++++++------------- 1 file changed, 41 insertions(+), 35 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index a977c854..eab5056b 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -19,11 +19,51 @@ AnalysesController.prototype.register = function(app) { userMiddleware, this.prepareContext, this.catalog.bind(this), + this.prepareResponse(), this.setCacheControlHeader(), this.sendResponse() ); }; +AnalysesController.prototype.prepareResponse = function () { + return function prepareResponseMiddleware (req, res, next) { + const { catalogWithTables } = res.locals; + + const analysisIdToTable = catalogWithTables.tables.reduce((analysisIdToTable, table) => { + const analysisId = table.relname.split('_')[2]; + if (analysisId && analysisId.length === 40) { + analysisIdToTable[analysisId] = table; + } + return analysisIdToTable; + }, {}); + + const catalog = catalogWithTables.catalog.map(analysis => { + if (analysisIdToTable.hasOwnProperty(analysis.node_id)) { + analysis.table = analysisIdToTable[analysis.node_id]; + } + return analysis; + }) + .sort((analysisA, analysisB) => { + if (!!analysisA.table && !!analysisB.table) { + return analysisB.table.size - analysisA.table.size; + } + + if (!!analysisA.table) { + return -1; + } + + if (!!analysisB.table) { + return 1; + } + + return -1; + }); + + res.body = { catalog }; + next(); + }; +}; + AnalysesController.prototype.setCacheControlHeader = function () { return function setCacheControlHeaderMiddleware (req, res, next) { res.set('Cache-Control', 'public,max-age=10,must-revalidate'); @@ -51,40 +91,6 @@ AnalysesController.prototype.catalog = function (req, res, next) { var pg = new PSQL(dbParamsFromReqParams(res.locals)); getMetadata(username, pg, this); }, - function prepareResponse(err, results) { - assert.ifError(err); - - var analysisIdToTable = results.tables.reduce(function(analysisIdToTable, table) { - var analysisId = table.relname.split('_')[2]; - if (analysisId && analysisId.length === 40) { - analysisIdToTable[analysisId] = table; - } - return analysisIdToTable; - }, {}); - - var catalogWithTables = results.catalog.map(function(analysis) { - if (analysisIdToTable.hasOwnProperty(analysis.node_id)) { - analysis.table = analysisIdToTable[analysis.node_id]; - } - return analysis; - }); - - return catalogWithTables.sort(function(analysisA, analysisB) { - if (!!analysisA.table && !!analysisB.table) { - return analysisB.table.size - analysisA.table.size; - } - - if (!!analysisA.table) { - return -1; - } - - if (!!analysisB.table) { - return 1; - } - - return -1; - }); - }, function prepareResponse(err, catalogWithTables) { if (err) { if (err.message.match(/permission\sdenied/)) { @@ -95,7 +101,7 @@ AnalysesController.prototype.catalog = function (req, res, next) { return next(err); } - res.body = { catalog: catalogWithTables }; + res.locals.catalogWithTables = catalogWithTables; next(); } ); From 804c6645fad2863a667d1551bbc5d90751e8bbb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 17 Nov 2017 18:28:37 +0100 Subject: [PATCH 129/167] Make catalog method a regular middleware factory --- lib/cartodb/controllers/analyses.js | 52 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index eab5056b..7a6d75f8 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -18,13 +18,39 @@ AnalysesController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.catalog.bind(this), + this.catalog(), this.prepareResponse(), this.setCacheControlHeader(), this.sendResponse() ); }; +AnalysesController.prototype.catalog = function () { + return function catalogMiddleware(req, res, next) { + const { user } = res.locals; + + step( + function catalogQuery() { + var pg = new PSQL(dbParamsFromReqParams(res.locals)); + getMetadata(user, pg, this); + }, + function prepareResponse(err, catalogWithTables) { + if (err) { + if (err.message.match(/permission\sdenied/)) { + err = new Error('Unauthorized'); + err.http_status = 401; + } + + return next(err); + } + + res.locals.catalogWithTables = catalogWithTables; + next(); + } + ); + }; +}; + AnalysesController.prototype.prepareResponse = function () { return function prepareResponseMiddleware (req, res, next) { const { catalogWithTables } = res.locals; @@ -83,30 +109,6 @@ AnalysesController.prototype.sendResponse = function() { }; }; -AnalysesController.prototype.catalog = function (req, res, next) { - var username = res.locals.user; - - step( - function catalogQuery() { - var pg = new PSQL(dbParamsFromReqParams(res.locals)); - getMetadata(username, pg, this); - }, - function prepareResponse(err, catalogWithTables) { - if (err) { - if (err.message.match(/permission\sdenied/)) { - err = new Error('Unauthorized'); - err.http_status = 401; - } - - return next(err); - } - - res.locals.catalogWithTables = catalogWithTables; - next(); - } - ); -}; - var catalogQueryTpl = dot.template( 'SELECT analysis_def->>\'type\' as type, * FROM cdb_analysis_catalog WHERE username = \'{{=it._username}}\'' ); From c1f2b96bfcf85e71b4b28585a3d51a7aa8f356fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 17 Nov 2017 18:32:19 +0100 Subject: [PATCH 130/167] Remove step from catalog middleware --- lib/cartodb/controllers/analyses.js | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 7a6d75f8..19c1f91b 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -28,26 +28,21 @@ AnalysesController.prototype.register = function(app) { AnalysesController.prototype.catalog = function () { return function catalogMiddleware(req, res, next) { const { user } = res.locals; + const pg = new PSQL(dbParamsFromReqParams(res.locals)); - step( - function catalogQuery() { - var pg = new PSQL(dbParamsFromReqParams(res.locals)); - getMetadata(user, pg, this); - }, - function prepareResponse(err, catalogWithTables) { - if (err) { - if (err.message.match(/permission\sdenied/)) { - err = new Error('Unauthorized'); - err.http_status = 401; - } - - return next(err); + getMetadata(user, pg, (err, catalogWithTables) => { + if (err) { + if (err.message.match(/permission\sdenied/)) { + err = new Error('Unauthorized'); + err.http_status = 401; } - res.locals.catalogWithTables = catalogWithTables; - next(); + return next(err); } - ); + + res.locals.catalogWithTables = catalogWithTables; + next(); + }); }; }; From b1b2054f0adae42517164ae1d70bbe2b15d7083d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 17 Nov 2017 19:14:31 +0100 Subject: [PATCH 131/167] Split metadata function into two middlewares --- lib/cartodb/controllers/analyses.js | 102 +++++++++++++--------------- 1 file changed, 48 insertions(+), 54 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 19c1f91b..3655702e 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -18,39 +18,58 @@ AnalysesController.prototype.register = function(app) { cors(), userMiddleware, this.prepareContext, - this.catalog(), + this.getCatalog(), + this.getTables(), this.prepareResponse(), this.setCacheControlHeader(), - this.sendResponse() + this.sendResponse(), + this.unathorizedError() ); }; -AnalysesController.prototype.catalog = function () { - return function catalogMiddleware(req, res, next) { +AnalysesController.prototype.getCatalog = function () { + return function getCatalogMiddleware(req, res, next) { + const pg = res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); const { user } = res.locals; - const pg = new PSQL(dbParamsFromReqParams(res.locals)); + const readOnlyTransactionOn = true; + const catalogQuery = catalogQueryTpl({ _username: user }); - getMetadata(user, pg, (err, catalogWithTables) => { + pg.query(catalogQuery, (err, resultSet = {}) => { if (err) { - if (err.message.match(/permission\sdenied/)) { - err = new Error('Unauthorized'); - err.http_status = 401; - } - return next(err); } - res.locals.catalogWithTables = catalogWithTables; + res.locals.catalog = resultSet.rows || []; + next(); - }); + }, readOnlyTransactionOn); + }; +}; + +AnalysesController.prototype.getTables = function () { + return function getTablesMiddleware(req, res, next) { + const pg = res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); + const { user } = res.locals; + const readOnlyTransactionOn = true; + const tablesQuery = tablesQueryTpl({ _username: user }); + + pg.query(tablesQuery, (err, resultSet = {}) => { + if (err) { + return next(err); + } + + res.locals.tables = resultSet.rows || []; + + next(); + }, readOnlyTransactionOn); }; }; AnalysesController.prototype.prepareResponse = function () { return function prepareResponseMiddleware (req, res, next) { - const { catalogWithTables } = res.locals; + const { catalog, tables } = res.locals; - const analysisIdToTable = catalogWithTables.tables.reduce((analysisIdToTable, table) => { + const analysisIdToTable = tables.reduce((analysisIdToTable, table) => { const analysisId = table.relname.split('_')[2]; if (analysisId && analysisId.length === 40) { analysisIdToTable[analysisId] = table; @@ -58,7 +77,7 @@ AnalysesController.prototype.prepareResponse = function () { return analysisIdToTable; }, {}); - const catalog = catalogWithTables.catalog.map(analysis => { + const analysisCatalog = catalog.map(analysis => { if (analysisIdToTable.hasOwnProperty(analysis.node_id)) { analysis.table = analysisIdToTable[analysis.node_id]; } @@ -80,7 +99,8 @@ AnalysesController.prototype.prepareResponse = function () { return -1; }); - res.body = { catalog }; + res.body = { catalog: analysisCatalog }; + next(); }; }; @@ -104,6 +124,17 @@ AnalysesController.prototype.sendResponse = function() { }; }; +AnalysesController.prototype.unathorizedError = function () { + return function unathorizedErrorMiddleware(err, req, res, next) { + if (err.message.match(/permission\sdenied/)) { + err = new Error('Unauthorized'); + err.http_status = 401; + } + + next(err); + }; +}; + var catalogQueryTpl = dot.template( 'SELECT analysis_def->>\'type\' as type, * FROM cdb_analysis_catalog WHERE username = \'{{=it._username}}\'' ); @@ -127,43 +158,6 @@ var tablesQueryTpl = dot.template([ "ORDER BY size DESC" ].join('\n')); - -function getMetadata(username, pg, callback) { - var results = { - catalog: [], - tables: [] - }; - step( - function getCatalog() { - pg.query(catalogQueryTpl({_username: username}), this, true); // use read-only transaction - }, - function handleCatalog(err, resultSet) { - assert.ifError(err); - resultSet = resultSet || {}; - results.catalog = resultSet.rows || []; - this(); - }, - function getTables(err) { - assert.ifError(err); - pg.query(tablesQueryTpl({_username: username}), this, true); // use read-only transaction - }, - function handleTables(err, resultSet) { - assert.ifError(err); - resultSet = resultSet || {}; - results.tables = resultSet.rows || []; - this(); - }, - function finish(err) { - if (err) { - return callback(err); - } - - return callback(null, results); - } - ); -} - - function dbParamsFromReqParams(params) { var dbParams = {}; if ( params.dbuser ) { From 4df46fe5eac4098a61bdd4bd7cb5f299626df16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 17 Nov 2017 19:20:42 +0100 Subject: [PATCH 132/167] Move creation of PG client to a middleware --- lib/cartodb/controllers/analyses.js | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 3655702e..80c47416 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -1,5 +1,3 @@ -var step = require('step'); -var assert = require('assert'); var dot = require('dot'); dot.templateSettings.strip = false; var PSQL = require('cartodb-psql'); @@ -12,12 +10,13 @@ function AnalysesController(prepareContext) { module.exports = AnalysesController; -AnalysesController.prototype.register = function(app) { +AnalysesController.prototype.register = function (app) { app.get( app.base_url_mapconfig + '/analyses/catalog', cors(), userMiddleware, this.prepareContext, + this.createPGClient(), this.getCatalog(), this.getTables(), this.prepareResponse(), @@ -27,12 +26,18 @@ AnalysesController.prototype.register = function(app) { ); }; +AnalysesController.prototype.createPGClient = function () { + return function createPGClientMiddleware (req, res, next) { + res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); + next(); + }; +}; + AnalysesController.prototype.getCatalog = function () { return function getCatalogMiddleware(req, res, next) { - const pg = res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); - const { user } = res.locals; - const readOnlyTransactionOn = true; + const { pg, user } = res.locals; const catalogQuery = catalogQueryTpl({ _username: user }); + const readOnlyTransactionOn = true; pg.query(catalogQuery, (err, resultSet = {}) => { if (err) { @@ -48,10 +53,9 @@ AnalysesController.prototype.getCatalog = function () { AnalysesController.prototype.getTables = function () { return function getTablesMiddleware(req, res, next) { - const pg = res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals)); - const { user } = res.locals; - const readOnlyTransactionOn = true; + const { pg, user } = res.locals; const tablesQuery = tablesQueryTpl({ _username: user }); + const readOnlyTransactionOn = true; pg.query(tablesQuery, (err, resultSet = {}) => { if (err) { From 37111f396d309f518e83695980f4a931a06d5075 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sun, 19 Nov 2017 12:37:09 +0100 Subject: [PATCH 133/167] Unify get catalog and tables middlewares --- lib/cartodb/controllers/analyses.js | 33 +++++++---------------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 80c47416..9bc53492 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -17,8 +17,8 @@ AnalysesController.prototype.register = function (app) { userMiddleware, this.prepareContext, this.createPGClient(), - this.getCatalog(), - this.getTables(), + this.getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }), + this.getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }), this.prepareResponse(), this.setCacheControlHeader(), this.sendResponse(), @@ -33,36 +33,19 @@ AnalysesController.prototype.createPGClient = function () { }; }; -AnalysesController.prototype.getCatalog = function () { +AnalysesController.prototype.getDataFromQuery = function ({ queryTemplate, key }) { + const readOnlyTransactionOn = true; + return function getCatalogMiddleware(req, res, next) { const { pg, user } = res.locals; - const catalogQuery = catalogQueryTpl({ _username: user }); - const readOnlyTransactionOn = true; + const sql = queryTemplate({ _username: user }); - pg.query(catalogQuery, (err, resultSet = {}) => { + pg.query(sql, (err, resultSet = {}) => { if (err) { return next(err); } - res.locals.catalog = resultSet.rows || []; - - next(); - }, readOnlyTransactionOn); - }; -}; - -AnalysesController.prototype.getTables = function () { - return function getTablesMiddleware(req, res, next) { - const { pg, user } = res.locals; - const tablesQuery = tablesQueryTpl({ _username: user }); - const readOnlyTransactionOn = true; - - pg.query(tablesQuery, (err, resultSet = {}) => { - if (err) { - return next(err); - } - - res.locals.tables = resultSet.rows || []; + res.locals[key] = resultSet.rows || []; next(); }, readOnlyTransactionOn); From bee04e2553a86c845c7869c35c7652135550f4af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sun, 19 Nov 2017 12:51:35 +0100 Subject: [PATCH 134/167] Usr ES6 string templates --- lib/cartodb/controllers/analyses.js | 44 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 9bc53492..6c2465bc 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -1,5 +1,3 @@ -var dot = require('dot'); -dot.templateSettings.strip = false; var PSQL = require('cartodb-psql'); var cors = require('../middleware/cors'); var userMiddleware = require('../middleware/user'); @@ -122,28 +120,28 @@ AnalysesController.prototype.unathorizedError = function () { }; }; -var catalogQueryTpl = dot.template( - 'SELECT analysis_def->>\'type\' as type, * FROM cdb_analysis_catalog WHERE username = \'{{=it._username}}\'' -); +const catalogQueryTpl = ctx => ` + SELECT analysis_def->>'type' as type, * FROM cdb_analysis_catalog WHERE username = '${ctx._username}' +`; -var tablesQueryTpl = dot.template([ - "WITH analysis_tables AS (", - " SELECT", - " n.nspname AS nspname,", - " c.relname AS relname,", - " pg_total_relation_size(", - " format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))", - " ) AS size,", - " format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name", - " FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n", - " WHERE c.relnamespace = n.oid", - " AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'", - " AND n.nspname IN ('{{=it._username}}', 'public')", - ")", - "SELECT *, pg_size_pretty(size) as size_pretty", - "FROM analysis_tables", - "ORDER BY size DESC" -].join('\n')); +var tablesQueryTpl = ctx => ` + WITH analysis_tables AS ( + SELECT + n.nspname AS nspname, + c.relname AS relname, + pg_total_relation_size( + format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname)) + ) AS size, + format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name + FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n + WHERE c.relnamespace = n.oid + AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$' + AND n.nspname IN ('${ctx._username}', 'public') + ) + SELECT *, pg_size_pretty(size) as size_pretty + FROM analysis_tables + ORDER BY size DESC +`; function dbParamsFromReqParams(params) { var dbParams = {}; From 967d9b76e67fc66fb86506f3c643944f5e9b35a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sun, 19 Nov 2017 13:47:07 +0100 Subject: [PATCH 135/167] Fix EOF --- test/acceptance/analysis/analyses-controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/analysis/analyses-controller.js b/test/acceptance/analysis/analyses-controller.js index 2591f0e1..dd1d138c 100644 --- a/test/acceptance/analysis/analyses-controller.js +++ b/test/acceptance/analysis/analyses-controller.js @@ -111,4 +111,4 @@ describe('analyses controller', function () { }); }); -}); \ No newline at end of file +}); From e131df601cc67c649e90f74ff695ba6b00298a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Sun, 19 Nov 2017 14:05:20 +0100 Subject: [PATCH 136/167] Use ES6 template string for route definition --- lib/cartodb/controllers/analyses.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/analyses.js b/lib/cartodb/controllers/analyses.js index 6c2465bc..fb75c633 100644 --- a/lib/cartodb/controllers/analyses.js +++ b/lib/cartodb/controllers/analyses.js @@ -10,7 +10,7 @@ module.exports = AnalysesController; AnalysesController.prototype.register = function (app) { app.get( - app.base_url_mapconfig + '/analyses/catalog', + `${app.base_url_mapconfig}/analyses/catalog`, cors(), userMiddleware, this.prepareContext, @@ -56,9 +56,11 @@ AnalysesController.prototype.prepareResponse = function () { const analysisIdToTable = tables.reduce((analysisIdToTable, table) => { const analysisId = table.relname.split('_')[2]; + if (analysisId && analysisId.length === 40) { analysisIdToTable[analysisId] = table; } + return analysisIdToTable; }, {}); @@ -66,6 +68,7 @@ AnalysesController.prototype.prepareResponse = function () { if (analysisIdToTable.hasOwnProperty(analysis.node_id)) { analysis.table = analysisIdToTable[analysis.node_id]; } + return analysis; }) .sort((analysisA, analysisB) => { From de376eef86188d758dd74eb75ee54e93b14d200a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 27 Nov 2017 14:52:30 +0100 Subject: [PATCH 137/167] vr-aggregation: link to windshaft#vr-aggregation --- package.json | 2 +- yarn.lock | 109 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 78 insertions(+), 33 deletions(-) diff --git a/package.json b/package.json index 8e4c1d84..f68f2acf 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.1", "underscore": "~1.6.0", - "windshaft": "4.0.1", + "windshaft": "cartodb/windshaft#vr-aggretation", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index 7b5394aa..3eafbd0d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -222,7 +222,7 @@ camshaft@0.59.4: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -240,15 +240,7 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: - version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -256,6 +248,14 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "1.8.3" +"carto@github:cartodb/carto#0.15.1-cdb1": + version "0.15.1-cdb1" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -484,6 +484,10 @@ destroy@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" +detect-libc@^1.0.2: + version "1.0.3" + resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-1.0.3.tgz#fa137c4bd698edf55cd5cd02ac559f91a4c4ba9b" + diff@3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/diff/-/diff-3.2.0.tgz#c9ce393a4b7cbd0b058a725c93df299027868ff9" @@ -746,7 +750,7 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -758,6 +762,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -768,7 +776,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -799,6 +807,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -963,8 +982,8 @@ inherits@2, inherits@2.0.3, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" ini@~1.3.0: - version "1.3.4" - resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.4.tgz#0537cb79daf59b59a1a517dff706c86ec039162e" + version "1.3.5" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927" invert-kv@^1.0.0: version "1.0.0" @@ -1300,7 +1319,7 @@ mime@~1.3.4: version "1.3.6" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: @@ -1360,10 +1379,14 @@ mv@~2: ncp "~2.0.0" rimraf "~2.4.0" -nan@^2.0.8, nan@^2.3.4, nan@^2.4.0, nan@~2.7.0: +nan@^2.0.8, nan@^2.3.4, nan@~2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" +nan@^2.4.0: + version "2.8.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.8.0.tgz#ed715f3fe9de02b57a5e6252d90a96675e1f085a" + nan@~2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232" @@ -1392,9 +1415,10 @@ nock@~2.11.0: propagate "0.3.x" node-pre-gyp@~0.6.30, node-pre-gyp@~0.6.36, node-pre-gyp@~0.6.38: - version "0.6.38" - resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.38.tgz#e92a20f83416415bb4086f6d1fb78b3da73d113d" + version "0.6.39" + resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649" dependencies: + detect-libc "^1.0.2" hawk "3.1.3" mkdirp "^0.5.1" nopt "^4.0.1" @@ -1449,10 +1473,14 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1721,8 +1749,8 @@ raw-body@2.3.2: unpipe "1.0.0" rc@^1.1.7: - version "1.2.1" - resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.1.tgz#2e03e8e42ee450b8cb3dce65be1bf8974e1dfd95" + version "1.2.2" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.2.tgz#d8ce9cb57e8d64d9c7badd9876c7c34cbe3c7077" dependencies: deep-extend "~0.4.0" ini "~1.3.0" @@ -1744,7 +1772,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1774,6 +1802,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1883,7 +1920,7 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: +"semver@2 || 3 || 4 || 5", semver@~5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" @@ -1891,6 +1928,10 @@ semver@4.3.2: version "4.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.2.tgz#c7a07158a80bedd052355b770d82d6640f803be7" +semver@^5.1.0, semver@^5.3.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" + semver@~4.3.3: version "4.3.6" resolved "https://registry.yarnpkg.com/semver/-/semver-4.3.6.tgz#300bc6e0e86374f7ba61068b5b1ecd57fc6532da" @@ -1996,10 +2037,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2114,8 +2159,8 @@ supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: has-flag "^1.0.0" tar-pack@^3.4.0: - version "3.4.0" - resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.0.tgz#23be2d7f671a8339376cbdb0b8fe3fdebf317984" + version "3.4.1" + resolved "https://registry.yarnpkg.com/tar-pack/-/tar-pack-3.4.1.tgz#e1dbc03a9b9d3ba07e896ad027317eb679a10a1f" dependencies: debug "^2.2.0" fstream "^1.0.10" @@ -2145,7 +2190,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2153,7 +2198,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2314,9 +2359,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.1.tgz#16230a0b5b28f08a443491339c37637e3caefa52" +windshaft@cartodb/windshaft#vr-aggretation: + version "4.0.2" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/e0cbbafdf7a65537337c3ca185b6fd2f9bac4e59" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 51c19c0b2eca9d86d4151031c3a32e47b0382765 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Tue, 28 Nov 2017 16:49:55 +0000 Subject: [PATCH 138/167] Skip test --- test/acceptance/multilayer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 9d287a58..51545416 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1164,7 +1164,7 @@ describe(suiteName, function() { }); // See https://github.com/CartoDB/Windshaft-cartodb/issues/133 - it("MapConfig with mapnik layer and no cartocss", function(done) { + it.skip("MapConfig with mapnik layer and no cartocss", function(done) { var layergroup = { version: '1.0.0', From 8270699b8ea1d32e32304210fbb00f9ec5e7cb2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 13:12:09 +0100 Subject: [PATCH 139/167] Tests to fect mvt tiles without styles --- test/acceptance/multilayer.js | 4 ++ test/acceptance/mvt.js | 120 +++++++++++++++++++++++++++++++++- 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 51545416..a1a93e40 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -1163,6 +1163,10 @@ describe(suiteName, function() { ); }); + // WARN: MapConfig with mapnik layer and no cartocss it's valid since + // vector & raster aggregation project, now we can request MVT format w/o defining styles + // for the layer. + // See https://github.com/CartoDB/Windshaft-cartodb/issues/133 it.skip("MapConfig with mapnik layer and no cartocss", function(done) { diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index ded1772e..c1958ca5 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -34,6 +34,124 @@ return function () { serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; }); + describe('vector-layergroup', function () { + const POLYGONS_SQL = ` + select + st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 1000000)::geometry as the_geom, + st_transform( + st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 1000000)::geometry, + 3857 + ) as the_geom_webmercator, + x as value + from generate_series(-3, 3) x + `; + + const POINTS_SQL = ` + select + st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, + x as value + from generate_series(-3, 3) x + `; + + function createVectorLayergroup () { + return { + version: '1.6.0', + layers: [ + { + type: 'cartodb', + options: { + sql: POINTS_SQL + } + }, + { + type: 'cartodb', + options: { + sql: POLYGONS_SQL + } + } + ] + }; + } + + beforeEach(function () { + this.mapConfig = createVectorLayergroup(); + this.testClient = new TestClient(this.mapConfig); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('should get vector tiles from layergroup with layers w/o cartocss', function (done) { + this.testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.equal(tile.tileSize, 4096); + assert.equal(tile.z, 0); + assert.equal(tile.x, 0); + assert.equal(tile.y, 0); + + const layer0 = JSON.parse(tile.toGeoJSONSync(0)); + + assert.equal(layer0.name, 'layer0'); + assert.equal(layer0.features[0].type, 'Feature'); + assert.equal(layer0.features[0].geometry.type, 'Point'); + + const layer1 = JSON.parse(tile.toGeoJSONSync(1)); + + assert.equal(layer1.name, 'layer1'); + assert.equal(layer1.features[0].type, 'Feature'); + assert.equal(layer1.features[0].geometry.type, 'Polygon'); + done(); + }); + }); + + it('should get vector tiles from specific layer (layer0)', function (done) { + this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.equal(tile.tileSize, 4096); + assert.equal(tile.z, 0); + assert.equal(tile.x, 0); + assert.equal(tile.y, 0); + + const layer = JSON.parse(tile.toGeoJSONSync(0)); + + assert.equal(layer.name, 'layer0'); + assert.equal(layer.features[0].type, 'Feature'); + assert.equal(layer.features[0].geometry.type, 'Point'); + + done(); + }); + }); + + it('should get vector tiles from specific layer (layer1)', function (done) { + this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 1 }, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.equal(tile.tileSize, 4096); + assert.equal(tile.z, 0); + assert.equal(tile.x, 0); + assert.equal(tile.y, 0); + + const layer = JSON.parse(tile.toGeoJSONSync(0)); + + assert.equal(layer.name, 'layer1'); + assert.equal(layer.features[0].type, 'Feature'); + assert.equal(layer.features[0].geometry.type, 'Polygon'); + + done(); + }); + }); + }); + describe('analysis-layers-dataviews-mvt', function () { function createMapConfig(layers, dataviews, analysis) { @@ -504,4 +622,4 @@ return function () { } }; -} \ No newline at end of file +} From 330f8f3cb514f5494159694c9a4dcd2caf2c4a17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 14:10:56 +0100 Subject: [PATCH 140/167] Test invalid format for vector layergroup --- test/acceptance/mvt.js | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index c1958ca5..556f0638 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -150,6 +150,37 @@ return function () { done(); }); }); + + it.only('should fail when the format requested is not mvt', function (done) { + const options = { + format: 'png', + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, options, (err, res, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + "errors": [ + "Invalid format: only mvt format is available for layers without CartoCSS defined" + ], + "errors_with_context":[ + { + "type":"tile", + "message":"Invalid format: only mvt format is available for layers without CartoCSS defined" + } + ] + }); + done(); + }); + }); }); describe('analysis-layers-dataviews-mvt', function () { From f9428682f936f6bc59ff5e0db8e110127440a277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 14:38:06 +0100 Subject: [PATCH 141/167] Remove mocha filter --- test/acceptance/mvt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 556f0638..0c566006 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -151,7 +151,7 @@ return function () { }); }); - it.only('should fail when the format requested is not mvt', function (done) { + it('should fail when the format requested is not mvt', function (done) { const options = { format: 'png', response: { From c9e0f330c0e3771594355235ba5143c9005d38a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 16:15:52 +0100 Subject: [PATCH 142/167] Add test to check incompatible layers --- test/acceptance/mvt.js | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 0c566006..25e32308 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -181,6 +181,38 @@ return function () { done(); }); }); + + it('should fail when the map-config mix layers with and without cartocss', function (done) { + const response = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.mapConfig.layers[0].options.cartocss = '#layer0 { marker-fill: red; marker-width: 10; }'; + this.testClient.mapConfig.layers[0].options.cartocss_version = '2.3.0'; + this.testClient.getLayergroup(response, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, { + "errors": [ + "The layergroup contains incompatible layers: don't mix styled layers with non styled layers (without cartocss)" + ], + "errors_with_context":[ + { + "type":"layergroup", + "message": "The layergroup contains incompatible layers: don't mix styled layers with non styled layers (without cartocss)" + } + ] + }); + + done(); + }); + }); + }); describe('analysis-layers-dataviews-mvt', function () { From f41af41bd497ddf5e1735a3b531b5767bd80493b Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 29 Nov 2017 16:32:22 +0100 Subject: [PATCH 143/167] Update to turbo-carto 0.20.2 (#805) --- NEWS.md | 6 ++++++ package.json | 2 +- yarn.lock | 16 ++++++++-------- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/NEWS.md b/NEWS.md index 409b5ebc..7748be37 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,11 @@ # Changelog +## 4.1.1 +Released 2017-mm-dd + +Announcements: + - Upgrades turbo-carto to [0.20.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.2). + ## 4.1.0 Released 2017-mm-dd diff --git a/package.json b/package.json index 8e4c1d84..43efe416 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "semver": "~5.3.0", "step": "~0.0.6", "step-profiler": "~0.3.0", - "turbo-carto": "0.20.1", + "turbo-carto": "0.20.2", "underscore": "~1.6.0", "windshaft": "4.0.1", "yargs": "~5.0.0" diff --git a/yarn.lock b/yarn.lock index 7b5394aa..7312213c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -222,7 +222,7 @@ camshaft@0.59.4: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -248,7 +248,7 @@ carto@CartoDB/carto#0.15.1-cdb1: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": +carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -2145,7 +2145,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2153,7 +2153,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2190,9 +2190,9 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" -turbo-carto@0.20.1: - version "0.20.1" - resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.1.tgz#e9f5fa1408d9d4325a1e79333e6d242170f89e6d" +turbo-carto@0.20.2: + version "0.20.2" + resolved "https://registry.yarnpkg.com/turbo-carto/-/turbo-carto-0.20.2.tgz#2b737597a65c2918432f70ea414f12fbec2b6a6f" dependencies: cartocolor "4.0.0" colorbrewer "1.0.0" From a20e789302cf997282f773c71662ae692c02bf15 Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 29 Nov 2017 16:34:48 +0100 Subject: [PATCH 144/167] Release 4.1.1 --- NEWS.md | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS.md b/NEWS.md index 7748be37..e38e1db6 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,7 @@ # Changelog ## 4.1.1 -Released 2017-mm-dd +Released 2017-11-29 Announcements: - Upgrades turbo-carto to [0.20.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.2). diff --git a/package.json b/package.json index 43efe416..75de7f0b 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.0.2", + "version": "4.1.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 1b6de9961a15dac8eb399df39afecc826c2541a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 16:47:56 +0100 Subject: [PATCH 145/167] Do not use polygons --- test/acceptance/mvt.js | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 25e32308..3953d3b4 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -35,21 +35,18 @@ return function () { }); describe('vector-layergroup', function () { - const POLYGONS_SQL = ` + const POINTS_SQL_1 = ` select - st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 1000000)::geometry as the_geom, - st_transform( - st_buffer(st_setsrid(st_makepoint(x*10, x*10), 4326)::geography, 1000000)::geometry, - 3857 - ) as the_geom_webmercator, + st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, x as value from generate_series(-3, 3) x `; - const POINTS_SQL = ` + const POINTS_SQL_2 = ` select - st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, - st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, + st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326), 3857) as the_geom_webmercator, x as value from generate_series(-3, 3) x `; @@ -61,13 +58,13 @@ return function () { { type: 'cartodb', options: { - sql: POINTS_SQL + sql: POINTS_SQL_1 } }, { type: 'cartodb', options: { - sql: POLYGONS_SQL + sql: POINTS_SQL_2 } } ] @@ -104,7 +101,7 @@ return function () { assert.equal(layer1.name, 'layer1'); assert.equal(layer1.features[0].type, 'Feature'); - assert.equal(layer1.features[0].geometry.type, 'Polygon'); + assert.equal(layer1.features[0].geometry.type, 'Point'); done(); }); }); @@ -145,7 +142,7 @@ return function () { assert.equal(layer.name, 'layer1'); assert.equal(layer.features[0].type, 'Feature'); - assert.equal(layer.features[0].geometry.type, 'Polygon'); + assert.equal(layer.features[0].geometry.type, 'Point'); done(); }); From cded5afdcb23b9f5f4bb034f861ba01c1628c5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 16:56:28 +0100 Subject: [PATCH 146/167] Ahh! jshint.. my old friend --- test/acceptance/mvt.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 3953d3b4..424ac95e 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -196,12 +196,14 @@ return function () { assert.deepEqual(body, { "errors": [ - "The layergroup contains incompatible layers: don't mix styled layers with non styled layers (without cartocss)" + "The layergroup contains incompatible layers: " + + "don't mix styled layers with non styled layers (without cartocss)" ], "errors_with_context":[ { "type":"layergroup", - "message": "The layergroup contains incompatible layers: don't mix styled layers with non styled layers (without cartocss)" + "message": "The layergroup contains incompatible layers: " + + "don't mix styled layers with non styled layers (without cartocss)" } ] }); From 0ba49753600bd73a6532f52fddba7b287d4503fb Mon Sep 17 00:00:00 2001 From: Mario de Frutos Date: Wed, 29 Nov 2017 18:02:46 +0100 Subject: [PATCH 147/167] Bump next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index e38e1db6..0aa1535d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 4.1.2 +Released 2017-mm-dd + + ## 4.1.1 Released 2017-11-29 diff --git a/package.json b/package.json index 75de7f0b..0ea64507 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.1.1", + "version": "4.1.2", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 496778c276b99af4c547378ccfeb5506221aae8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 18:17:51 +0100 Subject: [PATCH 148/167] Separate suites --- test/acceptance/vector-layergroup.js | 218 +++++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 test/acceptance/vector-layergroup.js diff --git a/test/acceptance/vector-layergroup.js b/test/acceptance/vector-layergroup.js new file mode 100644 index 00000000..8e811197 --- /dev/null +++ b/test/acceptance/vector-layergroup.js @@ -0,0 +1,218 @@ +require('../support/test_helper'); + +const assert = require('../support/assert'); +const TestClient = require('../support/test-client'); +const serverOptions = require('../../lib/cartodb/server_options'); + + +const POINTS_SQL_1 = ` +select + st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, + x as value +from generate_series(-3, 3) x +`; + +const POINTS_SQL_2 = ` +select + st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326) as the_geom, + st_transform(st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326), 3857) as the_geom_webmercator, + x as value +from generate_series(-3, 3) x +`; + +function createVectorLayergroup () { + return { + version: '1.6.0', + layers: [ + { + type: 'cartodb', + options: { + sql: POINTS_SQL_1 + } + }, + { + type: 'cartodb', + options: { + sql: POINTS_SQL_2 + } + } + ] + }; +} + +const INCOMPATIBLE_LAYERS_ERROR = { + "errors": [ + "The layergroup contains incompatible layers: " + + "don't mix styled layers with non styled layers (without cartocss)" + ], + "errors_with_context":[ + { + "type":"layergroup", + "message": "The layergroup contains incompatible layers: " + + "don't mix styled layers with non styled layers (without cartocss)" + } + ] +}; + +const INVALID_FORMAT_ERROR = { + "errors": [ + "Invalid format: only mvt format is available for layers without CartoCSS defined" + ], + "errors_with_context":[ + { + "type": "tile", + "message": "Invalid format:" + + " only mvt format is available for layers without CartoCSS defined" + } + ] +}; + +const suites = [{ + desc: 'mvt (mapnik)', + usePostGIS: false +}]; + +if (process.env.POSTGIS_VERSION === '2.4') { + suites.push({ + desc: 'mvt (postgis)', + usePostGIS: true + }); +} + +suites.forEach((suite) => { + const { desc, usePostGIS } = suite; + + describe(desc, function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + describe('vector-layergroup', function () { + beforeEach(function () { + this.mapConfig = createVectorLayergroup(); + this.testClient = new TestClient(this.mapConfig); + }); + + afterEach(function (done) { + this.testClient.drain(done); + }); + + it('should get vector tiles from layergroup with layers w/o cartocss', function (done) { + this.testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.equal(tile.tileSize, 4096); + assert.equal(tile.z, 0); + assert.equal(tile.x, 0); + assert.equal(tile.y, 0); + + const layer0 = JSON.parse(tile.toGeoJSONSync(0)); + + assert.equal(layer0.name, 'layer0'); + assert.equal(layer0.features[0].type, 'Feature'); + assert.equal(layer0.features[0].geometry.type, 'Point'); + + const layer1 = JSON.parse(tile.toGeoJSONSync(1)); + + assert.equal(layer1.name, 'layer1'); + assert.equal(layer1.features[0].type, 'Feature'); + assert.equal(layer1.features[0].geometry.type, 'Point'); + done(); + }); + }); + + it('should get vector tiles from specific layer (layer0)', function (done) { + this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.equal(tile.tileSize, 4096); + assert.equal(tile.z, 0); + assert.equal(tile.x, 0); + assert.equal(tile.y, 0); + + const layer = JSON.parse(tile.toGeoJSONSync(0)); + + assert.equal(layer.name, 'layer0'); + assert.equal(layer.features[0].type, 'Feature'); + assert.equal(layer.features[0].geometry.type, 'Point'); + + done(); + }); + }); + + it('should get vector tiles from specific layer (layer1)', function (done) { + this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 1 }, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.equal(tile.tileSize, 4096); + assert.equal(tile.z, 0); + assert.equal(tile.x, 0); + assert.equal(tile.y, 0); + + const layer = JSON.parse(tile.toGeoJSONSync(0)); + + assert.equal(layer.name, 'layer1'); + assert.equal(layer.features[0].type, 'Feature'); + assert.equal(layer.features[0].geometry.type, 'Point'); + + done(); + }); + }); + + it('should fail when the format requested is not mvt', function (done) { + const options = { + format: 'png', + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, options, (err, res, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, INVALID_FORMAT_ERROR); + done(); + }); + }); + + it('should fail when the map-config mix layers with and without cartocss', function (done) { + const response = { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + + this.testClient.mapConfig.layers[0].options.cartocss = '#layer0 { marker-fill: red; marker-width: 10; }'; + this.testClient.mapConfig.layers[0].options.cartocss_version = '2.3.0'; + this.testClient.getLayergroup(response, (err, body) => { + if (err) { + return done(err); + } + + assert.deepEqual(body, INCOMPATIBLE_LAYERS_ERROR); + + done(); + }); + }); + }); + }); +}); From e18e86f565ee27175ec7217442834e9a7e99abb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 29 Nov 2017 18:28:42 +0100 Subject: [PATCH 149/167] jshint, jshint, jshint --- test/acceptance/vector-layergroup.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/acceptance/vector-layergroup.js b/test/acceptance/vector-layergroup.js index 8e811197..794f8fbc 100644 --- a/test/acceptance/vector-layergroup.js +++ b/test/acceptance/vector-layergroup.js @@ -201,8 +201,12 @@ suites.forEach((suite) => { } }; - this.testClient.mapConfig.layers[0].options.cartocss = '#layer0 { marker-fill: red; marker-width: 10; }'; - this.testClient.mapConfig.layers[0].options.cartocss_version = '2.3.0'; + const cartocss = `#layer0 { marker-fill: red; marker-width: 10; }`; + const cartocssVersion = '2.3.0'; + + this.testClient.mapConfig.layers[0].options.cartocss = cartocss; + this.testClient.mapConfig.layers[0].options.cartocss_version = cartocssVersion; + this.testClient.getLayergroup(response, (err, body) => { if (err) { return done(err); From 23c0cb757d84d8e354ae3f70b1caf7c6e571a69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Fri, 1 Dec 2017 13:52:28 +0100 Subject: [PATCH 150/167] Fix tests according to the last changes in windshaft --- test/acceptance/mvt.js | 14 +++++++------- test/acceptance/vector-layergroup.js | 15 +++++++-------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 424ac95e..25544d5e 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -166,12 +166,12 @@ return function () { assert.deepEqual(body, { "errors": [ - "Invalid format: only mvt format is available for layers without CartoCSS defined" + "Unsupported format: 'cartocss' option is missing for png" ], "errors_with_context":[ { "type":"tile", - "message":"Invalid format: only mvt format is available for layers without CartoCSS defined" + "message":"Unsupported format: 'cartocss' option is missing for png" } ] }); @@ -196,14 +196,14 @@ return function () { assert.deepEqual(body, { "errors": [ - "The layergroup contains incompatible layers: " + - "don't mix styled layers with non styled layers (without cartocss)" + "The `mapnik` or `cartodb` layers must be consistent:" + + " `cartocss` option is either present or voided in all layers. Mixing is not allowed." ], "errors_with_context":[ { - "type":"layergroup", - "message": "The layergroup contains incompatible layers: " + - "don't mix styled layers with non styled layers (without cartocss)" + "type":"mapconfig", + "message": "The `mapnik` or `cartodb` layers must be consistent:" + + " `cartocss` option is either present or voided in all layers. Mixing is not allowed." } ] }); diff --git a/test/acceptance/vector-layergroup.js b/test/acceptance/vector-layergroup.js index 794f8fbc..54ec29ef 100644 --- a/test/acceptance/vector-layergroup.js +++ b/test/acceptance/vector-layergroup.js @@ -43,27 +43,26 @@ function createVectorLayergroup () { const INCOMPATIBLE_LAYERS_ERROR = { "errors": [ - "The layergroup contains incompatible layers: " + - "don't mix styled layers with non styled layers (without cartocss)" + "The `mapnik` or `cartodb` layers must be consistent:" + + " `cartocss` option is either present or voided in all layers. Mixing is not allowed." ], "errors_with_context":[ { - "type":"layergroup", - "message": "The layergroup contains incompatible layers: " + - "don't mix styled layers with non styled layers (without cartocss)" + "type":"mapconfig", + "message": "The `mapnik` or `cartodb` layers must be consistent:" + + " `cartocss` option is either present or voided in all layers. Mixing is not allowed." } ] }; const INVALID_FORMAT_ERROR = { "errors": [ - "Invalid format: only mvt format is available for layers without CartoCSS defined" + "Unsupported format: 'cartocss' option is missing for png" ], "errors_with_context":[ { "type": "tile", - "message": "Invalid format:" + - " only mvt format is available for layers without CartoCSS defined" + "message": "Unsupported format: 'cartocss' option is missing for png" } ] }; From 169b95809a830b69f6472fab0c402d48ae886325 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Dec 2017 13:06:45 +0100 Subject: [PATCH 151/167] Upgrade windshaft to version 4.1.0 --- package.json | 2 +- yarn.lock | 38 +++++++++++++++++++------------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 3cec519f..4f3a2546 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.2", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft#vr-aggretation", + "windshaft": "4.1.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index d58f2157..92315ec1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -222,7 +222,7 @@ camshaft@0.59.4: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -240,14 +240,6 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@cartodb/carto#0.15.1-cdb3: - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "1.8.3" - "carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" @@ -256,6 +248,14 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "~1.6.0" +"carto@github:cartodb/carto#0.15.1-cdb3": + version "0.15.1-cdb3" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "1.8.3" + cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -826,9 +826,9 @@ graceful-fs@^4.1.2: version "1.0.1" resolved "https://registry.yarnpkg.com/graceful-readlink/-/graceful-readlink-1.0.1.tgz#4cafad76bc62f02fa039b2f94e9a3dd3a391a725" -grainstore@~1.6.0: - version "1.6.4" - resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.6.4.tgz#617b93c5e2de8f544375202da89b9208a8b3d762" +grainstore@1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/grainstore/-/grainstore-1.7.0.tgz#28d78895c82e6201f7d0ff63af1056f3c0fda0d3" dependencies: carto "0.16.3" debug "~3.1.0" @@ -2190,7 +2190,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2198,7 +2198,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2359,9 +2359,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft#vr-aggretation: - version "4.0.2" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/e0cbbafdf7a65537337c3ca185b6fd2f9bac4e59" +windshaft@^4.1.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.1.0.tgz#dc17c8369570c305171d1ab5ca130369bba04d58" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 @@ -2369,7 +2369,7 @@ windshaft@cartodb/windshaft#vr-aggretation: cartodb-psql "^0.10.1" debug "^3.1.0" dot "~1.0.2" - grainstore "~1.6.0" + grainstore "1.7.0" mapnik "3.5.14" queue-async "~1.0.7" redis-mpool "0.4.1" From 0c8a31fad90d25f3c27789e2d818d076bee417d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Dec 2017 13:17:37 +0100 Subject: [PATCH 152/167] Remove duplicated code --- test/acceptance/mvt.js | 180 ----------------------------------------- 1 file changed, 180 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 25544d5e..fa9189f2 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -34,186 +34,6 @@ return function () { serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; }); - describe('vector-layergroup', function () { - const POINTS_SQL_1 = ` - select - st_setsrid(st_makepoint(x*10, x*10), 4326) as the_geom, - st_transform(st_setsrid(st_makepoint(x*10, x*10), 4326), 3857) as the_geom_webmercator, - x as value - from generate_series(-3, 3) x - `; - - const POINTS_SQL_2 = ` - select - st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326) as the_geom, - st_transform(st_setsrid(st_makepoint(x*10, x*10*(-1)), 4326), 3857) as the_geom_webmercator, - x as value - from generate_series(-3, 3) x - `; - - function createVectorLayergroup () { - return { - version: '1.6.0', - layers: [ - { - type: 'cartodb', - options: { - sql: POINTS_SQL_1 - } - }, - { - type: 'cartodb', - options: { - sql: POINTS_SQL_2 - } - } - ] - }; - } - - beforeEach(function () { - this.mapConfig = createVectorLayergroup(); - this.testClient = new TestClient(this.mapConfig); - }); - - afterEach(function (done) { - this.testClient.drain(done); - }); - - it('should get vector tiles from layergroup with layers w/o cartocss', function (done) { - this.testClient.getTile(0, 0, 0, { format: 'mvt' }, (err, res, tile) => { - if (err) { - return done(err); - } - - assert.equal(tile.tileSize, 4096); - assert.equal(tile.z, 0); - assert.equal(tile.x, 0); - assert.equal(tile.y, 0); - - const layer0 = JSON.parse(tile.toGeoJSONSync(0)); - - assert.equal(layer0.name, 'layer0'); - assert.equal(layer0.features[0].type, 'Feature'); - assert.equal(layer0.features[0].geometry.type, 'Point'); - - const layer1 = JSON.parse(tile.toGeoJSONSync(1)); - - assert.equal(layer1.name, 'layer1'); - assert.equal(layer1.features[0].type, 'Feature'); - assert.equal(layer1.features[0].geometry.type, 'Point'); - done(); - }); - }); - - it('should get vector tiles from specific layer (layer0)', function (done) { - this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, (err, res, tile) => { - if (err) { - return done(err); - } - - assert.equal(tile.tileSize, 4096); - assert.equal(tile.z, 0); - assert.equal(tile.x, 0); - assert.equal(tile.y, 0); - - const layer = JSON.parse(tile.toGeoJSONSync(0)); - - assert.equal(layer.name, 'layer0'); - assert.equal(layer.features[0].type, 'Feature'); - assert.equal(layer.features[0].geometry.type, 'Point'); - - done(); - }); - }); - - it('should get vector tiles from specific layer (layer1)', function (done) { - this.testClient.getTile(0, 0, 0, { format: 'mvt', layers: 1 }, (err, res, tile) => { - if (err) { - return done(err); - } - - assert.equal(tile.tileSize, 4096); - assert.equal(tile.z, 0); - assert.equal(tile.x, 0); - assert.equal(tile.y, 0); - - const layer = JSON.parse(tile.toGeoJSONSync(0)); - - assert.equal(layer.name, 'layer1'); - assert.equal(layer.features[0].type, 'Feature'); - assert.equal(layer.features[0].geometry.type, 'Point'); - - done(); - }); - }); - - it('should fail when the format requested is not mvt', function (done) { - const options = { - format: 'png', - response: { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - } - }; - - this.testClient.getTile(0, 0, 0, options, (err, res, body) => { - if (err) { - return done(err); - } - - assert.deepEqual(body, { - "errors": [ - "Unsupported format: 'cartocss' option is missing for png" - ], - "errors_with_context":[ - { - "type":"tile", - "message":"Unsupported format: 'cartocss' option is missing for png" - } - ] - }); - done(); - }); - }); - - it('should fail when the map-config mix layers with and without cartocss', function (done) { - const response = { - status: 400, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - - this.testClient.mapConfig.layers[0].options.cartocss = '#layer0 { marker-fill: red; marker-width: 10; }'; - this.testClient.mapConfig.layers[0].options.cartocss_version = '2.3.0'; - this.testClient.getLayergroup(response, (err, body) => { - if (err) { - return done(err); - } - - assert.deepEqual(body, { - "errors": [ - "The `mapnik` or `cartodb` layers must be consistent:" + - " `cartocss` option is either present or voided in all layers. Mixing is not allowed." - ], - "errors_with_context":[ - { - "type":"mapconfig", - "message": "The `mapnik` or `cartodb` layers must be consistent:" + - " `cartocss` option is either present or voided in all layers. Mixing is not allowed." - } - ] - }); - - done(); - }); - }); - - }); - describe('analysis-layers-dataviews-mvt', function () { function createMapConfig(layers, dataviews, analysis) { From 565cfb7fbe8e09bab95fe91c6e44f3c438bd32e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Dec 2017 13:23:09 +0100 Subject: [PATCH 153/167] Update NEWS --- NEWS.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/NEWS.md b/NEWS.md index e38e1db6..e8086f24 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,11 +1,21 @@ # Changelog +## 4.2.0 +Released 2017-mm-dd + +Announcements: + + - Allow to request MVT tiles without CartoCSS + - Upgrades windshaft to [4.1.0](https://github.com/CartoDB/windshaft/releases/tag/4.1.0). + + ## 4.1.1 Released 2017-11-29 Announcements: - Upgrades turbo-carto to [0.20.2](https://github.com/CartoDB/turbo-carto/releases/tag/0.20.2). + ## 4.1.0 Released 2017-mm-dd From 893b886a1e66d2e85cd8a5055b02f466ad6be006 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Dec 2017 13:34:02 +0100 Subject: [PATCH 154/167] Release 4.2.0 --- NEWS.md | 3 +-- package.json | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/NEWS.md b/NEWS.md index e8086f24..b4899cbe 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,9 @@ # Changelog ## 4.2.0 -Released 2017-mm-dd +Released 2017-12-04 Announcements: - - Allow to request MVT tiles without CartoCSS - Upgrades windshaft to [4.1.0](https://github.com/CartoDB/windshaft/releases/tag/4.1.0). diff --git a/package.json b/package.json index 4f823d55..b4b917d6 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.1.2", + "version": "4.2.0", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 16a36a9d7a8d5df18a72040df79dfbf8ee2005cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 4 Dec 2017 13:35:23 +0100 Subject: [PATCH 155/167] Stubs next version --- NEWS.md | 4 ++++ package.json | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index b4899cbe..84cfb4bf 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,9 @@ # Changelog +## 4.2.1 +Released 2017-mm-dd + + ## 4.2.0 Released 2017-12-04 diff --git a/package.json b/package.json index b4b917d6..0a6a9014 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.2.0", + "version": "4.2.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" From 90c9ad18e056c73d800cc2c7f51827fa0aaf115d Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 28 Nov 2017 17:51:28 +0100 Subject: [PATCH 156/167] Optimize histogram queries --- .../dataview/histograms/numeric-histogram.js | 225 ++++++++---------- 1 file changed, 101 insertions(+), 124 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index ea191bea..0396e2f8 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,44 +1,42 @@ const BaseHistogram = require('./base-histogram'); const debug = require('debug')('windshaft:dataview:numeric-histogram'); +/** Cast the column column to epoch */ const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; -const filterOutSpecialNumericValues = ctx => ` - ${ctx.column} != 'infinity'::float - AND - ${ctx.column} != '-infinity'::float - AND - ${ctx.column} != 'NaN'::float -`; +/** If the column type is float, ignore any non numeric result (infinity / NaN) */ +const handleFloatColum = ctx => `${!ctx.isFloatColumn ? `${ctx.column}` : + `nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)` +}`; -const filteredQueryTpl = ctx => ` +/** Count only infinity (positive and negative) appearances */ +const countInfinites = ctx => `${!ctx.isFloatColumn ? `0` : + `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)` +}`; + +/** Count only NaNs appearances*/ +const countNaNs = ctx => `${!ctx.isFloatColumn ? `0` : + `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` +}`; + +/** Query to get min and max values from the query */ +const irqQueryTpl = ctx => ` __cdb_filtered_source AS ( SELECT * FROM (${ctx.query}) __cdb_filtered_source_query - WHERE ${ctx.column} IS NOT NULL - ${ctx.isFloatColumn ? `AND ${filterOutSpecialNumericValues(ctx)}` : ''} - ) -`; - -const basicsQueryTpl = ctx => ` + WHERE ${handleFloatColum(ctx)} IS NOT NULL + ), __cdb_basics AS ( SELECT - max(${ctx.column}) AS __cdb_max_val, min(${ctx.column}) AS __cdb_min_val, - avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows + max(${ctx.column}) AS __cdb_max_val, + min(${ctx.column}) AS __cdb_min_val, + count(1) AS __cdb_total_rows FROM __cdb_filtered_source ) `; -const overrideBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(${ctx.end}) AS __cdb_max_val, min(${ctx.start}) AS __cdb_min_val, - avg(${ctx.column}) AS __cdb_avg_val, count(1) AS __cdb_total_rows - FROM __cdb_filtered_source - ) -`; - -const iqrQueryTpl = ctx => ` +/* Query to calculate the number of bins (needs irqQueryTpl before it*/ +const binsQueryTpl = ctx => ` __cdb_iqrange AS ( SELECT max(quartile_max) - min(quartile_max) AS __cdb_iqr FROM ( @@ -49,10 +47,7 @@ const iqrQueryTpl = ctx => ` WHERE quartile = 1 or quartile = 3 GROUP BY quartile ) __cdb_iqr - ) -`; - -const binsQueryTpl = ctx => ` + ), __cdb_bins AS ( SELECT CASE WHEN __cdb_total_rows = 0 OR __cdb_iqr = 0 @@ -70,83 +65,6 @@ const binsQueryTpl = ctx => ` ) `; -const overrideBinsQueryTpl = ctx => ` - __cdb_bins AS ( - SELECT ${ctx.override.bins} AS __cdb_bins_number - ) -`; - -const nullsQueryTpl = ctx => ` - __cdb_nulls AS ( - SELECT - count(*) AS __cdb_nulls_count - FROM (${ctx.query}) __cdb_histogram_nulls - WHERE ${ctx.column} IS NULL - ) -`; - -const infinitiesQueryTpl = ctx => ` - __cdb_infinities AS ( - SELECT - count(*) AS __cdb_infinities_count - FROM (${ctx.query}) __cdb_infinities_query - WHERE - ${ctx.column} = 'infinity'::float - OR - ${ctx.column} = '-infinity'::float - ) -`; - -const nansQueryTpl = ctx => ` - __cdb_nans AS ( - SELECT - count(*) AS __cdb_nans_count - FROM (${ctx.query}) __cdb_nans_query - WHERE ${ctx.column} = 'NaN'::float - ) -`; - -const specialNumericValuesColumnDefinitionTpl = () => ` - __cdb_infinities_count AS infinities_count, - __cdb_nans_count AS nans_count -`; - -const specialNumericValuesCTETpl = () => ` - __cdb_infinities, __cdb_nans -`; - -const specialNumericValuesColumnTpl = () => ` - infinities_count, nans_count -`; - -const histogramQueryTpl = ctx => ` - SELECT - (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, - __cdb_bins_number AS bins_number, - __cdb_nulls_count AS nulls_count, - ${ctx.isFloatColumn ? `${specialNumericValuesColumnDefinitionTpl()},` : ''} - __cdb_avg_val AS avg_val, - CASE WHEN __cdb_min_val = __cdb_max_val - THEN 0 - ELSE GREATEST( - 1, - LEAST( - WIDTH_BUCKET(${ctx.column}, __cdb_min_val, __cdb_max_val, __cdb_bins_number), - __cdb_bins_number - ) - ) - 1 - END AS bin, - min(${ctx.column})::numeric AS min, - max(${ctx.column})::numeric AS max, - avg(${ctx.column})::numeric AS avg, - count(*) AS freq - FROM __cdb_filtered_source, __cdb_basics, __cdb_nulls, __cdb_bins - ${ctx.isFloatColumn ? `, ${specialNumericValuesCTETpl()}` : ''} - GROUP BY bin, bins_number, bin_width, nulls_count, avg_val - ${ctx.isFloatColumn ? `, ${specialNumericValuesColumnTpl()}` : ''} - ORDER BY bin -`; - const BIN_MIN_NUMBER = 6; const BIN_MAX_NUMBER = 48; @@ -167,14 +85,14 @@ module.exports = class NumericHistogram extends BaseHistogram { _buildQuery (psql, override, callback) { const histogramSql = this._buildQueryTpl({ - override: override, column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, isFloatColumn: this._columnType === 'float', query: this.query, start: this._getBinStart(override), end: this._getBinEnd(override), + bins: this._getBinsCount(override), minBins: BIN_MIN_NUMBER, - maxBins: BIN_MAX_NUMBER, + maxBins: BIN_MAX_NUMBER }); debug(histogramSql); @@ -182,19 +100,61 @@ module.exports = class NumericHistogram extends BaseHistogram { return callback(null, histogramSql); } + +/** + * ctx: Object with the following values + * ctx.column -- Column for the histogram + * ctx.isFloatColumn - Whether the column is float or not + * ctx.query -- Subquery to extract data + * ctx.start -- Start value for the bins. [>= end to force calculation] + * ctx.end -- End value for the bins. + * ctx.bins -- Numbers of bins to generate [<0 to force calculation] + * ctx.minBins - If !full min bins to calculate [Optional] + * ctx.maxBins - If !full max bins to calculate [Optional] + */ _buildQueryTpl (ctx) { + var extra_tables = ``; + var extra_queries = ``; + var extra_groupby = ``; + + if (ctx.start >= ctx.end) { + ctx.end = `__cdb_basics.__cdb_max_val`; + ctx.start = `__cdb_basics.__cdb_min_val`; + extra_groupby = `, __cdb_basics.__cdb_max_val, __cdb_basics.__cdb_min_val`; + extra_tables = `, __cdb_basics`; + extra_queries = `WITH ${irqQueryTpl(ctx)}`; + } + + if (ctx.bins <= 0) { + ctx.bins = `__cdb_bins.__cdb_bins_number`; + extra_groupby += `, __cdb_bins.__cdb_bins_number`; + extra_tables += `, __cdb_bins`; + extra_queries = `WITH ${irqQueryTpl(ctx)}, ${binsQueryTpl(ctx)}`; + } + return ` - WITH - ${filteredQueryTpl(ctx)}, - ${this._hasOverridenRange(ctx.override) ? overrideBasicsQueryTpl(ctx) : basicsQueryTpl(ctx)}, - ${this._hasOverridenBins(ctx.override) ? - overrideBinsQueryTpl(ctx) : - `${iqrQueryTpl(ctx)}, ${binsQueryTpl(ctx)}` - }, - ${nullsQueryTpl(ctx)} - ${ctx.isFloatColumn ? `,${infinitiesQueryTpl(ctx)}, ${nansQueryTpl(ctx)}` : ''} - ${histogramQueryTpl(ctx)} - `; +${extra_queries} +SELECT + (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width, + ${ctx.bins} as bins_number, + sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END) AS nulls_count, + ${countInfinites(ctx)} AS infinities_count, + ${countNaNs(ctx)} AS nans_count, + min(${handleFloatColum(ctx)}) AS min, + max(${handleFloatColum(ctx)}) AS max, + avg(${handleFloatColum(ctx)}) AS avg, + sum(CASE WHEN (${handleFloatColum(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq, + CASE WHEN ${ctx.start} = ${ctx.end} + THEN 0 + ELSE GREATEST(1, LEAST(${ctx.bins}, + WIDTH_BUCKET(${handleFloatColum(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1 + END AS bin +FROM +( + ${ctx.query} +) __cdb_filtered_source_query${extra_tables} +GROUP BY bin${extra_groupby} +ORDER BY bin;`; } _hasOverridenBins (override) { @@ -204,14 +164,31 @@ module.exports = class NumericHistogram extends BaseHistogram { _getSummary (result, override) { const firstRow = result.rows[0] || {}; + var total_nulls = 0; + var total_infinities = 0; + var total_nans = 0; + var total_avg = 0; + var total_count = 0; + + result.rows.forEach(function(row) { + total_nulls += row.nulls_count; + total_infinities += row.infinities_count; + total_nans += row.nans_count; + total_avg += row.avg * row.freq; + total_count += row.freq; + }); + if (total_count !== 0) { + total_avg /= total_count; + } + return { bin_width: firstRow.bin_width, bins_count: firstRow.bins_number, bins_start: this._populateBinStart(firstRow, override), - nulls: firstRow.nulls_count, - infinities: firstRow.infinities_count, - nans: firstRow.nans_count, - avg: firstRow.avg_val, + nulls: total_nulls, + infinities: total_infinities, + nans: total_nans, + avg: total_avg }; } From fcbf5ffcc5d542d4318129a1192b0e23bdcfa545 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 29 Nov 2017 13:07:59 +0100 Subject: [PATCH 157/167] Move sql helper functions to query-utils.js --- .../dataview/histograms/numeric-histogram.js | 42 ++++++------------- lib/cartodb/utils/query-utils.js | 31 ++++++++++++++ 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/numeric-histogram.js b/lib/cartodb/models/dataview/histograms/numeric-histogram.js index 0396e2f8..a2d5662d 100644 --- a/lib/cartodb/models/dataview/histograms/numeric-histogram.js +++ b/lib/cartodb/models/dataview/histograms/numeric-histogram.js @@ -1,30 +1,13 @@ const BaseHistogram = require('./base-histogram'); const debug = require('debug')('windshaft:dataview:numeric-histogram'); - -/** Cast the column column to epoch */ -const columnCastTpl = ctx => `date_part('epoch', ${ctx.column})`; - -/** If the column type is float, ignore any non numeric result (infinity / NaN) */ -const handleFloatColum = ctx => `${!ctx.isFloatColumn ? `${ctx.column}` : - `nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)` -}`; - -/** Count only infinity (positive and negative) appearances */ -const countInfinites = ctx => `${!ctx.isFloatColumn ? `0` : - `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)` -}`; - -/** Count only NaNs appearances*/ -const countNaNs = ctx => `${!ctx.isFloatColumn ? `0` : - `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` -}`; +const utils = require('../../../utils/query-utils'); /** Query to get min and max values from the query */ const irqQueryTpl = ctx => ` __cdb_filtered_source AS ( SELECT * FROM (${ctx.query}) __cdb_filtered_source_query - WHERE ${handleFloatColum(ctx)} IS NOT NULL + WHERE ${utils.handleFloatColumn(ctx)} IS NOT NULL ), __cdb_basics AS ( SELECT @@ -85,7 +68,7 @@ module.exports = class NumericHistogram extends BaseHistogram { _buildQuery (psql, override, callback) { const histogramSql = this._buildQueryTpl({ - column: this._columnType === 'date' ? columnCastTpl({ column: this.column }) : this.column, + column: this._columnType === 'date' ? utils.columnCastTpl({ column: this.column }) : this.column, isFloatColumn: this._columnType === 'float', query: this.query, start: this._getBinStart(override), @@ -137,17 +120,18 @@ ${extra_queries} SELECT (${ctx.end} - ${ctx.start}) / ${ctx.bins}::float AS bin_width, ${ctx.bins} as bins_number, - sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END) AS nulls_count, - ${countInfinites(ctx)} AS infinities_count, - ${countNaNs(ctx)} AS nans_count, - min(${handleFloatColum(ctx)}) AS min, - max(${handleFloatColum(ctx)}) AS max, - avg(${handleFloatColum(ctx)}) AS avg, - sum(CASE WHEN (${handleFloatColum(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq, + ${utils.countNULLs(ctx)} AS nulls_count, + ${utils.countInfinites(ctx)} AS infinities_count, + ${utils.countNaNs(ctx)} AS nans_count, + min(${utils.handleFloatColumn(ctx)}) AS min, + max(${utils.handleFloatColumn(ctx)}) AS max, + avg(${utils.handleFloatColumn(ctx)}) AS avg, + sum(CASE WHEN (${utils.handleFloatColumn(ctx)} is not NULL) THEN 1 ELSE 0 END) as freq, CASE WHEN ${ctx.start} = ${ctx.end} THEN 0 - ELSE GREATEST(1, LEAST(${ctx.bins}, - WIDTH_BUCKET(${handleFloatColum(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1 + ELSE GREATEST(1, LEAST( + ${ctx.bins}, + WIDTH_BUCKET(${utils.handleFloatColumn(ctx)}, ${ctx.start}, ${ctx.end}, ${ctx.bins}))) - 1 END AS bin FROM ( diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index 47d730f4..8b0c81ba 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -24,3 +24,34 @@ module.exports.extractTableNames = function extractTableNames(query) { module.exports.getQueryRowCount = function getQueryRowEstimation(query) { return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; }; + +/** Cast the column to epoch */ +module.exports.columnCastTpl = function columnCastTpl(ctx) { + return `date_part('epoch', ${ctx.column})`; +}; + +/** If the column type is float, ignore any non numeric result (infinity / NaN) */ +module.exports.handleFloatColumn = function handleFloatColumn(ctx) { + return `${!ctx.isFloatColumn ? `${ctx.column}` : + `nullif(nullif(nullif(${ctx.column}, 'infinity'::float), '-infinity'::float), 'NaN'::float)` + }`; +}; + +/** Count NULL appearances */ +module.exports.countNULLs= function countNULLs(ctx) { + return `sum(CASE WHEN (${ctx.column} IS NULL) THEN 1 ELSE 0 END)`; +}; + +/** Count only infinity (positive and negative) appearances */ +module.exports.countInfinites = function countInfinites(ctx) { + return `${!ctx.isFloatColumn ? `0` : + `sum(CASE WHEN (${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float) THEN 1 ELSE 0 END)` + }`; +}; + +/** Count only NaNs appearances*/ +module.exports.countNaNs = function countNaNs(ctx) { + return `${!ctx.isFloatColumn ? `0` : + `sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)` + }`; +}; \ No newline at end of file From dfe01c836cd8d6c5b33829b2cf19e7050bcdd1e9 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 29 Nov 2017 15:25:35 +0100 Subject: [PATCH 158/167] Escape getQueryRowCount with $$ --- lib/cartodb/utils/query-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/utils/query-utils.js b/lib/cartodb/utils/query-utils.js index 8b0c81ba..f8b2a0f5 100644 --- a/lib/cartodb/utils/query-utils.js +++ b/lib/cartodb/utils/query-utils.js @@ -22,7 +22,7 @@ module.exports.extractTableNames = function extractTableNames(query) { }; module.exports.getQueryRowCount = function getQueryRowEstimation(query) { - return 'select CDB_EstimateRowCount(\'' + query + '\') as rows'; + return 'select CDB_EstimateRowCount($$' + query + '$$) as rows'; }; /** Cast the column to epoch */ From 243e982bd63b9d679da4874befb986e935d9efa4 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Wed, 29 Nov 2017 12:33:41 +0100 Subject: [PATCH 159/167] Optimize formulae queries --- lib/cartodb/models/dataview/formula.js | 36 ++---- .../models/dataview/overviews/formula.js | 122 +++++++----------- 2 files changed, 58 insertions(+), 100 deletions(-) diff --git a/lib/cartodb/models/dataview/formula.js b/lib/cartodb/models/dataview/formula.js index daaeaf0c..df009e4b 100644 --- a/lib/cartodb/models/dataview/formula.js +++ b/lib/cartodb/models/dataview/formula.js @@ -1,34 +1,14 @@ const BaseDataview = require('./base'); const debug = require('debug')('windshaft:dataview:formula'); +const utils = require('../../utils/query-utils'); -const countInfinitiesQueryTpl = ctx => ` - SELECT count(1) FROM (${ctx.query}) __cdb_formula_infinities - WHERE ${ctx.column} = 'infinity'::float OR ${ctx.column} = '-infinity'::float -`; - -const countNansQueryTpl = ctx => ` - SELECT count(1) FROM (${ctx.query}) __cdb_formula_nans - WHERE ${ctx.column} = 'NaN'::float -`; - -const filterOutSpecialNumericValuesTpl = ctx => ` - WHERE - ${ctx.column} != 'infinity'::float - AND - ${ctx.column} != '-infinity'::float - AND - ${ctx.column} != 'NaN'::float -`; - -const formulaQueryTpl = ctx => ` - SELECT - ${ctx.operation}(${ctx.column}) AS result, - (SELECT count(1) FROM (${ctx.query}) _cdb_formula_nulls WHERE ${ctx.column} IS NULL) AS nulls_count - ${ctx.isFloatColumn ? `,(${countInfinitiesQueryTpl(ctx)}) AS infinities_count` : ''} - ${ctx.isFloatColumn ? `,(${countNansQueryTpl(ctx)}) AS nans_count` : ''} - FROM (${ctx.query}) __cdb_formula - ${ctx.isFloatColumn && ctx.operation !== 'count' ? `${filterOutSpecialNumericValuesTpl(ctx)}` : ''} -`; +const formulaQueryTpl = ctx => +`SELECT + ${ctx.operation}(${utils.handleFloatColumn(ctx)}) AS result, + ${utils.countNULLs(ctx)} AS nulls_count + ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``} + ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``} +FROM (${ctx.query}) __cdb_formula`; const VALID_OPERATIONS = { count: true, diff --git a/lib/cartodb/models/dataview/overviews/formula.js b/lib/cartodb/models/dataview/overviews/formula.js index 64d612c9..2a97061e 100644 --- a/lib/cartodb/models/dataview/overviews/formula.js +++ b/lib/cartodb/models/dataview/overviews/formula.js @@ -1,55 +1,38 @@ var BaseOverviewsDataview = require('./base'); var BaseDataview = require('../formula'); var debug = require('debug')('windshaft:widget:formula:overview'); +const utils = require('../../../utils/query-utils'); var dot = require('dot'); dot.templateSettings.strip = false; -var formulaQueryTpls = { - 'count': dot.template([ - 'SELECT', - 'sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula' - ].join('\n')), - 'sum': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' - ].join('\n')), - 'avg': dot.template([ - 'SELECT', - 'sum({{=it._column}}*_feature_count)/sum(_feature_count) AS result,', - '(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nulls WHERE {{=it._column}} IS NULL) AS nulls_count', - '{{?it._isFloatColumn}},(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_infinities', - ' WHERE {{=it._column}} = \'infinity\'::float OR {{=it._column}} = \'-infinity\'::float) AS infinities_count', - ',(SELECT count(1) FROM ({{=it._query}}) _cdb_formula_nans', - ' WHERE {{=it._column}} = \'NaN\'::float) AS nans_count{{?}}', - 'FROM ({{=it._query}}) _cdb_formula', - '{{?it._isFloatColumn}}WHERE', - ' {{=it._column}} != \'infinity\'::float', - 'AND', - ' {{=it._column}} != \'-infinity\'::float', - 'AND', - ' {{=it._column}} != \'NaN\'::float{{?}}' - ].join('\n')), +const VALID_OPERATIONS = { + count: true, + sum: true, + avg: true }; +/** Formulae to calculate the end result using _feature_count from the overview table*/ +function dataviewResult(ctx) { + switch (ctx.operation) { + case 'count': + return `sum(_feature_count)`; + case 'sum': + return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)`; + case 'avg': + return `sum(${utils.handleFloatColumn(ctx)}*_feature_count)/sum(_feature_count) `; + } + return `${ctx.operation}(${utils.handleFloatColumn(ctx)})`; +} + +const formulaQueryTpl = ctx => +`SELECT + ${dataviewResult(ctx)} AS result, + ${utils.countNULLs(ctx)} AS nulls_count + ${ctx.isFloatColumn ? `,${utils.countInfinites(ctx)} AS infinities_count,` : ``} + ${ctx.isFloatColumn ? `${utils.countNaNs(ctx)} AS nans_count` : ``} +FROM (${ctx.query}) __cdb_formula`; + function Formula(query, options, queryRewriter, queryRewriteData, params, queries) { BaseOverviewsDataview.call(this, query, options, BaseDataview, queryRewriter, queryRewriteData, params, queries); this.column = options.column || '1'; @@ -65,36 +48,31 @@ module.exports = Formula; Formula.prototype.sql = function (psql, override, callback) { var self = this; - var formulaQueryTpl = formulaQueryTpls[this.operation]; - - if (formulaQueryTpl) { - // supported formula for use with overviews - if (this._isFloatColumn === null) { - this._isFloatColumn = false; - this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { - if (!err && !!type) { - self._isFloatColumn = type.float; - } - self.sql(psql, override, callback); - }); - return null; - } - - var formulaSql = formulaQueryTpl({ - _isFloatColumn: this._isFloatColumn, - _query: this.rewrittenQuery(this.query), - _operation: this.operation, - _column: this.column - }); - - callback = callback || override; - - debug(formulaSql); - - return callback(null, formulaSql); + if (!VALID_OPERATIONS[this.operation]) { + return this.defaultSql(psql, override, callback); } + if (this._isFloatColumn === null) { + this._isFloatColumn = false; + this.getColumnType(psql, this.column, this.queries.no_filters, function (err, type) { + if (!err && !!type) { + self._isFloatColumn = type.float; + } + self.sql(psql, override, callback); + }); + return null; + } - // default behaviour - return this.defaultSql(psql, override, callback); + var formulaSql = formulaQueryTpl({ + isFloatColumn: this._isFloatColumn, + query: this.rewrittenQuery(this.query), + operation: this.operation, + column: this.column + }); + + callback = callback || override; + + debug(formulaSql); + + return callback(null, formulaSql); }; From 088a8b81a64a51c54b24a5f6105baa8ac6f20420 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Dec 2017 12:44:51 +0100 Subject: [PATCH 160/167] Timeseries tests: Set timestamps to the start of the aggregations Also, increase the end of the range check so it includes some data --- .../dataview/histograms/date-histogram.js | 4 +-- test/acceptance/dataviews/histogram.js | 34 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 34a42f9a..f1790b15 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -264,8 +264,8 @@ module.exports = class DateHistogram extends BaseHistogram { offset: this._getOffset(override), timestamp_start: firstRow.timestamp_start, - bin_width: firstRow.bin_width, - bins_count: firstRow.bins_number, + bin_width: firstRow.bin_width || 0, + bins_count: firstRow.bins_number || 0, bins_start: firstRow.timestamp, nulls: firstRow.nulls_count, infinities: firstRow.infinities_count, diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index c9e16489..5331c443 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -186,7 +186,7 @@ describe('histogram-dataview for date column type', function() { }, minute_histogram: { source: { - id: 'minute-histogram-source' + id: 'minute-histogram-source-tz' }, type: 'histogram', options: { @@ -215,7 +215,7 @@ describe('histogram-dataview for date column type', function() { "query": [ "select null::geometry the_geom_webmercator, date AS d", "from generate_series(", - "'2007-02-15 01:00:00'::timestamptz, '2008-04-09 01:00:00'::timestamptz, '1 day'::interval", + "'2007-02-15 01:00:00+00'::timestamptz, '2008-04-09 01:00:00+00'::timestamptz, '1 day'::interval", ") date" ].join(' ') } @@ -233,13 +233,13 @@ describe('histogram-dataview for date column type', function() { } }, { - "id": "minute-histogram-source", + "id": "minute-histogram-source-tz", "type": "source", "params": { "query": [ "select null::geometry the_geom_webmercator, date AS d", "from generate_series(", - "'2007-02-15 23:50:00'::timestamp, '2007-02-16 00:10:00'::timestamp, '1 minute'::interval", + "'2007-02-15 23:50:00+00'::timestamptz, '2007-02-16 00:10:00+00'::timestamptz, '1 minute'::interval", ") date" ].join(' ') } @@ -323,7 +323,7 @@ describe('histogram-dataview for date column type', function() { assert.ok(!err, err); assert.equal(dataview.type, 'histogram'); assert.ok(dataview.bin_width > 0, 'Unexpected bin width: ' + dataview.bin_width); - assert.equal(dataview.bins.length, 6); + assert.equal(dataview.bins_count, 6); dataview.bins.forEach(function (bin) { assert.ok(bin.min <= bin.max, 'bin min < bin max: ' + JSON.stringify(bin)); }); @@ -335,7 +335,7 @@ describe('histogram-dataview for date column type', function() { it('should cast overridden start and end to float to avoid out of range errors ' + test.desc, function (done) { var params = { start: -2145916800, - end: 1009843199 + end: 1193792400 }; this.testClient = new TestClient(mapConfig, 1234); @@ -640,7 +640,7 @@ describe('histogram-dataview for date column type', function() { var dataviewWithDailyAggFixture = { aggregation: 'day', - bin_width: 600, + bin_width: 86400, bins_count: 2, bins_start: 1171497600, timestamp_start: 1171497600, @@ -650,17 +650,17 @@ describe('histogram-dataview for date column type', function() { [{ bin: 0, timestamp: 1171497600, - min: 1171583400, - max: 1171583940, - avg: 1171583670, + min: 1171497600, + max: 1171497600, + avg: 1171497600, freq: 10 }, { bin: 1, timestamp: 1171584000, min: 1171584000, - max: 1171584600, - avg: 1171584300, + max: 1171584000, + avg: 1171584000, freq: 11 }], type: 'histogram' @@ -687,19 +687,19 @@ describe('histogram-dataview for date column type', function() { var dataviewWithDailyAggAndOffsetFixture = { aggregation: 'day', - bin_width: 1200, + bin_width: 86400, bins_count: 1, bins_start: 1171501200, - timestamp_start: 1171497600, + timestamp_start: 1171501200, nulls: 0, offset: -3600, bins: [{ bin: 0, timestamp: 1171501200, - min: 1171583400, - max: 1171584600, - avg: 1171584000, + min: 1171501200, + max: 1171501200, + avg: 1171501200, freq: 21 }], type: 'histogram' From bb5bfd10eeed20825ea0854cbf27811879547dd0 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Dec 2017 17:02:40 +0100 Subject: [PATCH 161/167] Timeseries tests: Make them work with any DB setup --- test/acceptance/dataviews/histogram.js | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 5331c443..002d8617 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -256,6 +256,7 @@ describe('histogram-dataview for date column type', function() { }]; dateHistogramsUseCases.forEach(function (test) { + it('should create a date histogram aggregated in months (EDT) ' + test.desc, function (done) { var OFFSET_EDT_IN_MINUTES = -4 * 60; // EDT Eastern Daylight Time (GMT-4) in minutes @@ -348,27 +349,6 @@ describe('histogram-dataview for date column type', function() { }); }); - it('should return same histogram ' + test.desc, function (done) { - var params = { - start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) - end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) - }; - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview(test.dataviewId, {}, function (err, dataview) { - assert.ok(!err, err); - - this.testClient = new TestClient(mapConfig, 1234); - this.testClient.getDataview(test.dataviewId, params, function (err, filteredDataview) { - assert.ok(!err, err); - - assert.deepEqual(dataview, filteredDataview); - done(); - }); - }); - }); - - it('should aggregate histogram overriding default offset to CEST ' + test.desc, function (done) { var OFFSET_CEST_IN_SECONDS = 2 * 3600; // Central European Summer Time (Daylight Saving Time) var OFFSET_CEST_IN_MINUTES = 2 * 60; // Central European Summer Time (Daylight Saving Time) @@ -533,6 +513,26 @@ describe('histogram-dataview for date column type', function() { }); }); + it('should return same histogram ', function (done) { + var params = { + start: 1171501200, // 2007-02-15 01:00:00 = min(date_colum) + end: 1207702800 // 2008-04-09 01:00:00 = max(date_colum) + }; + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_tz', {}, function (err, dataview) { + assert.ok(!err, err); + + this.testClient = new TestClient(mapConfig, 1234); + this.testClient.getDataview('datetime_histogram_tz', params, function (err, filteredDataview) { + assert.ok(!err, err); + + assert.deepEqual(dataview, filteredDataview); + done(); + }); + }); + }); + it('should find the best aggregation (automatic mode) to build the histogram', function (done) { var params = {}; this.testClient = new TestClient(mapConfig, 1234); From 1b405e42c2804f2ab85ae200be077c9f3ad4a0d5 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Dec 2017 17:03:31 +0100 Subject: [PATCH 162/167] Date histogram optimizations --- .../dataview/histograms/date-histogram.js | 229 +++++++++--------- 1 file changed, 121 insertions(+), 108 deletions(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index f1790b15..785d132a 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -1,5 +1,102 @@ const BaseHistogram = require('./base-histogram'); const debug = require('debug')('windshaft:dataview:date-histogram'); +const utils = require('../../../utils/query-utils'); + +/** + * Gets the name of a timezone with the same offset as the required + * using the pg_timezone_names table. We do this because it's simpler to pass + * the name than to pass the offset itself as PostgreSQL uses different + * sign convention. For example: TIME ZONE 'CET' is equal to TIME ZONE 'UTC-1', + * not 'UTC+1' which would be expected. + * Gives priority to Etc/GMT±N timezones but still support odd offsets like 8.5 + * hours for Asia/Pyongyang. + * It also makes it easier to, in the future, support the input of expected timezone + * instead of the offset; that is using 'Europe/Madrid' instead of + * '+3600' or '+7200'. The daylight saving status can be handled by postgres. + */ +const offsetNameQueryTpl = ctx => ` +WITH __wd_tz AS +( + SELECT name + FROM pg_timezone_names + WHERE utc_offset = interval '${ctx.offset} hours' + ORDER BY CASE WHEN name LIKE 'Etc/GMT%' THEN 0 ELSE 1 END + LIMIT 1 +),`; + +/** + * Function to get the subquery that places each row in its bin depending on + * the aggregation. Since the data stored is in epoch we need to adapt it to + * our timezone so when calling date_trunc it falls into the correct bin + */ +function dataBucketsQuery(ctx) { + var condition_str = ''; + + if (ctx.start !== 0) { + condition_str = `WHERE ${ctx.column} >= to_timestamp(${ctx.start})`; + } + if (ctx.end !== 0) { + if (condition_str === '') { + condition_str = `WHERE ${ctx.column} <= to_timestamp(${ctx.end})`; + } + else { + condition_str += ` and ${ctx.column} <= to_timestamp(${ctx.end})`; + } + } + + return ` +__wd_buckets AS +( + SELECT + date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, ${ctx.column}::timestamptz)) as timestamp, + count(*) as freq, + ${utils.countNULLs(ctx)} as nulls_count + FROM + ( + ${ctx.query} + ) __source, __wd_tz + ${condition_str} + GROUP BY timestamp +),`; +} + +/** + * Function that generates an array with all the possible bins between the + * start and end date. If not provided we use the min and max generated from + * the dataBucketsQuery + */ +function allBucketsArrayQuery(ctx) { + var extra_from = ``; + var series_start = ``; + var series_end = ``; + + if (ctx.start === 0) { + extra_from = `, __wd_buckets GROUP BY __wd_tz.name`; + series_start = `min(__wd_buckets.timestamp)`; + } else { + series_start = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.start})))`; + } + + if (ctx.end === 0) { + extra_from = `, __wd_buckets GROUP BY __wd_tz.name`; + series_end = `max(__wd_buckets.timestamp)`; + } else { + series_end = `date_trunc('${ctx.aggregation}', timezone(__wd_tz.name, to_timestamp(${ctx.end})))`; + } + + return ` +__wd_all_buckets AS +( + SELECT ARRAY( + SELECT + generate_series( + ${series_start}, + ${series_end}, + interval '${ctx.interval}') as bin_start + FROM __wd_tz${extra_from} + ) as bins +)`; +} const dateIntervalQueryTpl = ctx => ` WITH @@ -41,107 +138,6 @@ const dateIntervalQueryTpl = ctx => ` FROM __cdb_interval_in_days, __cdb_interval_in_hours, __cdb_interval_in_minutes, __cdb_interval_in_seconds `; -const nullsQueryTpl = ctx => ` - __cdb_nulls AS ( - SELECT - count(*) AS __cdb_nulls_count - FROM (${ctx.query}) __cdb_histogram_nulls - WHERE ${ctx.column} IS NULL - ) -`; - -const dateBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(date_part('epoch', ${ctx.column})) AS __cdb_max_val, - min(date_part('epoch', ${ctx.column})) AS __cdb_min_val, - avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, - min( - date_trunc( - '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' - ) - ) AS __cdb_start_date, - max(${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}') AS __cdb_end_date, - count(1) AS __cdb_total_rows - FROM (${ctx.query}) __cdb_basics_query - ) -`; - -const dateOverrideBasicsQueryTpl = ctx => ` - __cdb_basics AS ( - SELECT - max(${ctx.end})::float AS __cdb_max_val, - min(${ctx.start})::float AS __cdb_min_val, - avg(date_part('epoch', ${ctx.column})) AS __cdb_avg_val, - min( - date_trunc( - '${ctx.aggregation}', - TO_TIMESTAMP(${ctx.start})::timestamp AT TIME ZONE '${ctx.offset}' - ) - ) AS __cdb_start_date, - max( - TO_TIMESTAMP(${ctx.end})::timestamp AT TIME ZONE '${ctx.offset}' - ) AS __cdb_end_date, - count(1) AS __cdb_total_rows - FROM (${ctx.query}) __cdb_basics_query - ) -`; - -const dateBinsQueryTpl = ctx => ` - __cdb_bins AS ( - SELECT - __cdb_bins_array, - ARRAY_LENGTH(__cdb_bins_array, 1) AS __cdb_bins_number - FROM ( - SELECT - ARRAY( - SELECT GENERATE_SERIES( - __cdb_start_date::timestamptz, - __cdb_end_date::timestamptz, - ${ctx.aggregation === 'quarter' ? `'3 month'::interval` : `'1 ${ctx.aggregation}'::interval`} - ) - ) AS __cdb_bins_array - FROM __cdb_basics - ) __cdb_bins_array_query - ) -`; - -const dateHistogramQueryTpl = ctx => ` - SELECT - (__cdb_max_val - __cdb_min_val) / cast(__cdb_bins_number as float) AS bin_width, - __cdb_bins_number AS bins_number, - __cdb_nulls_count AS nulls_count, - CASE WHEN __cdb_min_val = __cdb_max_val - THEN 0 - ELSE GREATEST( - 1, - LEAST( - WIDTH_BUCKET( - ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}', - __cdb_bins_array - ), - __cdb_bins_number - ) - ) - 1 - END AS bin, - min( - date_part( - 'epoch', - date_trunc( - '${ctx.aggregation}', ${ctx.column}::timestamp AT TIME ZONE '${ctx.offset}' - ) AT TIME ZONE '${ctx.offset}' - ) - )::numeric AS timestamp, - date_part('epoch', __cdb_start_date)::numeric AS timestamp_start, - min(date_part('epoch', ${ctx.column}))::numeric AS min, - max(date_part('epoch', ${ctx.column}))::numeric AS max, - avg(date_part('epoch', ${ctx.column}))::numeric AS avg, - count(*) AS freq - FROM (${ctx.query}) __cdb_histogram, __cdb_basics, __cdb_bins, __cdb_nulls - WHERE date_part('epoch', ${ctx.column}) IS NOT NULL - GROUP BY bin, bins_number, bin_width, nulls_count, timestamp_start - ORDER BY bin -`; const MAX_INTERVAL_VALUE = 366; @@ -176,12 +172,21 @@ module.exports = class DateHistogram extends BaseHistogram { _buildQueryTpl (ctx) { return ` - WITH - ${this._hasOverridenRange(ctx.override) ? dateOverrideBasicsQueryTpl(ctx) : dateBasicsQueryTpl(ctx)}, - ${dateBinsQueryTpl(ctx)}, - ${nullsQueryTpl(ctx)} - ${dateHistogramQueryTpl(ctx)} - `; +${offsetNameQueryTpl(ctx)} +${dataBucketsQuery(ctx)} +${allBucketsArrayQuery(ctx)} +SELECT + array_position(__wd_all_buckets.bins, __wd_buckets.timestamp) - 1 as bin, + date_part('epoch', timezone(__wd_tz.name, __wd_buckets.timestamp)) AS timestamp, + __wd_buckets.freq as freq, + date_part('epoch', timezone(__wd_tz.name, (__wd_all_buckets.bins)[1])) as timestamp_start, + array_length(__wd_all_buckets.bins, 1) as bins_number, + date_part('epoch', interval '${ctx.interval}') as bin_width, + __wd_buckets.nulls_count as nulls_count +FROM __wd_buckets, __wd_all_buckets, __wd_tz +GROUP BY __wd_tz.name, __wd_all_buckets.bins, __wd_buckets.timestamp, __wd_buckets.nulls_count, __wd_buckets.freq +ORDER BY bin ASC; +`; } _buildQuery (psql, override, callback) { @@ -204,6 +209,9 @@ module.exports = class DateHistogram extends BaseHistogram { return null; } + var interval = this._getAggregation(override) === 'quarter' ? + '3 months' : '1 ' + this._getAggregation(override); + const histogramSql = this._buildQueryTpl({ override: override, query: this.query, @@ -211,7 +219,8 @@ module.exports = class DateHistogram extends BaseHistogram { aggregation: this._getAggregation(override), start: this._getBinStart(override), end: this._getBinEnd(override), - offset: this._parseOffset(override) + offset: this._parseOffset(override), + interval: interval }); debug(histogramSql); @@ -275,6 +284,10 @@ module.exports = class DateHistogram extends BaseHistogram { } _getBuckets (result) { + result.rows.forEach(function(row) { + row.min = row.max = row.avg = row.timestamp; + }); + return result.rows.map(({ bin, min, max, avg, freq, timestamp }) => ({ bin, min, max, avg, freq, timestamp })); } From 88f02458db675aeedf1e03122cff468f6704a3a7 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Dec 2017 17:15:06 +0100 Subject: [PATCH 163/167] Timeseries tests: Make jshint happy --- test/acceptance/dataviews/histogram.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/acceptance/dataviews/histogram.js b/test/acceptance/dataviews/histogram.js index 002d8617..af8ce59d 100644 --- a/test/acceptance/dataviews/histogram.js +++ b/test/acceptance/dataviews/histogram.js @@ -214,8 +214,8 @@ describe('histogram-dataview for date column type', function() { "params": { "query": [ "select null::geometry the_geom_webmercator, date AS d", - "from generate_series(", - "'2007-02-15 01:00:00+00'::timestamptz, '2008-04-09 01:00:00+00'::timestamptz, '1 day'::interval", + "from generate_series('2007-02-15 01:00:00+00'::timestamptz,", + "'2008-04-09 01:00:00+00'::timestamptz, '1 day'::interval", ") date" ].join(' ') } @@ -238,8 +238,8 @@ describe('histogram-dataview for date column type', function() { "params": { "query": [ "select null::geometry the_geom_webmercator, date AS d", - "from generate_series(", - "'2007-02-15 23:50:00+00'::timestamptz, '2007-02-16 00:10:00+00'::timestamptz, '1 minute'::interval", + "from generate_series('2007-02-15 23:50:00+00'::timestamptz,", + "'2007-02-16 00:10:00+00'::timestamptz, '1 minute'::interval", ") date" ].join(' ') } From eb4a49ec9256ba34cc30e7df680b38c1265d9385 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 4 Dec 2017 17:56:11 +0100 Subject: [PATCH 164/167] Update NEWs file with the changes --- NEWS.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/NEWS.md b/NEWS.md index 84cfb4bf..4c737e7c 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,13 @@ ## 4.2.1 Released 2017-mm-dd +Announcements: +- Optimize Formula queries. +- Optimize Formula queries in overviews. +- Optimize Numeric Histogram queries. +- Optimize Date Histogram queries. +- Date Histograms: Now returns the same value for max/min/avg/timestamp per bin. +- Date Histograms: Now it should return the same no matter the DB/Client time zone. ## 4.2.0 Released 2017-12-04 From 7c5b7641d8ba0328737b61008be859f8a615ad08 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Tue, 5 Dec 2017 16:25:36 +0100 Subject: [PATCH 165/167] Date histogram: Add missing GROUP BY column --- lib/cartodb/models/dataview/histograms/date-histogram.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/models/dataview/histograms/date-histogram.js b/lib/cartodb/models/dataview/histograms/date-histogram.js index 785d132a..af151a72 100644 --- a/lib/cartodb/models/dataview/histograms/date-histogram.js +++ b/lib/cartodb/models/dataview/histograms/date-histogram.js @@ -56,7 +56,7 @@ __wd_buckets AS ${ctx.query} ) __source, __wd_tz ${condition_str} - GROUP BY timestamp + GROUP BY timestamp, __wd_tz.name ),`; } From aa57cdefb370ef577bad6dacf26b1a88d82feb3a Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 11 Dec 2017 11:21:16 +0100 Subject: [PATCH 166/167] Release 4.3.0 --- NEWS.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 4c737e7c..e70b8df2 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,8 +1,13 @@ # Changelog -## 4.2.1 +## 4.3.1 Released 2017-mm-dd +Announcements: + +## 4.3.0 +Released 2017-12-11 + Announcements: - Optimize Formula queries. - Optimize Formula queries in overviews. From f2fa65066138296b405cd5a80676f1a73f339ad1 Mon Sep 17 00:00:00 2001 From: Raul Marin Date: Mon, 11 Dec 2017 17:24:26 +0100 Subject: [PATCH 167/167] Stub for next release --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0a6a9014..00331d94 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "4.2.1", + "version": "4.3.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb"