From ad1abb28afea3c0c16f23ffd438cf9c9dedfee36 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 28 Sep 2017 12:08:22 +0200 Subject: [PATCH 01/35] Add configuration flag to enable/disable direct PgSQL MVT --- config/environments/development.js.example | 3 +++ config/environments/production.js.example | 3 +++ config/environments/staging.js.example | 3 +++ config/environments/test.js.example | 3 +++ lib/cartodb/server.js | 3 ++- lib/cartodb/server_options.js | 1 + 6 files changed, 15 insertions(+), 1 deletion(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 48d03592..f09a3808 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -107,6 +107,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/production.js.example b/config/environments/production.js.example index ba7fec6f..1dd80212 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -101,6 +101,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 1792779c..7dd9f5b0 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -101,6 +101,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 374467c0..b266c923 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -100,6 +100,9 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + usePgSQL: false + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 350e52a0..723289ba 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -154,7 +154,8 @@ module.exports = function(serverOptions) { grainstore: serverOptions.grainstore, mapnik: serverOptions.renderer.mapnik }, - http: serverOptions.renderer.http + http: serverOptions.renderer.http, + mvt: serverOptions.renderer.mvt }); // initialize render cache diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 6d8575a7..25e60c2f 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -81,6 +81,7 @@ module.exports = { statsInterval: rendererConfig.statsInterval }, renderer: { + mvt: rendererConfig.mvt, mapnik: _.defaults(rendererConfig.mapnik, { geojson: { dbPoolParams: { From e5cae8b8e3e530fc891cfccb1be0e4c9cdbcfcd0 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 28 Sep 2017 13:24:50 +0200 Subject: [PATCH 02/35] Added flag documentation --- config/environments/development.js.example | 3 +++ config/environments/production.js.example | 3 +++ config/environments/staging.js.example | 3 +++ config/environments/test.js.example | 3 +++ 4 files changed, 12 insertions(+) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index f09a3808..2bc6d2d6 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -108,6 +108,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 1dd80212..87214c75 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -102,6 +102,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 7dd9f5b0..5e05c83f 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -102,6 +102,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index b266c923..c01d52d1 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -101,6 +101,9 @@ var config = { cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation usePgSQL: false }, mapnik: { From 54632485787b666978a1a691e1d413a8c8ad9fe5 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 28 Sep 2017 17:57:59 +0200 Subject: [PATCH 03/35] Changed PostGIS MVT flag name --- config/environments/development.js.example | 10 +++++++++- config/environments/production.js.example | 10 +++++++++- config/environments/staging.js.example | 10 +++++++++- config/environments/test.js.example | 10 +++++++++- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 2bc6d2d6..91cf2a39 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -111,7 +111,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 87214c75..7d7a00cb 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -105,7 +105,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 5e05c83f..3d1098fb 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -105,7 +105,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend diff --git a/config/environments/test.js.example b/config/environments/test.js.example index c01d52d1..ea788a22 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -104,7 +104,15 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePgSQL: false + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } }, mapnik: { // The size of the pool of internal mapnik backend From 226653207ab3d11bf00e507c1368645afa2da945 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 12:17:55 +0200 Subject: [PATCH 04/35] target windshaft master branch --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a3ca339d..649759cd 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.0", "underscore": "~1.6.0", - "windshaft": "3.3.2", + "windshaft": "cartodb/windshaft", "yargs": "~5.0.0" }, "devDependencies": { @@ -58,7 +58,8 @@ "scripts": { "lint": "jshint lib test", "preinstall": "make pre-install", - "test": "make test-all" + "test": "make test-all", + "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn" }, "engines": { "node": ">=6.9", From 72a9a3e0970d93ed98df7f5707876edac42fbba0 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 12:26:19 +0200 Subject: [PATCH 05/35] updated deps --- yarn.lock | 110 +++++++++++++++--------------------------------------- 1 file changed, 31 insertions(+), 79 deletions(-) diff --git a/yarn.lock b/yarn.lock index 6cae1a4a..d6e719a2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -"abaculus@github:cartodb/abaculus#2.0.3-cdb1": +abaculus@cartodb/abaculus#2.0.3-cdb1: version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,11 +10,7 @@ mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" - -abbrev@1.0.x: +abbrev@1, abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -226,7 +222,7 @@ camshaft@0.59.2: dot "^1.0.3" request "^2.69.0" -"canvas@github:cartodb/node-canvas#1.6.2-cdb2": +canvas@cartodb/node-canvas#1.6.2-cdb2: version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -244,6 +240,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@cartodb/carto#0.15.1-cdb3: + version "0.15.1-cdb3" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "1.8.3" + "carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" @@ -252,14 +256,6 @@ carto@0.16.3: optimist "~0.6.0" underscore "~1.6.0" -"carto@github:cartodb/carto#0.15.1-cdb3": - version "0.15.1-cdb3" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "1.8.3" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -520,11 +516,7 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3: - version "1.1.2" - resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" - -dot@~1.0.2: +dot@^1.0.3, dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -754,7 +746,7 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generic-pool@2.4.3: +generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -766,10 +758,6 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" -generic-pool@~2.4.0, generic-pool@~2.4.1: - version "2.4.6" - resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" - get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -780,7 +768,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1: +glob@7.1.1, glob@^7.0.5, glob@^7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -811,17 +799,6 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.0.5, glob@^7.1.1: - version "7.1.2" - resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" - dependencies: - fs.realpath "^1.0.0" - inflight "^1.0.4" - inherits "2" - minimatch "^3.0.4" - once "^1.3.0" - path-is-absolute "^1.0.0" - graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1323,13 +1300,13 @@ mime@~1.3.4: version "1.3.6" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8: +minimist@0.0.8, minimist@~0.0.1: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1337,10 +1314,6 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" -minimist@~0.0.1: - version "0.0.10" - resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" - minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1476,14 +1449,10 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0: +object-assign@4.1.0, object-assign@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" -object-assign@^4.1.0: - version "4.1.1" - resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1661,8 +1630,8 @@ postcss@5.0.19: supports-color "^3.1.2" postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.17" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b" + version "5.2.18" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.18.tgz#badfa1497d46244f6390f58b319830d9107853c5" dependencies: chalk "^1.1.3" js-base64 "^2.1.9" @@ -1771,7 +1740,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1: +readable-stream@1.1, readable-stream@~1.1.9: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1801,15 +1770,6 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@~1.1.9: - version "1.1.14" - resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" - dependencies: - core-util-is "~1.0.0" - inherits "~2.0.1" - isarray "0.0.1" - string_decoder "~0.10.x" - redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1919,9 +1879,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" semver@4.3.2: version "4.3.2" @@ -1935,10 +1895,6 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" -semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" - send@0.16.1: version "0.16.1" resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" @@ -2036,14 +1992,10 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4: +sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" -sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: - version "1.0.5" - resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" - split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2141,7 +2093,7 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" -supports-color@3.1.2: +supports-color@3.1.2, supports-color@^3.1.0, supports-color@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" dependencies: @@ -2151,7 +2103,7 @@ supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" -supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: +supports-color@^3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.2.3.tgz#65ac0504b3954171d8a64946b2ae3cbb8a5f54f6" dependencies: @@ -2189,7 +2141,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": +tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2197,7 +2149,7 @@ through@2: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": +tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2358,9 +2310,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.3.tgz#0582e6a0d9cf91c533134787ace64a3337200e33" +windshaft@cartodb/windshaft: + version "4.0.0" + resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/5b21e24e66c8e24cb88640b606aaf5fefcd3f365" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 54b7ee85c2e2d0e97ee85ddfe9ce96dae8849cfa Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 14:38:43 +0200 Subject: [PATCH 06/35] Geojson tests adapted to MVT --- .../analysis/analysis-layers-geojson.js | 81 ---- test/acceptance/geojson-renderer.js | 343 -------------- test/acceptance/mvt.js | 433 +++++++++++++++++- test/support/test-client.js | 2 +- 4 files changed, 431 insertions(+), 428 deletions(-) delete mode 100644 test/acceptance/analysis/analysis-layers-geojson.js delete mode 100644 test/acceptance/geojson-renderer.js diff --git a/test/acceptance/analysis/analysis-layers-geojson.js b/test/acceptance/analysis/analysis-layers-geojson.js deleted file mode 100644 index 59a0900b..00000000 --- a/test/acceptance/analysis/analysis-layers-geojson.js +++ /dev/null @@ -1,81 +0,0 @@ -require('../../support/test_helper'); - -var assert = require('../../support/assert'); -var TestClient = require('../../support/test-client'); - -describe('analysis-layers-dataviews-geojson', function() { - - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - - var CARTOCSS = [ - "#points {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 1.0;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" - ].join('\n'); - - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "2570e105-7b37-40d2-bdf4-1af889598745" - }, - "cartocss": CARTOCSS, - "cartocss_version": "2.3.0" - } - } - ], - { - pop_max_histogram: { - source: { - id: '2570e105-7b37-40d2-bdf4-1af889598745' - }, - type: 'histogram', - options: { - column: 'pop_max' - } - } - }, - [ - { - "id": "2570e105-7b37-40d2-bdf4-1af889598745", - "type": "source", - "params": { - "query": "select * from populated_places_simple_reduced" - } - } - ] - ); - - it('should get pop_max column from dataview', function(done) { - var testClient = new TestClient(mapConfig, 1234); - - testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(Array.isArray(geojson.features)); - assert.ok(geojson.features.length > 0); - var feature = geojson.features[0]; - assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); - - testClient.drain(done); - }); - }); - -}); diff --git a/test/acceptance/geojson-renderer.js b/test/acceptance/geojson-renderer.js deleted file mode 100644 index 089e09b5..00000000 --- a/test/acceptance/geojson-renderer.js +++ /dev/null @@ -1,343 +0,0 @@ -require('../support/test_helper'); - -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); - - -describe('use only needed columns', function() { - - function getFeatureByCartodbId(features, cartodbId) { - for (var i = 0, len = features.length; i < len; i++) { - if (features[i].properties.cartodb_id === cartodbId) { - return features[i]; - } - } - return {}; - } - - var options = { format: 'geojson', layer: 0 }; - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - it('with aggregation widget, interactivity and cartocss columns', function(done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', - cartocss_version: '2.0.1', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_min" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373, - pop_min: 57586 - }); - done(); - }); - }); - - it('should not duplicate columns', function(done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: ['#layer0 {', - 'marker-fill: red;', - 'marker-width: 10;', - '[name="Madrid"] { marker-fill: green; } ', - '[pop_max>100000] { marker-fill: black; } ', - '}'].join('\n'), - cartocss_version: '2.3.0', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_max" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373 - }); - done(); - }); - }); - - it('with formula widget, no interactivity and no cartocss columns', function(done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id', - widgets: { - pop_max_f: { - type: 'formula', - options: { - column: 'pop_max', - operation: 'count' - } - } - } - } - }] - }; - - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max: 71373 - }); - done(); - }); - }); - it('with cartocss with multiple expressions', function(done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + - '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id' - } - }] - }; - - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max:71373, - name:"Mardin", - adm0name:"Turkey" - }); - done(); - }); - }); - - it('should work with mapnik substitution tokens', function(done) { - var cartocss = [ - "#layer {", - " line-width: 2;", - " line-color: #3B3B58;", - " line-opacity: 1;", - " polygon-opacity: 0.7;", - " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", - "}" - ].join('\n'); - - var sql = [ - 'WITH hgrid AS (', - ' SELECT CDB_HexagonGrid(', - ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', - ' greatest(!pixel_width!,!pixel_height!) * 100', - ' ) as cell', - ')', - 'SELECT', - ' hgrid.cell as the_geom_webmercator,', - ' count(1) as points_count,', - ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', - ' 1 as cartodb_id', - 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', - 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', - 'GROUP BY hgrid.cell' - ].join('\n'); - - var mapConfig = { - "version": "1.4.0", - "layers": [ - { - "type": 'mapnik', - "options": { - "cartocss_version": '2.3.0', - "sql": sql, - "cartocss": cartocss - } - } - ] - }; - - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(geojson); - assert.equal(geojson.features.length, 5); - - done(); - }); - }); - - it('should skip empty and null columns for geojson tiles', function(done) { - - var mapConfig = { - "analyses": [ - { - "id": "a0", - "params": { - "query": "SELECT * FROM test_table" - }, - "type": "source" - } - ], - "dataviews": { - "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "74f590f8-625c-4e95-922f-34ad3e9919c0": { - "options": { - "aggregation": "sum", - "aggregationColumn": "cartodb_id", - "column": "name" - }, - "source": { - "id": "a0" - }, - "type": "aggregation" - }, - "98a75757-3006-400a-b028-fb613a6c0b69": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "sum" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { - "options": { - "aggregationColumn": null, - "bins": "12", - "column": "cartodb_id" - }, - "source": { - "id": "a0" - }, - "type": "histogram" - }, - "ebc0653f-3581-469c-8b31-c969e440a865": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - } - }, - "layers": [ - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" - }, - "type": "http" - }, - { - "options": { - "attributes": { - "columns": [ - "name", - "address" - ], - "id": "cartodb_id" - }, - "cartocss": "#layer { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0", - "interactivity": "cartodb_id", - "layer_name": "wadus", - "source": { - "id": "a0" - } - }, - "type": "cartodb" - }, - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" - }, - "type": "http" - } - ] - }; - - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(geojson); - assert.equal(geojson.features.length, 5); - - assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']); - - done(); - }); - }); - -}); diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b5a59cd8..b6a04d79 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -1,9 +1,9 @@ require('../support/test_helper'); -const assert = require('../support/assert'); -const TestClient = require('../support/test-client'); +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); -function createMapConfig (sql = TestClient.SQL.ONE_POINT) { +function createMapConfig(sql = TestClient.SQL.ONE_POINT) { return { version: '1.6.0', layers: [{ @@ -19,6 +19,86 @@ function createMapConfig (sql = TestClient.SQL.ONE_POINT) { } describe('mvt', function () { + + describe('analysis-layers-dataviews-mvt', function () { + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var CARTOCSS = [ + "#points {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 1.0;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", + "}" + ].join('\n'); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": CARTOCSS, + "cartocss_version": "2.3.0" + } + } + ], + { + pop_max_histogram: { + source: { + id: '2570e105-7b37-40d2-bdf4-1af889598745' + }, + type: 'histogram', + options: { + column: 'pop_max' + } + } + }, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ); + + it('should get pop_max column from dataview', function (done) { + var testClient = new TestClient(mapConfig, 1234); + + testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(Array.isArray(geojson.features)); + assert.ok(geojson.features.length > 0); + var feature = geojson.features[0]; + assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); + + testClient.drain(done); + }); + }); + + }); + + const testCases = [ { desc: 'should get empty mvt with code 204 (no content)', @@ -60,4 +140,351 @@ describe('mvt', function () { }); }); }); + + + + describe('use only needed columns', function () { + + function getFeatureByCartodbId(features, cartodbId) { + for (var i = 0, len = features.length; i < len; i++) { + if (features[i].properties.cartodb_id === cartodbId) { + return features[i]; + } + } + return {}; + } + + var options = { format: 'mvt', layer: 0 }; + + afterEach(function (done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + it('with aggregation widget, interactivity and cartocss columns', function (done) { + var widgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', + cartocss_version: '2.0.1', + widgets: { + adm0name: { + type: 'aggregation', + options: { + column: 'adm0name', + aggregation: 'sum', + aggregationColumn: 'pop_max' + } + } + }, + interactivity: "cartodb_id,pop_min" + } + }] + }; + + this.testClient = new TestClient(widgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + name: 'Mardin', + adm0name: 'Turkey', + pop_max: 71373, + pop_min: 57586 + }); + done(); + }); + }); + + it('should not duplicate columns', function (done) { + var widgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: ['#layer0 {', + 'marker-fill: red;', + 'marker-width: 10;', + '[name="Madrid"] { marker-fill: green; } ', + '[pop_max>100000] { marker-fill: black; } ', + '}'].join('\n'), + cartocss_version: '2.3.0', + widgets: { + adm0name: { + type: 'aggregation', + options: { + column: 'adm0name', + aggregation: 'sum', + aggregationColumn: 'pop_max' + } + } + }, + interactivity: "cartodb_id,pop_max" + } + }] + }; + + this.testClient = new TestClient(widgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + name: 'Mardin', + adm0name: 'Turkey', + pop_max: 71373 + }); + done(); + }); + }); + + it('with formula widget, no interactivity and no cartocss columns', function (done) { + var formulaWidgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id', + widgets: { + pop_max_f: { + type: 'formula', + options: { + column: 'pop_max', + operation: 'count' + } + } + } + } + }] + }; + + this.testClient = new TestClient(formulaWidgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + pop_max: 71373 + }); + done(); + }); + }); + it('with cartocss with multiple expressions', function (done) { + var formulaWidgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + + '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + + '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + + '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id' + } + }] + }; + + this.testClient = new TestClient(formulaWidgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + pop_max: 71373, + name: "Mardin", + adm0name: "Turkey" + }); + done(); + }); + }); + + it('should work with mapnik substitution tokens', function (done) { + var cartocss = [ + "#layer {", + " line-width: 2;", + " line-color: #3B3B58;", + " line-opacity: 1;", + " polygon-opacity: 0.7;", + " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", + "}" + ].join('\n'); + + var sql = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + var mapConfig = { + "version": "1.4.0", + "layers": [ + { + "type": 'mapnik', + "options": { + "cartocss_version": '2.3.0', + "sql": sql, + "cartocss": cartocss + } + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(geojsonTile); + assert.equal(geojsonTile.features.length, 5); + + done(); + }); + }); + + it('should skip empty and null columns for geojson tiles', function (done) { + + var mapConfig = { + "analyses": [ + { + "id": "a0", + "params": { + "query": "SELECT * FROM test_table" + }, + "type": "source" + } + ], + "dataviews": { + "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "avg" + }, + "source": { + "id": "a0" + }, + "type": "formula" + }, + "74f590f8-625c-4e95-922f-34ad3e9919c0": { + "options": { + "aggregation": "sum", + "aggregationColumn": "cartodb_id", + "column": "name" + }, + "source": { + "id": "a0" + }, + "type": "aggregation" + }, + "98a75757-3006-400a-b028-fb613a6c0b69": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "sum" + }, + "source": { + "id": "a0" + }, + "type": "formula" + }, + "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { + "options": { + "aggregationColumn": null, + "bins": "12", + "column": "cartodb_id" + }, + "source": { + "id": "a0" + }, + "type": "histogram" + }, + "ebc0653f-3581-469c-8b31-c969e440a865": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "avg" + }, + "source": { + "id": "a0" + }, + "type": "formula" + } + }, + "layers": [ + { + "options": { + "subdomains": "abcd", + "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" + }, + "type": "http" + }, + { + "options": { + "attributes": { + "columns": [ + "name", + "address" + ], + "id": "cartodb_id" + }, + "cartocss": "#layer { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0", + "interactivity": "cartodb_id", + "layer_name": "wadus", + "source": { + "id": "a0" + } + }, + "type": "cartodb" + }, + { + "options": { + "subdomains": "abcd", + "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" + }, + "type": "http" + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(geojsonTile); + assert.equal(geojsonTile.features.length, 5); + + assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']); + + done(); + }); + }); + + }); }); + diff --git a/test/support/test-client.js b/test/support/test-client.js index 21918665..b3c6566b 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -20,7 +20,7 @@ const MAPNIK_SUPPORTED_FORMATS = { 'png': true, 'png32': true, 'grid.json': true, - 'geojson': true, + 'mvt': true, 'mvt': true } From 34d9e5a4eb3aaf9e72c282589b253e259ed2476a Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 16:08:05 +0200 Subject: [PATCH 07/35] Fix MVT test --- test/acceptance/mvt.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b6a04d79..dbfadd26 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -87,9 +87,9 @@ describe('mvt', function () { var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); assert.ok(!err, err); - assert.ok(Array.isArray(geojson.features)); - assert.ok(geojson.features.length > 0); - var feature = geojson.features[0]; + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + var feature = geojsonTile.features[0]; assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); testClient.drain(done); From f387f2ee6f2b5169ffb1bfba3016dc065f5181f9 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Thu, 5 Oct 2017 16:08:31 +0200 Subject: [PATCH 08/35] Testing dockerified --- Dockerfile | 33 +++++++++++++++++++++++++++++++++ package.json | 7 ++++++- run_tests.sh | 2 +- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..3b4fd258 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +FROM ubuntu:xenial + + +RUN apt-get -y update +RUN apt-get install -y software-properties-common +RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test +RUN apt-get -y update +RUN apt-get -y install curl wget protobuf-c-compiler + +#Use UTF8 to avoid encoding problems with pgsql +RUN apt-get -y install locales +ENV LANG C.UTF-8 +RUN locale-gen en_US.UTF-8 +RUN update-locale LANG=en_US.UTF-8 + +#Add 6.X node ppa +RUN curl -sL https://deb.nodesource.com/setup_6.x | bash +RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev + +# Install PostGIS 2.4 from sources +RUN wget http://download.osgeo.org/postgis/source/postgis-2.4.0.tar.gz && tar xvfz postgis-2.4.0.tar.gz && cd postgis-2.4.0 && ./configure && make && make install && cd .. && rm -rf postgis-2.4.0 + +# Configure PostgreSQL +RUN echo "local all all trust" > /etc/postgresql/9.5/main/pg_hba.conf +RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf +RUN echo "host all all ::1/128 trust" >> /etc/postgresql/9.5/main/pg_hba.conf +RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf + +RUN apt-get -y remove wget protobuf-c-compiler +RUN apt-get -y autoremove + +WORKDIR /srv +CMD export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && npm install && export PGUSER=postgres && /etc/init.d/postgresql start && createdb template_postgis && createuser publicuser && psql -c "CREATE EXTENSION postgis" template_postgis && npm test diff --git a/package.json b/package.json index 9f0581e4..8ff78dce 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,12 @@ "lint": "jshint lib test", "preinstall": "make pre-install", "test": "make test-all", - "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn" + "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", + "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", + "docker-pull": "docker pull cartoimages/windshaft-carto-testing", + "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing", + "docker-build": "docker build -t cartoimages/windshaft-carto-testing .", + "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, "engines": { "node": ">=6.9", diff --git a/run_tests.sh b/run_tests.sh index 92e8621b..d1c63614 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS} else echo "Running tests" - mocha -u tdd -t 5000 ${TESTS} + ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS} fi ret=$? From cab2d6d5d4e51e120a3912db6cce616c502dcca8 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 11:03:08 +0200 Subject: [PATCH 09/35] package.json docker-bash script added --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 8ff78dce..5af0b9d3 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", "docker-pull": "docker pull cartoimages/windshaft-carto-testing", "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing", + "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-carto-testing bash", "docker-build": "docker build -t cartoimages/windshaft-carto-testing .", "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, From 4e6253b71794e5acf7fb164c4599d6ca08886854 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:02:16 +0200 Subject: [PATCH 10/35] Return HTTP 204 for empty tiles --- lib/cartodb/controllers/layergroup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index 9644aa06..f6089a00 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -313,7 +313,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.sendResponse(req, res, tile, 200, headers); + this.sendResponse(req, res, tile, tile.length===0? 204:200, headers); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } From d29da0bcc35ab995b880c23081442068779c9f6c Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:07:24 +0200 Subject: [PATCH 11/35] Test both MVT paths: mapnik and PostGIS --- test/acceptance/mvt.js | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index dbfadd26..444c72fe 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -2,6 +2,7 @@ require('../support/test_helper'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); function createMapConfig(sql = TestClient.SQL.ONE_POINT) { return { @@ -18,7 +19,14 @@ function createMapConfig(sql = TestClient.SQL.ONE_POINT) { }; } -describe('mvt', function () { +describe('mvt (mapnik)', mvt(false)); +describe('mvt (postgis)', mvt(true)); + +function mvt(usePostGIS) { +return function () { + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); describe('analysis-layers-dataviews-mvt', function () { @@ -81,7 +89,7 @@ describe('mvt', function () { ); it('should get pop_max column from dataview', function (done) { - var testClient = new TestClient(mapConfig, 1234); + var testClient = new TestClient(mapConfig); testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); @@ -128,23 +136,25 @@ describe('mvt', function () { testCases.forEach(function (test) { it(test.desc, done => { - const testClient = new TestClient(test.mapConfig, 1234); + var testClient = new TestClient(test.mapConfig); const { z, x, y } = test.coords; const { format, response } = test; testClient.getTile(z, x, y, { format, response }, (err, res) => { assert.ifError(err); - - assert.equal(res.statusCode, test.response.status); testClient.drain(done); }); }); }); + if (usePostGIS){ + describe('use only needed columns', onlyNeededColumns); + }else{ + describe.skip('use only needed columns', onlyNeededColumns); + } - describe('use only needed columns', function () { - + function onlyNeededColumns() { function getFeatureByCartodbId(features, cartodbId) { for (var i = 0, len = features.length; i < len; i++) { if (features[i].properties.cartodb_id === cartodbId) { @@ -268,7 +278,6 @@ describe('mvt', function () { } }] }; - this.testClient = new TestClient(formulaWidgetMapConfig); this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); @@ -485,6 +494,6 @@ describe('mvt', function () { }); }); - }); -}); - + } +} +} \ No newline at end of file From 78671aa499eabbfccc5e3c2a57238beb20ea8a32 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:07:47 +0200 Subject: [PATCH 12/35] remove redundant format support --- test/support/test-client.js | 1 - 1 file changed, 1 deletion(-) diff --git a/test/support/test-client.js b/test/support/test-client.js index b3c6566b..1f5272f6 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -20,7 +20,6 @@ const MAPNIK_SUPPORTED_FORMATS = { 'png': true, 'png32': true, 'grid.json': true, - 'mvt': true, 'mvt': true } From 43885f130bd56adcd1d6fb16578eb48a53b7f472 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:19:00 +0200 Subject: [PATCH 13/35] Fix HTTP status code distinction between 200 and 204 --- lib/cartodb/controllers/layergroup.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index f6089a00..ed0f3eb5 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -278,6 +278,10 @@ LayergroupController.prototype.tileOrLayer = function (req, res) { ); }; +function getStatusCode(tile, format){ + return tile.length===0 && format==='mvt'? 204:200; +} + // This function is meant for being called as the very last // step by all endpoints serving tiles or grids LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers) { @@ -313,7 +317,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.sendResponse(req, res, tile, tile.length===0? 204:200, headers); + this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } From a706fd81ba33d512061c54873fb247766f679242 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:21:34 +0200 Subject: [PATCH 14/35] Restore MVT path configuration after each suite pass --- test/acceptance/mvt.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 444c72fe..af67f8bf 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -24,9 +24,13 @@ describe('mvt (postgis)', mvt(true)); function mvt(usePostGIS) { return function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; before(function () { serverOptions.renderer.mvt.usePostGIS = usePostGIS; }); + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); describe('analysis-layers-dataviews-mvt', function () { From cac16f8b66069af278a3fd6989e4ab10ec20d84b Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:30:35 +0200 Subject: [PATCH 15/35] Travis dockerified --- .travis.yml | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8f3993a2..d55cc94a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,14 @@ +sudo: required dist: trusty -addons: - postgresql: "9.5" - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - postgresql-9.5-postgis-2.3 - - postgresql-plpython-9.5 - - pkg-config - - libcairo2-dev - - libjpeg8-dev - - libgif-dev - - libpango1.0-dev - - g++-4.9 + +services: + - docker before_install: - - createdb template_postgis - - createuser publicuser - - psql -c "CREATE EXTENSION postgis" template_postgis + - docker pull dmanzanares/windshaft-carto-testing -env: - - NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9 +script: + - docker run -v `pwd`:/srv dmanzanares/windshaft-carto-testing + +language: generic -language: node_js -node_js: - - "6" From 7f66189164964101398c2caddd948d19cd18db4f Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 16:44:28 +0200 Subject: [PATCH 16/35] Fix travis.yaml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d55cc94a..b946378f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,10 +5,10 @@ services: - docker before_install: - - docker pull dmanzanares/windshaft-carto-testing + - docker pull cartoimages/windshaft-carto-testing script: - - docker run -v `pwd`:/srv dmanzanares/windshaft-carto-testing + - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing language: generic From f456237aa7197aef5f5cb2323faaa7da1c7e6817 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 15:53:47 +0000 Subject: [PATCH 17/35] Drain client on after hook --- test/acceptance/buffer-size-format.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index c32f62d1..236478b8 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -124,21 +124,28 @@ describe('buffer size per format', function () { } ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.mapConfig, 1234); + this.testClient = new TestClient(test.mapConfig, 1234); var coords = test.coords; var options = { format: test.format, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: // tile.save(test.fixturePath); test.assert(tile, function (err) { assert.ifError(err); - testClient.drain(done); + return done(); }); }); }); From c004e105efd6e63a51c8322052d45963d52147e4 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 6 Oct 2017 16:17:26 +0000 Subject: [PATCH 18/35] Drain client on after hooks --- test/acceptance/buffer-size-format.js | 37 +++++++++++++++------------ 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 236478b8..e41cf1d3 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -143,10 +143,7 @@ describe('buffer size per format', function () { assert.ifError(err); // To generate images use: // tile.save(test.fixturePath); - test.assert(tile, function (err) { - assert.ifError(err); - return done(); - }); + test.assert(tile, done); }); }); }); @@ -267,23 +264,27 @@ describe('buffer size per format for named maps', function () { } ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.template, 1234); + this.testClient = new TestClient(test.template, 1234); var coords = test.coords; var options = { format: test.format, placeholders: test.placeholders, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png'); - test.assert(tile, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + test.assert(tile, done); }); }); }); @@ -423,25 +424,29 @@ describe('buffer size per format for named maps w/o placeholders', function () { ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.template, 1234); + this.testClient = new TestClient(test.template, 1234); var coords = test.coords; var options = { format: test.format, placeholders: test.placeholders, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: //tile.save(test.fixturePath); // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); - test.assert(tile, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + test.assert(tile, done); }); }); }); From 28cb05e45b5f3b6123149d2dea7342edf3df3627 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 18:24:11 +0200 Subject: [PATCH 19/35] Use windshaft docker image --- Dockerfile | 13 ++++++++----- docker-test.sh | 10 ++++++++++ package.json | 7 +++---- 3 files changed, 21 insertions(+), 9 deletions(-) create mode 100644 docker-test.sh diff --git a/Dockerfile b/Dockerfile index 3b4fd258..d16731c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,23 @@ FROM ubuntu:xenial - +# Add external repos RUN apt-get -y update RUN apt-get install -y software-properties-common RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test RUN apt-get -y update -RUN apt-get -y install curl wget protobuf-c-compiler -#Use UTF8 to avoid encoding problems with pgsql +# Use UTF8 to avoid encoding problems with pgsql RUN apt-get -y install locales ENV LANG C.UTF-8 RUN locale-gen en_US.UTF-8 RUN update-locale LANG=en_US.UTF-8 -#Add 6.X node ppa +# Add 6.X node ppa +RUN apt-get -y install curl RUN curl -sL https://deb.nodesource.com/setup_6.x | bash -RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev + +# Install dependencies +RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev wget protobuf-c-compiler # Install PostGIS 2.4 from sources RUN wget http://download.osgeo.org/postgis/source/postgis-2.4.0.tar.gz && tar xvfz postgis-2.4.0.tar.gz && cd postgis-2.4.0 && ./configure && make && make install && cd .. && rm -rf postgis-2.4.0 @@ -26,6 +28,7 @@ RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/ RUN echo "host all all ::1/128 trust" >> /etc/postgresql/9.5/main/pg_hba.conf RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf +# Clean RUN apt-get -y remove wget protobuf-c-compiler RUN apt-get -y autoremove diff --git a/docker-test.sh b/docker-test.sh new file mode 100644 index 00000000..e340369e --- /dev/null +++ b/docker-test.sh @@ -0,0 +1,10 @@ +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres + +npm install + + /etc/init.d/postgresql start + +createdb template_postgis && createuser publicuser +psql -c "CREATE EXTENSION postgis" template_postgis + +npm test \ No newline at end of file diff --git a/package.json b/package.json index 5af0b9d3..b65a64d1 100644 --- a/package.json +++ b/package.json @@ -61,10 +61,9 @@ "test": "make test-all", "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", - "docker-pull": "docker pull cartoimages/windshaft-carto-testing", - "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing", - "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-carto-testing bash", - "docker-build": "docker build -t cartoimages/windshaft-carto-testing .", + "docker-pull": "docker pull cartoimages/windshaft-testing", + "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh", + "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-testing bash", "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, "engines": { From 0e20958220aed2c3e8261c302b76608cc976ef04 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Fri, 6 Oct 2017 18:24:29 +0200 Subject: [PATCH 20/35] Remove dockerfile --- Dockerfile | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d16731c5..00000000 --- a/Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -FROM ubuntu:xenial - -# Add external repos -RUN apt-get -y update -RUN apt-get install -y software-properties-common -RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test -RUN apt-get -y update - -# Use UTF8 to avoid encoding problems with pgsql -RUN apt-get -y install locales -ENV LANG C.UTF-8 -RUN locale-gen en_US.UTF-8 -RUN update-locale LANG=en_US.UTF-8 - -# Add 6.X node ppa -RUN apt-get -y install curl -RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - -# Install dependencies -RUN apt-get -y install make libpixman-1-dev pkg-config postgresql-9.5 libcairo2-dev libjpeg8-dev libgif-dev libpango1.0-dev libgdal1i libgeos-dev libxml2-dev libgdal-dev libproj-dev postgresql-server-dev-9.5 redis-server nodejs gcc-4.9 g++-4.9 libprotobuf-c-dev git postgresql-plpython-9.5 libjson-c-dev wget protobuf-c-compiler - -# Install PostGIS 2.4 from sources -RUN wget http://download.osgeo.org/postgis/source/postgis-2.4.0.tar.gz && tar xvfz postgis-2.4.0.tar.gz && cd postgis-2.4.0 && ./configure && make && make install && cd .. && rm -rf postgis-2.4.0 - -# Configure PostgreSQL -RUN echo "local all all trust" > /etc/postgresql/9.5/main/pg_hba.conf -RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/9.5/main/pg_hba.conf -RUN echo "host all all ::1/128 trust" >> /etc/postgresql/9.5/main/pg_hba.conf -RUN echo "listen_addresses='*'" >> /etc/postgresql/9.5/main/postgresql.conf - -# Clean -RUN apt-get -y remove wget protobuf-c-compiler -RUN apt-get -y autoremove - -WORKDIR /srv -CMD export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && npm install && export PGUSER=postgres && /etc/init.d/postgresql start && createdb template_postgis && createuser publicuser && psql -c "CREATE EXTENSION postgis" template_postgis && npm test From 28501f6b9d8c6a78312ce35af67669f12495f6f1 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 14:46:01 +0200 Subject: [PATCH 21/35] MVT PostGIS limit test fixed --- test/acceptance/user-render-timeout-limit.js | 84 +++++++++++--------- 1 file changed, 46 insertions(+), 38 deletions(-) diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 45e41f89..eaf98805 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -2,6 +2,7 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; @@ -200,49 +201,56 @@ describe('user render timeout limit', function () { }); }); - describe('vector', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); - }); - - afterEach(function (done) { - this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); + describe('vector (PostGIS)', vector(true)); + describe('vector (mapnik)', vector(false)); + function vector(usePostGIS) { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + return function () { + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(50, done); }); - }); - it('layergroup creation works but vector tile request fails due to render timeout', function (done) { - const params = { - format: 'mvt', - response: { - status: 429, - headers: { - 'Content-Type': 'application/json; charset=utf-8' + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); } - } - }; - - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { - assert.ifError(err); - - assert.deepEqual(tile, { - errors: ['You are over platform\'s limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'render', - message: 'You are over platform\'s limits. Please contact us to know more details' - }] + this.testClient.drain(done); }); - - done(); }); - }); - }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details' + }] + }); + + done(); + }); + }); + }; + } describe('interativity', function () { beforeEach(function (done) { From 6af83d7630e23f6432bdcfd1828f9e7858756c64 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 14:54:03 +0200 Subject: [PATCH 22/35] Use yarn instead of npm in docker --- docker-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-test.sh b/docker-test.sh index e340369e..6bf5360e 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,6 +1,7 @@ export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres -npm install +npm install -g yarn@0.27.5 +yarn /etc/init.d/postgresql start From 1766cd0ad4235dcc4c5ce034427dd85a1aa4ed6f Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 15:42:24 +0200 Subject: [PATCH 23/35] Buffersize test fixed to test PostGIS and mapnik MVT paths --- test/acceptance/buffer-size-format.js | 88 ++++++++++++++++++--------- 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index e41cf1d3..b888e094 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -3,6 +3,7 @@ require('../support/test_helper'); var fs = require('fs'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); var mapnik = require('windshaft').mapnik; var IMAGE_TOLERANCE_PER_MIL = 5; @@ -131,21 +132,35 @@ describe('buffer size per format', function () { return done(); }); + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { - it(test.desc, function (done) { - this.testClient = new TestClient(test.mapConfig, 1234); - var coords = test.coords; - var options = { - format: test.format, - layers: test.layers - }; - this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - // tile.save(test.fixturePath); - test.assert(tile, done); + var testFn = (usePostGIS) => { + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + it(test.desc, function (done) { + this.testClient = new TestClient(test.mapConfig, 1234); + var coords = test.coords; + var options = { + format: test.format, + layers: test.layers + }; + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + // tile.save(test.fixturePath); + test.assert(tile, done); + }); }); - }); + }; + if (test.format === 'mvt'){ + testFn(true); + } + testFn(false); }); }); @@ -431,23 +446,36 @@ describe('buffer size per format for named maps w/o placeholders', function () { return done(); }); + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { - it(test.desc, function (done) { - this.testClient = new TestClient(test.template, 1234); - var coords = test.coords; - var options = { - format: test.format, - placeholders: test.placeholders, - layers: test.layers - }; - this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - //tile.save(test.fixturePath); - // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); - // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); - test.assert(tile, done); - }); - }); + var testFn = (usePostGIS) => { + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + after(function () { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + it(test.desc, function (done) { + this.testClient = new TestClient(test.template, 1234); + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders, + layers: test.layers + }; + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save(test.fixturePath); + // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); + // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); + test.assert(tile, done); + }); + }); + }; + if (test.format === 'mvt'){ + testFn(true); + } + testFn(false); }); }); From 82648df21cc05a4e31be11e645ad50d4142bcca6 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 15:49:51 +0200 Subject: [PATCH 24/35] Fix jshint --- test/acceptance/mvt.js | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index af67f8bf..cccfbcf4 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -144,7 +144,7 @@ return function () { const { z, x, y } = test.coords; const { format, response } = test; - testClient.getTile(z, x, y, { format, response }, (err, res) => { + testClient.getTile(z, x, y, { format, response }, err => { assert.ifError(err); testClient.drain(done); }); @@ -185,7 +185,8 @@ return function () { type: 'mapnik', options: { sql: 'select * from populated_places_simple_reduced', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', + cartocss: + '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', cartocss_version: '2.0.1', widgets: { adm0name: { @@ -336,20 +337,20 @@ return function () { ].join('\n'); var sql = [ - 'WITH hgrid AS (', - ' SELECT CDB_HexagonGrid(', - ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', - ' greatest(!pixel_width!,!pixel_height!) * 100', - ' ) as cell', - ')', - 'SELECT', - ' hgrid.cell as the_geom_webmercator,', - ' count(1) as points_count,', - ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', - ' 1 as cartodb_id', - 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', - 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', - 'GROUP BY hgrid.cell' + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' ].join('\n'); var mapConfig = { @@ -499,5 +500,5 @@ return function () { }); } -} +}; } \ No newline at end of file From 382790153585c24ba1ec0d276db8b3d68892fb63 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 15:53:50 +0200 Subject: [PATCH 25/35] Skip test to ignore MVT v2 compliance --- test/acceptance/mvt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index cccfbcf4..2bead7e4 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -325,7 +325,7 @@ return function () { }); }); - it('should work with mapnik substitution tokens', function (done) { + it.skip('should work with mapnik substitution tokens', function (done) { var cartocss = [ "#layer {", " line-width: 2;", From d77739dfa43a7ea5a9ed8870e41a5d1a20271b9a Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 16:46:23 +0200 Subject: [PATCH 26/35] PostGIS MVT v2 compliance test skipped without skipping mapnik --- test/acceptance/mvt.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 2bead7e4..883f050f 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -325,7 +325,8 @@ return function () { }); }); - it.skip('should work with mapnik substitution tokens', function (done) { + var skipOnPostGIS = usePostGIS ? it.skip: it; + skipOnPostGIS('should work with mapnik substitution tokens', function (done) { var cartocss = [ "#layer {", " line-width: 2;", From 8a88b29665ae3499efd10df3f6722f718cdaa43b Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 17:16:29 +0200 Subject: [PATCH 27/35] update dependencies --- yarn.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/yarn.lock b/yarn.lock index d6e719a2..cf95c05b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -240,6 +240,14 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" +carto@CartoDB/carto#0.15.1-cdb1: + version "0.15.1-cdb1" + resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + dependencies: + mapnik-reference "~6.0.2" + optimist "~0.6.0" + underscore "~1.6.0" + carto@cartodb/carto#0.15.1-cdb3: version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" @@ -248,14 +256,6 @@ carto@cartodb/carto#0.15.1-cdb3: optimist "~0.6.0" underscore "1.8.3" -"carto@github:cartodb/carto#0.15.1-cdb1": - version "0.15.1-cdb1" - resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" - dependencies: - mapnik-reference "~6.0.2" - optimist "~0.6.0" - underscore "~1.6.0" - cartocolor@4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/cartocolor/-/cartocolor-4.0.0.tgz#841a3222d8b5b22718d9d545b1e5b972cb26eb36" @@ -1572,7 +1572,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -"pg@github:cartodb/node-postgres#6.1.6-cdb1": +pg@cartodb/node-postgres#6.1.6-cdb1: version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: From b18bf967fdcee108d6ea8bda8eaeeb3a1def81a4 Mon Sep 17 00:00:00 2001 From: David M Date: Mon, 9 Oct 2017 18:27:57 +0200 Subject: [PATCH 28/35] Workaround lack of template creation clean-up --- test/acceptance/buffer-size-format.js | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index b888e094..2366d035 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -135,15 +135,10 @@ describe('buffer size per format', function () { const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { var testFn = (usePostGIS) => { - before(function () { - serverOptions.renderer.mvt.usePostGIS = usePostGIS; - }); - after(function () { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - }); - - it(test.desc, function (done) { + it(test.desc, function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; this.testClient = new TestClient(test.mapConfig, 1234); + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; var coords = test.coords; var options = { format: test.format, @@ -449,14 +444,11 @@ describe('buffer size per format for named maps w/o placeholders', function () { const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { var testFn = (usePostGIS) => { - before(function () { + it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) { serverOptions.renderer.mvt.usePostGIS = usePostGIS; - }); - after(function () { - serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; - }); - it(test.desc, function (done) { + test.template.name += '_1'; this.testClient = new TestClient(test.template, 1234); + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; var coords = test.coords; var options = { format: test.format, From 8cc4fe5b563f1c1bbe601c4ab8632aa14da5fb13 Mon Sep 17 00:00:00 2001 From: David Manzanares Date: Wed, 11 Oct 2017 11:34:27 +0200 Subject: [PATCH 29/35] Target Windshaft ~4.0.0, yarn.lock updated --- package.json | 2 +- yarn.lock | 106 ++++++++++++++++++++++++++++++++++++++------------- 2 files changed, 80 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index b65a64d1..26f3ed73 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.1", "underscore": "~1.6.0", - "windshaft": "cartodb/windshaft", + "windshaft": "~4.0.0", "yargs": "~5.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index cf95c05b..6d36931f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -10,7 +10,11 @@ abaculus@cartodb/abaculus#2.0.3-cdb1: mapnik "~3.5.0" sphericalmercator "1.0.x" -abbrev@1, abbrev@1.0.x: +abbrev@1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" + +abbrev@1.0.x: version "1.0.9" resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135" @@ -222,7 +226,7 @@ camshaft@0.59.2: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -240,15 +244,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -516,7 +520,11 @@ domutils@1.5: dom-serializer "0" domelementtype "1" -dot@^1.0.3, dot@~1.0.2: +dot@^1.0.3: + version "1.1.2" + resolved "https://registry.yarnpkg.com/dot/-/dot-1.1.2.tgz#c7377019fc4e550798928b2b9afeb66abfa1f2f9" + +dot@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/dot/-/dot-1.0.3.tgz#f8750bfb6b03c7664eb0e6cb1eb4c66419af9427" @@ -602,8 +610,8 @@ exit@0.1.2, exit@0.1.x: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" express@~4.16.0: - version "4.16.1" - resolved "https://registry.yarnpkg.com/express/-/express-4.16.1.tgz#6b33b560183c9b253b7b62144df33a4654ac9ed0" + version "4.16.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" dependencies: accepts "~1.3.4" array-flatten "1.1.1" @@ -746,7 +754,7 @@ gdal@~0.9.2: nan "~2.6.2" node-pre-gyp "~0.6.36" -generic-pool@2.4.3, generic-pool@~2.4.0, generic-pool@~2.4.1: +generic-pool@2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.3.tgz#780c36f69dfad05a5a045dd37be7adca11a4f6ff" @@ -758,6 +766,10 @@ generic-pool@~2.2.0, generic-pool@~2.2.1: version "2.2.2" resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.2.2.tgz#7a89f491d575b42f9f069a0e8e2c6dbaa3c241be" +generic-pool@~2.4.0, generic-pool@~2.4.1: + version "2.4.6" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-2.4.6.tgz#f1b55e572167dba2fe75d5aa91ebb1e9f72642d7" + get-caller-file@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5" @@ -768,7 +780,7 @@ getpass@^0.1.1: dependencies: assert-plus "^1.0.0" -glob@7.1.1, glob@^7.0.5, glob@^7.1.1: +glob@7.1.1: version "7.1.1" resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8" dependencies: @@ -799,6 +811,17 @@ glob@^6.0.1: once "^1.3.0" path-is-absolute "^1.0.0" +glob@^7.0.5, glob@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" + dependencies: + fs.realpath "^1.0.0" + inflight "^1.0.4" + inherits "2" + minimatch "^3.0.4" + once "^1.3.0" + path-is-absolute "^1.0.0" + graceful-fs@^4.1.2: version "4.1.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" @@ -1300,13 +1323,13 @@ mime@~1.3.4: version "1.3.6" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" -"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@~3.0.2: +"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2, minimatch@^3.0.4, minimatch@~3.0.2: version "3.0.4" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" dependencies: brace-expansion "^1.1.7" -minimist@0.0.8, minimist@~0.0.1: +minimist@0.0.8: version "0.0.8" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d" @@ -1314,6 +1337,10 @@ minimist@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284" +minimist@~0.0.1: + version "0.0.10" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.10.tgz#de3f98543dbf96082be48ad1a0c7cda836301dcf" + minimist@~0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.2.0.tgz#4dffe525dae2b864c66c2e23c6271d7afdecefce" @@ -1340,7 +1367,11 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@~2.18.1: +moment@^2.10.6: + version "2.19.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.0.tgz#44f675ef6b944942762581b1c179fb679e599d67" + +moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1449,10 +1480,14 @@ oauth-sign@~0.8.1, oauth-sign@~0.8.2: version "0.8.2" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43" -object-assign@4.1.0, object-assign@^4.1.0: +object-assign@4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0" +object-assign@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" + object-keys@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-0.4.0.tgz#28a6aae7428dd2c3a92f3d95f21335dd204e0336" @@ -1572,7 +1607,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1740,7 +1775,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@1.1, readable-stream@~1.1.9: +readable-stream@1.1: version "1.1.13" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.13.tgz#f6eef764f514c89e2b9e23146a75ba106756d23e" dependencies: @@ -1770,6 +1805,15 @@ readable-stream@~1.0.2: isarray "0.0.1" string_decoder "~0.10.x" +readable-stream@~1.1.9: + version "1.1.14" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.1.14.tgz#7cf4c54ef648e3813084c636dd2079e166c081d9" + dependencies: + core-util-is "~1.0.0" + inherits "~2.0.1" + isarray "0.0.1" + string_decoder "~0.10.x" + redis-mpool@0.4.1, redis-mpool@~0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/redis-mpool/-/redis-mpool-0.4.1.tgz#d917c0a4ed57a1291a9c6eb35434e6c0b7046f80" @@ -1879,9 +1923,9 @@ safe-json-stringify@~1: version "1.0.4" resolved "https://registry.yarnpkg.com/safe-json-stringify/-/safe-json-stringify-1.0.4.tgz#81a098f447e4bbc3ff3312a243521bc060ef5911" -"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" +"semver@2 || 3 || 4 || 5", semver@^5.1.0, semver@^5.3.0: + version "5.4.1" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" semver@4.3.2: version "4.3.2" @@ -1895,6 +1939,10 @@ semver@~5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/semver/-/semver-5.0.3.tgz#77466de589cd5d3c95f138aa78bc569a3cb5d27a" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + send@0.16.1: version "0.16.1" resolved "https://registry.yarnpkg.com/send/-/send-0.16.1.tgz#a70e1ca21d1382c11d0d9f6231deb281080d7ab3" @@ -1992,10 +2040,14 @@ speedometer@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/speedometer/-/speedometer-0.1.4.tgz#9876dbd2a169d3115402d48e6ea6329c8816a50d" -sphericalmercator@1.0.4, sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: +sphericalmercator@1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.4.tgz#baad4e34187f06e87f2e92fc1280199fa1b01d4e" +sphericalmercator@1.0.x, sphericalmercator@~1.0.1, sphericalmercator@~1.0.4: + version "1.0.5" + resolved "https://registry.yarnpkg.com/sphericalmercator/-/sphericalmercator-1.0.5.tgz#ddc5a049e360e000d0fad9fc22c4071882584980" + split@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9" @@ -2093,7 +2145,7 @@ strip-json-comments@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" -supports-color@3.1.2, supports-color@^3.1.0, supports-color@^3.1.2: +supports-color@3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.1.2.tgz#72a262894d9d408b956ca05ff37b2ed8a6e2a2d5" dependencies: @@ -2103,7 +2155,7 @@ supports-color@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-2.0.0.tgz#535d045ce6b6363fa40117084629995e9df324c7" -supports-color@^3.2.3: +supports-color@^3.1.0, supports-color@^3.1.2, supports-color@^3.2.3: version "3.2.3" resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-3.2.3.tgz#65ac0504b3954171d8a64946b2ae3cbb8a5f54f6" dependencies: @@ -2141,7 +2193,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2149,7 +2201,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2310,9 +2362,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@cartodb/windshaft: +windshaft@~4.0.0: version "4.0.0" - resolved "https://codeload.github.com/cartodb/windshaft/tar.gz/5b21e24e66c8e24cb88640b606aaf5fefcd3f365" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.0.tgz#b28ffb3775db3a4f2578e79d73c9b854a1ae5854" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2 From 67ab12e8e779f7341f4a7eb4c10d2b0b944e364f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 11:08:46 +0200 Subject: [PATCH 30/35] Add environment variable to check whether to run test against postgis 2.4 or not --- docker-test.sh | 2 +- test/acceptance/mvt.js | 6 ++++-- test/acceptance/user-render-timeout-limit.js | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docker-test.sh b/docker-test.sh index 6bf5360e..8cc8c093 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -8,4 +8,4 @@ yarn createdb template_postgis && createuser publicuser psql -c "CREATE EXTENSION postgis" template_postgis -npm test \ No newline at end of file +POSTGIS_VERSION=2.4 npm test \ No newline at end of file diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index 883f050f..ded1772e 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -20,7 +20,9 @@ function createMapConfig(sql = TestClient.SQL.ONE_POINT) { } describe('mvt (mapnik)', mvt(false)); -describe('mvt (postgis)', mvt(true)); +if (process.env.POSTGIS_VERSION === '2.4') { + describe('mvt (postgis)', mvt(true)); +} function mvt(usePostGIS) { return function () { @@ -185,7 +187,7 @@ return function () { type: 'mapnik', options: { sql: 'select * from populated_places_simple_reduced', - cartocss: + cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', cartocss_version: '2.0.1', widgets: { diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index eaf98805..0bb45c2b 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -201,7 +201,10 @@ describe('user render timeout limit', function () { }); }); - describe('vector (PostGIS)', vector(true)); + if (process.env.POSTGIS_VERSION === '2.4') { + describe('vector (PostGIS)', vector(true)); + } + describe('vector (mapnik)', vector(false)); function vector(usePostGIS) { const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; From f8ff41be01cfd9bbf76efe91ff8a376937645b7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 12:15:28 +0200 Subject: [PATCH 31/35] Do not run test if postgis version is lower than or equal to 2.4 --- docker-test.sh | 4 ++-- test/acceptance/buffer-size-format.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-test.sh b/docker-test.sh index 8cc8c093..00cc63b3 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,4 +1,4 @@ -export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres export POSTGIS_VERSION=2.4 npm install -g yarn@0.27.5 yarn @@ -8,4 +8,4 @@ yarn createdb template_postgis && createuser publicuser psql -c "CREATE EXTENSION postgis" template_postgis -POSTGIS_VERSION=2.4 npm test \ No newline at end of file +npm test \ No newline at end of file diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index 2366d035..ba8955e7 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -152,7 +152,7 @@ describe('buffer size per format', function () { }); }); }; - if (test.format === 'mvt'){ + if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ testFn(true); } testFn(false); @@ -465,7 +465,7 @@ describe('buffer size per format for named maps w/o placeholders', function () { }); }); }; - if (test.format === 'mvt'){ + if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ testFn(true); } testFn(false); From a1c08f9bf74f2c80a61312cc3b95970d11da7058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 12:53:23 +0200 Subject: [PATCH 32/35] Revert docker-test config --- docker-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-test.sh b/docker-test.sh index 00cc63b3..6bf5360e 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -1,4 +1,4 @@ -export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres export POSTGIS_VERSION=2.4 +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres npm install -g yarn@0.27.5 yarn From 515146bf285d40dffeae54592f504be6a817ba8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 13:12:06 +0200 Subject: [PATCH 33/35] Configure travis to export environment variable (POSTGIS_VERSION) --- .travis.yml | 3 +++ docker-test.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b946378f..9ac20ae7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,5 +10,8 @@ before_install: script: - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing +after_success: + - export POSTGIS_VERSION=2.4 + language: generic diff --git a/docker-test.sh b/docker-test.sh index 6bf5360e..8cc8c093 100644 --- a/docker-test.sh +++ b/docker-test.sh @@ -8,4 +8,4 @@ yarn createdb template_postgis && createuser publicuser psql -c "CREATE EXTENSION postgis" template_postgis -npm test \ No newline at end of file +POSTGIS_VERSION=2.4 npm test \ No newline at end of file From 294a222669b39318a76f4cb0b1b2f65a56548eb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 13:20:30 +0200 Subject: [PATCH 34/35] Configure travis to export environment variable (POSTGIS_VERSION) --- .travis.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ac20ae7..d881854c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,10 +8,7 @@ before_install: - docker pull cartoimages/windshaft-carto-testing script: - - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing - -after_success: - - export POSTGIS_VERSION=2.4 + - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing -e POSTGIS_VERSION=2.4 language: generic From c2bf7b075ca859d9687f6000ddad1f73c056b313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 16 Oct 2017 13:24:46 +0200 Subject: [PATCH 35/35] Fix travis to export environment variable (POSTGIS_VERSION) --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d881854c..5bc0cf1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ before_install: - docker pull cartoimages/windshaft-carto-testing script: - - docker run -v `pwd`:/srv cartoimages/windshaft-carto-testing -e POSTGIS_VERSION=2.4 + - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing language: generic