diff --git a/.travis.yml b/.travis.yml index 8f3993a2..5bc0cf1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,27 +1,14 @@ +sudo: required dist: trusty -addons: - postgresql: "9.5" - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - postgresql-9.5-postgis-2.3 - - postgresql-plpython-9.5 - - pkg-config - - libcairo2-dev - - libjpeg8-dev - - libgif-dev - - libpango1.0-dev - - g++-4.9 + +services: + - docker before_install: - - createdb template_postgis - - createuser publicuser - - psql -c "CREATE EXTENSION postgis" template_postgis + - docker pull cartoimages/windshaft-carto-testing -env: - - NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9 +script: + - docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing + +language: generic -language: node_js -node_js: - - "6" diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 48d03592..91cf2a39 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -107,6 +107,20 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/production.js.example b/config/environments/production.js.example index ba7fec6f..7d7a00cb 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -101,6 +101,20 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 1792779c..3d1098fb 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -101,6 +101,20 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 374467c0..ea788a22 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -100,6 +100,20 @@ var config = { // Milliseconds since last access before renderer cache item expires cache_ttl: 60000, statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status + mvt: { + //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, + //PostGIS 2.4 is required for this to work + //If disabled it will use Mapnik MVT generation + usePostGIS: false, + dbPoolParams: { + // maximum number of resources to create at any given time + size: 16, + // max milliseconds a resource can go unused before it should be destroyed + idleTimeout: 3000, + // frequency to check for idle resources + reapInterval: 1000 + } + }, mapnik: { // The size of the pool of internal mapnik backend // This pool size is per mapnik renderer created in Windshaft's RendererFactory diff --git a/docker-test.sh b/docker-test.sh new file mode 100644 index 00000000..8cc8c093 --- /dev/null +++ b/docker-test.sh @@ -0,0 +1,11 @@ +export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres + +npm install -g yarn@0.27.5 +yarn + + /etc/init.d/postgresql start + +createdb template_postgis && createuser publicuser +psql -c "CREATE EXTENSION postgis" template_postgis + +POSTGIS_VERSION=2.4 npm test \ No newline at end of file diff --git a/lib/cartodb/controllers/layergroup.js b/lib/cartodb/controllers/layergroup.js index a814ee43..c5e4c53a 100644 --- a/lib/cartodb/controllers/layergroup.js +++ b/lib/cartodb/controllers/layergroup.js @@ -294,6 +294,10 @@ LayergroupController.prototype.tileOrLayer = function (req, res, next) { ); }; +function getStatusCode(tile, format){ + return tile.length===0 && format==='mvt'? 204:200; +} + // This function is meant for being called as the very last // step by all endpoints serving tiles or grids LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) { @@ -331,7 +335,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t global.statsClient.increment('windshaft.tiles.error'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.error'); } else { - this.sendResponse(req, res, tile, 200, headers); + this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers); global.statsClient.increment('windshaft.tiles.success'); global.statsClient.increment('windshaft.tiles.' + formatStat + '.success'); } diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index 5bb510fd..08ec1a66 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -158,7 +158,8 @@ module.exports = function(serverOptions) { grainstore: serverOptions.grainstore, mapnik: serverOptions.renderer.mapnik }, - http: serverOptions.renderer.http + http: serverOptions.renderer.http, + mvt: serverOptions.renderer.mvt }); // initialize render cache diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 6d8575a7..25e60c2f 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -81,6 +81,7 @@ module.exports = { statsInterval: rendererConfig.statsInterval }, renderer: { + mvt: rendererConfig.mvt, mapnik: _.defaults(rendererConfig.mapnik, { geojson: { dbPoolParams: { diff --git a/package.json b/package.json index 204bf63c..3a52e3d9 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "step-profiler": "~0.3.0", "turbo-carto": "0.20.1", "underscore": "~1.6.0", - "windshaft": "3.3.3", + "windshaft": "~4.0.0", "yargs": "~5.0.0" }, "devDependencies": { @@ -59,7 +59,13 @@ "scripts": { "lint": "jshint lib test", "preinstall": "make pre-install", - "test": "make test-all" + "test": "make test-all", + "update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn", + "docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)", + "docker-pull": "docker pull cartoimages/windshaft-testing", + "docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh", + "docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-testing bash", + "docker-publish": "docker push cartoimages/windshaft-carto-testing" }, "engines": { "node": ">=6.9", diff --git a/run_tests.sh b/run_tests.sh index 92e8621b..d1c63614 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS} else echo "Running tests" - mocha -u tdd -t 5000 ${TESTS} + ./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS} fi ret=$? diff --git a/test/acceptance/analysis/analysis-layers-geojson.js b/test/acceptance/analysis/analysis-layers-geojson.js deleted file mode 100644 index 59a0900b..00000000 --- a/test/acceptance/analysis/analysis-layers-geojson.js +++ /dev/null @@ -1,81 +0,0 @@ -require('../../support/test_helper'); - -var assert = require('../../support/assert'); -var TestClient = require('../../support/test-client'); - -describe('analysis-layers-dataviews-geojson', function() { - - function createMapConfig(layers, dataviews, analysis) { - return { - version: '1.5.0', - layers: layers, - dataviews: dataviews || {}, - analyses: analysis || [] - }; - } - - var CARTOCSS = [ - "#points {", - " marker-fill-opacity: 1.0;", - " marker-line-color: #FFF;", - " marker-line-width: 0.5;", - " marker-line-opacity: 1.0;", - " marker-placement: point;", - " marker-type: ellipse;", - " marker-width: 8;", - " marker-fill: red;", - " marker-allow-overlap: true;", - "}" - ].join('\n'); - - var mapConfig = createMapConfig( - [ - { - "type": "cartodb", - "options": { - "source": { - "id": "2570e105-7b37-40d2-bdf4-1af889598745" - }, - "cartocss": CARTOCSS, - "cartocss_version": "2.3.0" - } - } - ], - { - pop_max_histogram: { - source: { - id: '2570e105-7b37-40d2-bdf4-1af889598745' - }, - type: 'histogram', - options: { - column: 'pop_max' - } - } - }, - [ - { - "id": "2570e105-7b37-40d2-bdf4-1af889598745", - "type": "source", - "params": { - "query": "select * from populated_places_simple_reduced" - } - } - ] - ); - - it('should get pop_max column from dataview', function(done) { - var testClient = new TestClient(mapConfig, 1234); - - testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(Array.isArray(geojson.features)); - assert.ok(geojson.features.length > 0); - var feature = geojson.features[0]; - assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); - - testClient.drain(done); - }); - }); - -}); diff --git a/test/acceptance/buffer-size-format.js b/test/acceptance/buffer-size-format.js index c32f62d1..ba8955e7 100644 --- a/test/acceptance/buffer-size-format.js +++ b/test/acceptance/buffer-size-format.js @@ -3,6 +3,7 @@ require('../support/test_helper'); var fs = require('fs'); var assert = require('../support/assert'); var TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); var mapnik = require('windshaft').mapnik; var IMAGE_TOLERANCE_PER_MIL = 5; @@ -124,24 +125,37 @@ describe('buffer size per format', function () { } ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { - it(test.desc, function (done) { - var testClient = new TestClient(test.mapConfig, 1234); - var coords = test.coords; - var options = { - format: test.format, - layers: test.layers - }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - // tile.save(test.fixturePath); - test.assert(tile, function (err) { + var testFn = (usePostGIS) => { + it(test.desc, function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + this.testClient = new TestClient(test.mapConfig, 1234); + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + var coords = test.coords; + var options = { + format: test.format, + layers: test.layers + }; + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); - testClient.drain(done); + // To generate images use: + // tile.save(test.fixturePath); + test.assert(tile, done); }); }); - }); + }; + if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ + testFn(true); + } + testFn(false); }); }); @@ -260,23 +274,27 @@ describe('buffer size per format for named maps', function () { } ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + testCases.forEach(function (test) { it(test.desc, function (done) { - var testClient = new TestClient(test.template, 1234); + this.testClient = new TestClient(test.template, 1234); var coords = test.coords; var options = { format: test.format, placeholders: test.placeholders, layers: test.layers }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { assert.ifError(err); // To generate images use: //tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png'); - test.assert(tile, function (err) { - assert.ifError(err); - testClient.drain(done); - }); + test.assert(tile, done); }); }); }); @@ -416,26 +434,40 @@ describe('buffer size per format for named maps w/o placeholders', function () { ]; + afterEach(function(done) { + if (this.testClient) { + return this.testClient.drain(done); + } + return done(); + }); + + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; testCases.forEach(function (test) { - it(test.desc, function (done) { - var testClient = new TestClient(test.template, 1234); - var coords = test.coords; - var options = { - format: test.format, - placeholders: test.placeholders, - layers: test.layers - }; - testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { - assert.ifError(err); - // To generate images use: - //tile.save(test.fixturePath); - // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); - // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); - test.assert(tile, function (err) { - assert.ifError(err); - testClient.drain(done); + var testFn = (usePostGIS) => { + it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + test.template.name += '_1'; + this.testClient = new TestClient(test.template, 1234); + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + var coords = test.coords; + var options = { + format: test.format, + placeholders: test.placeholders, + layers: test.layers + }; + this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) { + assert.ifError(err); + // To generate images use: + //tile.save(test.fixturePath); + // require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile)); + // require('fs').writeFileSync(test.fixturePath, tile.getDataSync()); + test.assert(tile, done); + }); }); - }); - }); + }; + if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){ + testFn(true); + } + testFn(false); }); }); diff --git a/test/acceptance/geojson-renderer.js b/test/acceptance/geojson-renderer.js deleted file mode 100644 index 089e09b5..00000000 --- a/test/acceptance/geojson-renderer.js +++ /dev/null @@ -1,343 +0,0 @@ -require('../support/test_helper'); - -var assert = require('../support/assert'); -var TestClient = require('../support/test-client'); - - -describe('use only needed columns', function() { - - function getFeatureByCartodbId(features, cartodbId) { - for (var i = 0, len = features.length; i < len; i++) { - if (features[i].properties.cartodb_id === cartodbId) { - return features[i]; - } - } - return {}; - } - - var options = { format: 'geojson', layer: 0 }; - - afterEach(function(done) { - if (this.testClient) { - this.testClient.drain(done); - } else { - done(); - } - }); - - it('with aggregation widget, interactivity and cartocss columns', function(done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', - cartocss_version: '2.0.1', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_min" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373, - pop_min: 57586 - }); - done(); - }); - }); - - it('should not duplicate columns', function(done) { - var widgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced', - cartocss: ['#layer0 {', - 'marker-fill: red;', - 'marker-width: 10;', - '[name="Madrid"] { marker-fill: green; } ', - '[pop_max>100000] { marker-fill: black; } ', - '}'].join('\n'), - cartocss_version: '2.3.0', - widgets: { - adm0name: { - type: 'aggregation', - options: { - column: 'adm0name', - aggregation: 'sum', - aggregationColumn: 'pop_max' - } - } - }, - interactivity: "cartodb_id,pop_max" - } - }] - }; - - this.testClient = new TestClient(widgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - name: 'Mardin', - adm0name: 'Turkey', - pop_max: 71373 - }); - done(); - }); - }); - - it('with formula widget, no interactivity and no cartocss columns', function(done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id', - widgets: { - pop_max_f: { - type: 'formula', - options: { - column: 'pop_max', - operation: 'count' - } - } - } - } - }] - }; - - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max: 71373 - }); - done(); - }); - }); - it('with cartocss with multiple expressions', function(done) { - var formulaWidgetMapConfig = { - version: '1.5.0', - layers: [{ - type: 'mapnik', - options: { - sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', - cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + - '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + - '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', - cartocss_version: '2.0.1', - interactivity: 'cartodb_id' - } - }] - }; - - this.testClient = new TestClient(formulaWidgetMapConfig); - this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) { - assert.ok(!err, err); - assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { - cartodb_id: 1109, - pop_max:71373, - name:"Mardin", - adm0name:"Turkey" - }); - done(); - }); - }); - - it('should work with mapnik substitution tokens', function(done) { - var cartocss = [ - "#layer {", - " line-width: 2;", - " line-color: #3B3B58;", - " line-opacity: 1;", - " polygon-opacity: 0.7;", - " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", - "}" - ].join('\n'); - - var sql = [ - 'WITH hgrid AS (', - ' SELECT CDB_HexagonGrid(', - ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', - ' greatest(!pixel_width!,!pixel_height!) * 100', - ' ) as cell', - ')', - 'SELECT', - ' hgrid.cell as the_geom_webmercator,', - ' count(1) as points_count,', - ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', - ' 1 as cartodb_id', - 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', - 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', - 'GROUP BY hgrid.cell' - ].join('\n'); - - var mapConfig = { - "version": "1.4.0", - "layers": [ - { - "type": 'mapnik', - "options": { - "cartocss_version": '2.3.0', - "sql": sql, - "cartocss": cartocss - } - } - ] - }; - - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(geojson); - assert.equal(geojson.features.length, 5); - - done(); - }); - }); - - it('should skip empty and null columns for geojson tiles', function(done) { - - var mapConfig = { - "analyses": [ - { - "id": "a0", - "params": { - "query": "SELECT * FROM test_table" - }, - "type": "source" - } - ], - "dataviews": { - "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "74f590f8-625c-4e95-922f-34ad3e9919c0": { - "options": { - "aggregation": "sum", - "aggregationColumn": "cartodb_id", - "column": "name" - }, - "source": { - "id": "a0" - }, - "type": "aggregation" - }, - "98a75757-3006-400a-b028-fb613a6c0b69": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "sum" - }, - "source": { - "id": "a0" - }, - "type": "formula" - }, - "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { - "options": { - "aggregationColumn": null, - "bins": "12", - "column": "cartodb_id" - }, - "source": { - "id": "a0" - }, - "type": "histogram" - }, - "ebc0653f-3581-469c-8b31-c969e440a865": { - "options": { - "aggregationColumn": null, - "column": "cartodb_id", - "operation": "avg" - }, - "source": { - "id": "a0" - }, - "type": "formula" - } - }, - "layers": [ - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" - }, - "type": "http" - }, - { - "options": { - "attributes": { - "columns": [ - "name", - "address" - ], - "id": "cartodb_id" - }, - "cartocss": "#layer { marker-width: 10; marker-fill: red; }", - "cartocss_version": "2.3.0", - "interactivity": "cartodb_id", - "layer_name": "wadus", - "source": { - "id": "a0" - } - }, - "type": "cartodb" - }, - { - "options": { - "subdomains": "abcd", - "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" - }, - "type": "http" - } - ] - }; - - this.testClient = new TestClient(mapConfig); - this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) { - assert.ok(!err, err); - - assert.ok(geojson); - assert.equal(geojson.features.length, 5); - - assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']); - - done(); - }); - }); - -}); diff --git a/test/acceptance/mvt.js b/test/acceptance/mvt.js index b5a59cd8..ded1772e 100644 --- a/test/acceptance/mvt.js +++ b/test/acceptance/mvt.js @@ -1,9 +1,10 @@ require('../support/test_helper'); -const assert = require('../support/assert'); -const TestClient = require('../support/test-client'); +var assert = require('../support/assert'); +var TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); -function createMapConfig (sql = TestClient.SQL.ONE_POINT) { +function createMapConfig(sql = TestClient.SQL.ONE_POINT) { return { version: '1.6.0', layers: [{ @@ -18,7 +19,100 @@ function createMapConfig (sql = TestClient.SQL.ONE_POINT) { }; } -describe('mvt', function () { +describe('mvt (mapnik)', mvt(false)); +if (process.env.POSTGIS_VERSION === '2.4') { + describe('mvt (postgis)', mvt(true)); +} + +function mvt(usePostGIS) { +return function () { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + before(function () { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + }); + after(function (){ + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + }); + + describe('analysis-layers-dataviews-mvt', function () { + + function createMapConfig(layers, dataviews, analysis) { + return { + version: '1.5.0', + layers: layers, + dataviews: dataviews || {}, + analyses: analysis || [] + }; + } + + var CARTOCSS = [ + "#points {", + " marker-fill-opacity: 1.0;", + " marker-line-color: #FFF;", + " marker-line-width: 0.5;", + " marker-line-opacity: 1.0;", + " marker-placement: point;", + " marker-type: ellipse;", + " marker-width: 8;", + " marker-fill: red;", + " marker-allow-overlap: true;", + "}" + ].join('\n'); + + var mapConfig = createMapConfig( + [ + { + "type": "cartodb", + "options": { + "source": { + "id": "2570e105-7b37-40d2-bdf4-1af889598745" + }, + "cartocss": CARTOCSS, + "cartocss_version": "2.3.0" + } + } + ], + { + pop_max_histogram: { + source: { + id: '2570e105-7b37-40d2-bdf4-1af889598745' + }, + type: 'histogram', + options: { + column: 'pop_max' + } + } + }, + [ + { + "id": "2570e105-7b37-40d2-bdf4-1af889598745", + "type": "source", + "params": { + "query": "select * from populated_places_simple_reduced" + } + } + ] + ); + + it('should get pop_max column from dataview', function (done) { + var testClient = new TestClient(mapConfig); + + testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(Array.isArray(geojsonTile.features)); + assert.ok(geojsonTile.features.length > 0); + var feature = geojsonTile.features[0]; + assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property'); + + testClient.drain(done); + }); + }); + + }); + + const testCases = [ { desc: 'should get empty mvt with code 204 (no content)', @@ -48,16 +142,366 @@ describe('mvt', function () { testCases.forEach(function (test) { it(test.desc, done => { - const testClient = new TestClient(test.mapConfig, 1234); + var testClient = new TestClient(test.mapConfig); const { z, x, y } = test.coords; const { format, response } = test; - testClient.getTile(z, x, y, { format, response }, (err, res) => { + testClient.getTile(z, x, y, { format, response }, err => { assert.ifError(err); - - assert.equal(res.statusCode, test.response.status); testClient.drain(done); }); }); }); -}); + + + if (usePostGIS){ + describe('use only needed columns', onlyNeededColumns); + }else{ + describe.skip('use only needed columns', onlyNeededColumns); + } + + function onlyNeededColumns() { + function getFeatureByCartodbId(features, cartodbId) { + for (var i = 0, len = features.length; i < len; i++) { + if (features[i].properties.cartodb_id === cartodbId) { + return features[i]; + } + } + return {}; + } + + var options = { format: 'mvt', layer: 0 }; + + afterEach(function (done) { + if (this.testClient) { + this.testClient.drain(done); + } else { + done(); + } + }); + + it('with aggregation widget, interactivity and cartocss columns', function (done) { + var widgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: + '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }', + cartocss_version: '2.0.1', + widgets: { + adm0name: { + type: 'aggregation', + options: { + column: 'adm0name', + aggregation: 'sum', + aggregationColumn: 'pop_max' + } + } + }, + interactivity: "cartodb_id,pop_min" + } + }] + }; + + this.testClient = new TestClient(widgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + name: 'Mardin', + adm0name: 'Turkey', + pop_max: 71373, + pop_min: 57586 + }); + done(); + }); + }); + + it('should not duplicate columns', function (done) { + var widgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced', + cartocss: ['#layer0 {', + 'marker-fill: red;', + 'marker-width: 10;', + '[name="Madrid"] { marker-fill: green; } ', + '[pop_max>100000] { marker-fill: black; } ', + '}'].join('\n'), + cartocss_version: '2.3.0', + widgets: { + adm0name: { + type: 'aggregation', + options: { + column: 'adm0name', + aggregation: 'sum', + aggregationColumn: 'pop_max' + } + } + }, + interactivity: "cartodb_id,pop_max" + } + }] + }; + + this.testClient = new TestClient(widgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + name: 'Mardin', + adm0name: 'Turkey', + pop_max: 71373 + }); + done(); + }); + }); + + it('with formula widget, no interactivity and no cartocss columns', function (done) { + var formulaWidgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id', + widgets: { + pop_max_f: { + type: 'formula', + options: { + column: 'pop_max', + operation: 'count' + } + } + } + } + }] + }; + this.testClient = new TestClient(formulaWidgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + pop_max: 71373 + }); + done(); + }); + }); + it('with cartocss with multiple expressions', function (done) { + var formulaWidgetMapConfig = { + version: '1.5.0', + layers: [{ + type: 'mapnik', + options: { + sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000', + cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' + + '#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + + '#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' + + '#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }', + cartocss_version: '2.0.1', + interactivity: 'cartodb_id' + } + }] + }; + + this.testClient = new TestClient(formulaWidgetMapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, { + cartodb_id: 1109, + pop_max: 71373, + name: "Mardin", + adm0name: "Turkey" + }); + done(); + }); + }); + + var skipOnPostGIS = usePostGIS ? it.skip: it; + skipOnPostGIS('should work with mapnik substitution tokens', function (done) { + var cartocss = [ + "#layer {", + " line-width: 2;", + " line-color: #3B3B58;", + " line-opacity: 1;", + " polygon-opacity: 0.7;", + " polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))", + "}" + ].join('\n'); + + var sql = [ + 'WITH hgrid AS (', + ' SELECT CDB_HexagonGrid(', + ' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),', + ' greatest(!pixel_width!,!pixel_height!) * 100', + ' ) as cell', + ')', + 'SELECT', + ' hgrid.cell as the_geom_webmercator,', + ' count(1) as points_count,', + ' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,', + ' 1 as cartodb_id', + 'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i', + 'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)', + 'GROUP BY hgrid.cell' + ].join('\n'); + + var mapConfig = { + "version": "1.4.0", + "layers": [ + { + "type": 'mapnik', + "options": { + "cartocss_version": '2.3.0', + "sql": sql, + "cartocss": cartocss + } + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(geojsonTile); + assert.equal(geojsonTile.features.length, 5); + + done(); + }); + }); + + it('should skip empty and null columns for geojson tiles', function (done) { + + var mapConfig = { + "analyses": [ + { + "id": "a0", + "params": { + "query": "SELECT * FROM test_table" + }, + "type": "source" + } + ], + "dataviews": { + "4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "avg" + }, + "source": { + "id": "a0" + }, + "type": "formula" + }, + "74f590f8-625c-4e95-922f-34ad3e9919c0": { + "options": { + "aggregation": "sum", + "aggregationColumn": "cartodb_id", + "column": "name" + }, + "source": { + "id": "a0" + }, + "type": "aggregation" + }, + "98a75757-3006-400a-b028-fb613a6c0b69": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "sum" + }, + "source": { + "id": "a0" + }, + "type": "formula" + }, + "ebbc97b2-87d2-4895-9e1f-2f012df3679d": { + "options": { + "aggregationColumn": null, + "bins": "12", + "column": "cartodb_id" + }, + "source": { + "id": "a0" + }, + "type": "histogram" + }, + "ebc0653f-3581-469c-8b31-c969e440a865": { + "options": { + "aggregationColumn": null, + "column": "cartodb_id", + "operation": "avg" + }, + "source": { + "id": "a0" + }, + "type": "formula" + } + }, + "layers": [ + { + "options": { + "subdomains": "abcd", + "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png" + }, + "type": "http" + }, + { + "options": { + "attributes": { + "columns": [ + "name", + "address" + ], + "id": "cartodb_id" + }, + "cartocss": "#layer { marker-width: 10; marker-fill: red; }", + "cartocss_version": "2.3.0", + "interactivity": "cartodb_id", + "layer_name": "wadus", + "source": { + "id": "a0" + } + }, + "type": "cartodb" + }, + { + "options": { + "subdomains": "abcd", + "urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png" + }, + "type": "http" + } + ] + }; + + this.testClient = new TestClient(mapConfig); + this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) { + var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0)); + assert.ok(!err, err); + + assert.ok(geojsonTile); + assert.equal(geojsonTile.features.length, 5); + + assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']); + + done(); + }); + }); + + } +}; +} \ No newline at end of file diff --git a/test/acceptance/user-render-timeout-limit.js b/test/acceptance/user-render-timeout-limit.js index 45e41f89..0bb45c2b 100644 --- a/test/acceptance/user-render-timeout-limit.js +++ b/test/acceptance/user-render-timeout-limit.js @@ -2,6 +2,7 @@ require('../support/test_helper'); const assert = require('../support/assert'); const TestClient = require('../support/test-client'); +var serverOptions = require('../../lib/cartodb/server_options'); const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`; @@ -200,49 +201,59 @@ describe('user render timeout limit', function () { }); }); - describe('vector', function () { - beforeEach(function (done) { - const mapconfig = createMapConfig(); - this.testClient = new TestClient(mapconfig, 1234); - this.testClient.setUserRenderTimeoutLimit('localhost', 50, done); - }); + if (process.env.POSTGIS_VERSION === '2.4') { + describe('vector (PostGIS)', vector(true)); + } - afterEach(function (done) { - this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => { - if (err) { - return done(err); - } - this.testClient.drain(done); + describe('vector (mapnik)', vector(false)); + function vector(usePostGIS) { + const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS; + return function () { + beforeEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = usePostGIS; + const mapconfig = createMapConfig(); + this.testClient = new TestClient(mapconfig, 1234); + this.testClient.setUserDatabaseTimeoutLimit(50, done); }); - }); - it('layergroup creation works but vector tile request fails due to render timeout', function (done) { - const params = { - format: 'mvt', - response: { - status: 429, - headers: { - 'Content-Type': 'application/json; charset=utf-8' + afterEach(function (done) { + serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS; + this.testClient.setUserDatabaseTimeoutLimit(0, (err) => { + if (err) { + return done(err); } - } - }; - - this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { - assert.ifError(err); - - assert.deepEqual(tile, { - errors: ['You are over platform\'s limits. Please contact us to know more details'], - errors_with_context: [{ - type: 'limit', - subtype: 'render', - message: 'You are over platform\'s limits. Please contact us to know more details' - }] + this.testClient.drain(done); }); - - done(); }); - }); - }); + + it('layergroup creation works but vector tile request fails due to render timeout', function (done) { + const params = { + format: 'mvt', + response: { + status: 429, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 0, params, (err, res, tile) => { + assert.ifError(err); + + assert.deepEqual(tile, { + errors: ['You are over platform\'s limits. Please contact us to know more details'], + errors_with_context: [{ + type: 'limit', + subtype: 'datasource', + message: 'You are over platform\'s limits. Please contact us to know more details' + }] + }); + + done(); + }); + }); + }; + } describe('interativity', function () { beforeEach(function (done) { diff --git a/test/support/test-client.js b/test/support/test-client.js index e027537e..cd7b4f60 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -20,7 +20,6 @@ const MAPNIK_SUPPORTED_FORMATS = { 'png': true, 'png32': true, 'grid.json': true, - 'geojson': true, 'mvt': true }; diff --git a/yarn.lock b/yarn.lock index ccb6cd6d..db232491 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,7 +2,7 @@ # yarn lockfile v1 -abaculus@cartodb/abaculus#2.0.3-cdb1: +"abaculus@github:cartodb/abaculus#2.0.3-cdb1": version "2.0.3-cdb1" resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf" dependencies: @@ -210,7 +210,7 @@ camshaft@0.59.2: dot "^1.0.3" request "^2.69.0" -canvas@cartodb/node-canvas#1.6.2-cdb2: +"canvas@github:cartodb/node-canvas#1.6.2-cdb2": version "1.6.2-cdb2" resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766" dependencies: @@ -228,15 +228,15 @@ carto@0.16.3: semver "^5.1.0" yargs "^4.2.0" -carto@CartoDB/carto#0.15.1-cdb1: +"carto@github:cartodb/carto#0.15.1-cdb1": version "0.15.1-cdb1" - resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" + resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398" dependencies: mapnik-reference "~6.0.2" optimist "~0.6.0" underscore "~1.6.0" -carto@cartodb/carto#0.15.1-cdb3: +"carto@github:cartodb/carto#0.15.1-cdb3": version "0.15.1-cdb3" resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7" dependencies: @@ -584,8 +584,8 @@ exit@0.1.2, exit@0.1.x: resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c" express@~4.16.0: - version "4.16.1" - resolved "https://registry.yarnpkg.com/express/-/express-4.16.1.tgz#6b33b560183c9b253b7b62144df33a4654ac9ed0" + version "4.16.2" + resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c" dependencies: accepts "~1.3.4" array-flatten "1.1.1" @@ -1286,7 +1286,11 @@ mocha@~3.4.1: mkdirp "0.5.1" supports-color "3.1.2" -moment@^2.10.6, moment@~2.18.1: +moment@^2.10.6: + version "2.19.0" + resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.0.tgz#44f675ef6b944942762581b1c179fb679e599d67" + +moment@~2.18.1: version "2.18.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f" @@ -1522,7 +1526,7 @@ pg-types@1.*: postgres-date "~1.0.0" postgres-interval "^1.1.0" -pg@cartodb/node-postgres#6.1.6-cdb1: +"pg@github:cartodb/node-postgres#6.1.6-cdb1": version "6.1.6" resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44" dependencies: @@ -1580,8 +1584,8 @@ postcss@5.0.19: supports-color "^3.1.2" postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8: - version "5.2.17" - resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b" + version "5.2.18" + resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.18.tgz#badfa1497d46244f6390f58b319830d9107853c5" dependencies: chalk "^1.1.3" js-base64 "^2.1.9" @@ -2089,7 +2093,7 @@ through@2: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" -tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: +"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4": version "2.3.1-cdb4" resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6" dependencies: @@ -2097,7 +2101,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4: mapnik-pool "~0.1.3" sphericalmercator "1.0.x" -tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3: +"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3": version "0.6.18-cdb3" resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01" dependencies: @@ -2258,9 +2262,9 @@ window-size@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075" -windshaft@3.3.3: - version "3.3.3" - resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.3.tgz#0582e6a0d9cf91c533134787ace64a3337200e33" +windshaft@~4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.0.tgz#b28ffb3775db3a4f2578e79d73c9b854a1ae5854" dependencies: abaculus cartodb/abaculus#2.0.3-cdb1 canvas cartodb/node-canvas#1.6.2-cdb2