diff --git a/NEWS.md b/NEWS.md index fb2d8ea2..2aa7f849 100644 --- a/NEWS.md +++ b/NEWS.md @@ -39,6 +39,7 @@ Bug Fixes: - Static maps fails for unsupported formats - Handling errors extracting the column type on dataviews - Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens +- Static maps filters correctly if `layer` option is passed in the url. ## 6.1.0 Released 2018-04-16 diff --git a/config/environments/development.js.example b/config/environments/development.js.example index d6345b82..87c98032 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -61,13 +61,13 @@ var config = { // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', @@ -87,12 +87,12 @@ var config = { // idle socket timeout, in milliseconds ,socket_timeout: 600000 ,enable_cors: true - ,cache_enabled: false + ,cache_enabled: true ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - ,log_filename: undefined + ,log_filename: 'logs/node-windshaft.log' // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'development_cartodb_user_<%= user_id %>' @@ -118,7 +118,7 @@ var config = { ,statsd: { host: 'localhost', port: 8125, - prefix: 'dev.', + prefix: 'dev.', // could be hostname, better not containing dots cacheDns: true // support all allowed node-statsd options } @@ -204,7 +204,8 @@ var config = { persist_connection: false, simplify_geometries: true, use_overviews: true, // use overviews to retrieve raster - max_size: 500 + max_size: 500, + twkb_encoding: true }, limits: { @@ -267,7 +268,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: '/tmp/analysis.log' + filename: 'logs/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -337,6 +338,12 @@ var config = { // X-Tiler-Profile header containing elapsed timing for various // steps taken for producing the response. ,useProfiler:true + ,serverMetadata: { + cdn_url: { + http: undefined, + https: undefined + } + } // Settings for the health check available at /health ,health: { enabled: false, diff --git a/config/environments/production.js.example b/config/environments/production.js.example index d601dbc5..8e071acf 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -61,13 +61,13 @@ var config = { // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map', @@ -130,15 +130,7 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + usePostGIS: false }, mapnik: { // The size of the pool of internal mapnik backend @@ -212,7 +204,8 @@ var config = { persist_connection: false, simplify_geometries: true, use_overviews: true, // use overviews to retrieve raster - max_size: 500 + max_size: 500, + twkb_encoding: true }, limits: { @@ -275,7 +268,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'logs/analysis.log' + filename: 'logs/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 76260b8f..60301a94 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -61,16 +61,16 @@ var config = { // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { - http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map', https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' } @@ -88,7 +88,7 @@ var config = { ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: true - ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type] (:res[X-Tiler-Errors])' + ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal @@ -118,7 +118,7 @@ var config = { ,statsd: { host: 'localhost', port: 8125, - prefix: 'stage.:host.', + prefix: 'stage.:host.', // could be hostname, better not containing dots cacheDns: true // support all allowed node-statsd options } @@ -204,7 +204,8 @@ var config = { persist_connection: false, simplify_geometries: true, use_overviews: true, // use overviews to retrieve raster - max_size: 500 + max_size: 500, + twkb_encoding: true }, limits: { @@ -267,7 +268,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'logs/analysis.log' + filename: 'logs/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -345,7 +346,7 @@ var config = { } // Settings for the health check available at /health ,health: { - enabled: false, + enabled: true, username: 'localhost', z: 0, x: 0, diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 0dd475fe..d106995e 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -61,16 +61,17 @@ var config = { // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. // - // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // This URLs depend on how `routes` and `user_from_host` are configured: the application can be // configured to accept request with the {user} in the header host or in the request path. // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. // // This template allows to make the endpoints generation more flexible, the template exposes the following params: // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. - // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { - http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map' + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map' } // Maximum number of connections for one process @@ -87,11 +88,11 @@ var config = { ,socket_timeout: 600000 ,enable_cors: true ,cache_enabled: false - ,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' + ,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])' // If log_filename is given logs will be written // there, in append mode. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - //,log_filename: 'logs/node-windshaft.log' + ,log_filename: '/tmp/node-windshaft.log' // Templated database username for authorized user // Supported labels: 'user_id' (read from redis) ,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>' @@ -99,8 +100,6 @@ var config = { // Supported labels: 'user_id', 'user_password' (both read from redis) ,postgres_auth_pass: 'test_windshaft_cartodb_user_<%= user_id %>_pass' ,postgres: { - // Parameters to pass to datasource plugin of mapnik - // See http://github.com/mapnik/mapnik/wiki/PostGIS user: "test_windshaft_publicuser", password: "public", host: '127.0.0.1', @@ -114,12 +113,12 @@ var config = { reapInterval: 1000 } } - ,mapnik_version: '' + ,mapnik_version: undefined ,mapnik_tile_format: 'png8:m=h' ,statsd: { host: 'localhost', port: 8125, - prefix: 'test.:host.', + prefix: 'test.:host.', // could be hostname, better not containing dots cacheDns: true // support all allowed node-statsd options } @@ -131,15 +130,7 @@ var config = { //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik, //PostGIS 2.4 is required for this to work //If disabled it will use Mapnik MVT generation - usePostGIS: false, - dbPoolParams: { - // maximum number of resources to create at any given time - size: 16, - // max milliseconds a resource can go unused before it should be destroyed - idleTimeout: 3000, - // frequency to check for idle resources - reapInterval: 1000 - } + usePostGIS: false }, mapnik: { // The size of the pool of internal mapnik backend @@ -213,7 +204,8 @@ var config = { persist_connection: false, simplify_geometries: true, use_overviews: true, // use overviews to retrieve raster - max_size: 500 + max_size: 500, + twkb_encoding: false }, limits: { @@ -231,10 +223,15 @@ var config = { // If enabled Mapnik will reuse the features retrieved from the database // instead of requesting them once per style inside a layer - 'cache-features': true, + 'cache-features': false, // Require metrics to the renderer - metrics: false + metrics: false, + + // Options for markers attributes, ellipses and images caches + markers_symbolizer_caches: { + disabled: false + } }, http: { timeout: 2000, // the timeout in ms for a http tile request @@ -273,7 +270,7 @@ var config = { // If filename is given logs comming from analysis client will be written // there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default). // Log file will be re-opened on receiving the HUP signal - filename: 'node-windshaft.log' + filename: '/tmp/node-windshaft-analysis.log' }, // Define max execution time in ms for analyses or tags // If analysis or tag are not found in redis this values will be used as default. @@ -343,6 +340,12 @@ var config = { // X-Tiler-Profile header containing elapsed timing for various // steps taken for producing the response. ,useProfiler:true + ,serverMetadata: { + cdn_url: { + http: undefined, + https: undefined + } + } // Settings for the health check available at /health ,health: { enabled: false, diff --git a/docs/static_maps_api.md b/docs/static_maps_api.md index 34e4e907..554f892d 100644 --- a/docs/static_maps_api.md +++ b/docs/static_maps_api.md @@ -11,7 +11,7 @@ Begin by instantiating either a Named or Anonymous Map using the `layergroupid t #### Definition ```bash -GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format} +GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format}{{?}extra_options} ``` #### Params @@ -58,6 +58,9 @@ Note: you can see this endpoint as GET /api/v1/map/static/bbox/{token}/{west},{south},{east},{north}/{width}/{height}.{format}` ``` +#### Extra options + * Layer: List of layers to be shown in the image (by default `all`), for example `?layer=0,1`. + ### Named Map #### Definition diff --git a/lib/cartodb/api/middlewares/map-store-map-config-provider.js b/lib/cartodb/api/middlewares/map-store-map-config-provider.js index 58cdb79e..a5b4b1f6 100644 --- a/lib/cartodb/api/middlewares/map-store-map-config-provider.js +++ b/lib/cartodb/api/middlewares/map-store-map-config-provider.js @@ -10,12 +10,13 @@ module.exports = function createMapStoreMapConfigProvider ( return function createMapStoreMapConfigProviderMiddleware (req, res, next) { const { user, token, cache_buster, api_key } = res.locals; const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals; - const { layer, z, x, y, scale_factor, format } = req.params; + const { layer: layerFromParams, z, x, y, scale_factor, format } = req.params; + const { layer: layerFromQuery } = req.query; const params = { user, token, cache_buster, api_key, dbuser, dbname, dbpassword, dbhost, dbport, - layer, z, x, y, scale_factor, format + layer: (layerFromQuery || layerFromParams), z, x, y, scale_factor, format }; if (forcedFormat) { diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index dff1280e..78036f0d 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,4 +1,4 @@ -var os = require('os'); +const fqdn = require('@carto/fqdn-sync'); var _ = require('underscore'); var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter'); @@ -36,8 +36,7 @@ rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter; // See https://github.com/CartoDB/Windshaft-cartodb/issues/153 if (global.environment.statsd) { if (global.environment.statsd.prefix) { - var host_token = os.hostname().split('.').reverse().join('.'); - global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token); + global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, fqdn.reverse()); } } diff --git a/package.json b/package.json index 7216fab2..6bb21e49 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "Simon Martin " ], "dependencies": { + "@carto/fqdn-sync": "0.2.2", "basic-auth": "2.0.0", "body-parser": "1.18.3", "camshaft": "0.61.10", diff --git a/test/acceptance/analysis/named-maps.js b/test/acceptance/analysis/named-maps.js index 98316d2d..5172b110 100644 --- a/test/acceptance/analysis/named-maps.js +++ b/test/acceptance/analysis/named-maps.js @@ -261,6 +261,27 @@ describe('named-maps analysis', function() { ); }); + it('should fail to retrieve static map preview via layergroup ' + + 'when filtering by invalid layers', function(done) { + assert.response( + server, + { + url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.png?layer=1', + method: 'GET', + encoding: 'binary', + headers: { + host: username + } + }, + { + status: 400 + }, + function(res, err) { + done(err); + } + ); + }); + it('should return and an error requesting unsupported image format', function(done) { assert.response( server, diff --git a/test/support/test-client.js b/test/support/test-client.js index 2a5840f5..db014c9a 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -1268,9 +1268,6 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 }); const publicuser = global.environment.postgres.user; - // we need to guarantee all new connections have the new settings - helper.cleanPGPoolConnections(); - const psql = new PSQL({ user: 'postgres', dbname: dbname, @@ -1278,6 +1275,9 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb port: global.environment.postgres.port }); + // we need to guarantee all new connections have the new settings + psql.end(); + step( function configureTimeouts () { const timeoutSQLs = [ diff --git a/test/support/test_helper.js b/test/support/test_helper.js index e381e34d..5770a42f 100644 --- a/test/support/test_helper.js +++ b/test/support/test_helper.js @@ -13,7 +13,6 @@ var lzmaWorker = new LZMA(); var redis = require('redis'); var log4js = require('log4js'); -var pg = require('pg'); const setICUEnvVariable = require('../../lib/cartodb/utils/icu_data_env_setter'); // set environment specific variables @@ -147,11 +146,6 @@ afterEach(function(done) { }); }); -function cleanPGPoolConnections () { - // TODO: this method will be replaced by psql.end - pg.end(); -} - function deleteRedisKeys(keysToDelete, callback) { if (Object.keys(keysToDelete).length === 0) { @@ -214,6 +208,5 @@ module.exports = { checkSurrogateKey: checkSurrogateKey, checkCache: checkCache, rmdirRecursiveSync: rmdirRecursiveSync, - configureMetadata, - cleanPGPoolConnections + configureMetadata }; diff --git a/yarn.lock b/yarn.lock index 593866df..84ec39ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2,6 +2,10 @@ # yarn lockfile v1 +"@carto/fqdn-sync@0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@carto/fqdn-sync/-/fqdn-sync-0.2.2.tgz#cd7c645ed66690a09249b554d254a0f53963b2dc" + "@carto/mapnik@3.6.2-carto.10": version "3.6.2-carto.10" resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.10.tgz#a97c951dcdac09d0eb35b3ea71e5eeaa206c1af6"