diff --git a/.travis.yml b/.travis.yml index ef58e4f9..bbfbe798 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,9 @@ addons: postgresql: "9.3" before_install: + - sudo apt-get update - sudo apt-get install -y pkg-config libcairo2-dev libjpeg8-dev libgif-dev + - sudo apt-get install postgresql-plpython-9.3 - createdb template_postgis - psql -c "CREATE EXTENSION postgis" template_postgis diff --git a/Makefile b/Makefile index 795b997c..32a48859 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,11 @@ config/environments/test.js: config.status--test check-local: config/environments/test.js ./run_tests.sh ${RUNTESTFLAGS} \ - test/unit/cartodb/*.js \ - test/acceptance/*.js + test/unit/cartodb/*.js \ + test/unit/cartodb/cache/model/*.js \ + test/integration/*.js \ + test/acceptance/*.js \ + test/acceptance/cache/*.js check-submodules: PATH="$$PATH:$(srcdir)/node_modules/.bin/"; \ diff --git a/NEWS.md b/NEWS.md index 509dbc9b..7939285b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,7 +1,60 @@ -1.26.3 -- 2015-mm-dd +1.28.5 -- 2015-mm-dd -------------------- +1.28.4 -- 2015-02-18 +-------------------- + +Announcements: +- Upgrades windshaft to [0.37.4](https://github.com/CartoDB/Windshaft/releases/tag/0.37.4) + + +1.28.3 -- 2015-02-17 +-------------------- + +Announcements: +- Upgrades windshaft to [0.37.3](https://github.com/CartoDB/Windshaft/releases/tag/0.37.3) + + +1.28.2 -- 2015-02-17 +-------------------- + +Announcements: +- Upgrades windshaft to [0.37.2](https://github.com/CartoDB/Windshaft/releases/tag/0.37.2) + + +1.28.1 -- 2015-02-17 +-------------------- + +Announcements: +- Upgrades windshaft to [0.37.1](https://github.com/CartoDB/Windshaft/releases/tag/0.37.1) + + +1.28.0 -- 2015-02-17 +-------------------- + +Announcements: +- Upgrades windshaft to [0.37.0](https://github.com/CartoDB/Windshaft/releases/tag/0.37.0) + +New features: + - QueryTablesApi will always use an authenticated query to retrieve last update, this allows to query affected private + tables last update (#253) + + +1.27.0 -- 2015-02-16 +-------------------- + +Announcements: +- Adds default image placeholder for http renderer to use as fallback + +New features: +- `named` layers type, see [MapConfig-NamedMaps-extension](docs/MapConfig-NamedMaps-extension.md) + - Starts using datasource per layer feature from Windshaft ([2c7bc6a](https://github.com/CartoDB/Windshaft-cartodb/commit/2c7bc6adde561b20ed955b905e3c7bcd6795d128)) + +Bugfixes: +- Fixes tests with beforeEach and afterEach triggers + + 1.26.2 -- 2015-01-28 -------------------- diff --git a/assets/default-placeholder.png b/assets/default-placeholder.png new file mode 100644 index 00000000..de4079ef Binary files /dev/null and b/assets/default-placeholder.png differ diff --git a/assets/default-placeholder@2x.png b/assets/default-placeholder@2x.png new file mode 100644 index 00000000..a52d3513 Binary files /dev/null and b/assets/default-placeholder@2x.png differ diff --git a/config/environments/development.js.example b/config/environments/development.js.example index e28c51e6..2c391dd1 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -94,7 +94,13 @@ var config = { proxy: undefined, // the url for a proxy server whitelist: [ // the whitelist of urlTemplates that can be used 'http://{s}.example.com/{z}/{x}/{y}.png' - ] + ], + // image to use as placeholder when urlTemplate is not in the whitelist + // if provided the http renderer will use it instead of throw an error + fallbackImage: { + type: 'fs', // 'fs' and 'url' supported + src: __dirname + '/../../assets/default-placeholder.png' + } } } ,millstone: { diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 6c4ea756..0d4e999a 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -88,7 +88,13 @@ var config = { proxy: undefined, // the url for a proxy server whitelist: [ // the whitelist of urlTemplates that can be used 'http://{s}.example.com/{z}/{x}/{y}.png' - ] + ], + // image to use as placeholder when urlTemplate is not in the whitelist + // if provided the http renderer will use it instead of throw an error + fallbackImage: { + type: 'fs', // 'fs' and 'url' supported + src: __dirname + '/../../assets/default-placeholder.png' + } } } ,millstone: { diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index f8da00db..0c88a9c7 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -88,7 +88,13 @@ var config = { proxy: undefined, // the url for a proxy server whitelist: [ // the whitelist of urlTemplates that can be used 'http://{s}.example.com/{z}/{x}/{y}.png' - ] + ], + // image to use as placeholder when urlTemplate is not in the whitelist + // if provided the http renderer will use it instead of throw an error + fallbackImage: { + type: 'fs', // 'fs' and 'url' supported + src: __dirname + '/../../assets/default-placeholder.png' + } } } ,millstone: { diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 39bf1d15..67551249 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -50,7 +50,7 @@ var config = { ,postgres: { // Parameters to pass to datasource plugin of mapnik // See http://github.com/mapnik/mapnik/wiki/PostGIS - user: "testpublicuser", + user: "test_windshaft_publicuser", password: "public", host: '127.0.0.1', port: 5432, @@ -90,7 +90,13 @@ var config = { 'http://{s}.example.com/{z}/{x}/{y}.png', // for testing purposes 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png' - ] + ], + // image to use as placeholder when urlTemplate is not in the whitelist + // if provided the http renderer will use it instead of throw an error + fallbackImage: { + type: 'fs', // 'fs' and 'url' supported + src: __dirname + '/../../assets/default-placeholder.png' + } } } ,millstone: { diff --git a/configure b/configure index fef3bc71..052a3502 100755 --- a/configure +++ b/configure @@ -56,9 +56,8 @@ while test -n "$1"; do ENVIRONMENT=`echo "$1" | cut -d= -f2` ;; *) - echo "Unknown option '$1'" >&2 - usage >&2 - exit 1 + echo "Unused option '$1'" >&2 + ;; esac shift done diff --git a/docs/MapConfig-NamedMaps-extension.md b/docs/MapConfig-NamedMaps-extension.md new file mode 100644 index 00000000..a6418174 --- /dev/null +++ b/docs/MapConfig-NamedMaps-extension.md @@ -0,0 +1,56 @@ +# 1. Purpose + +This specification describes an extension for +[MapConfig 1.3.0](https://github.com/CartoDB/Windshaft/blob/master/doc/MapConfig-1.3.0.md) version. + + +# 2. Changes over specification + +This extension introduces a new layer type so it's possible to use a named map by its name as a layer. + +## 2.1 Named layers definition + +```javascript +{ + // REQUIRED + // string, `named` is the only supported value + type: "named", + + // REQUIRED + // object, set `named` map layers configuration + options: { + + // REQUIRED + // string, the name for the named map to use + name: "world_borders", + + // OPTIONAL + // object, the replacement values for the named map's template placeholders + // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#instantiate-1 for more details + config: { + "color": "#000" + }, + + // OPTIONAL + // string array, the authorized tokens in case the named map has auth method set to `token` + // See https://github.com/CartoDB/Windshaft-cartodb/blob/master/docs/Map-API.md#named-maps-1 for more details + auth_tokens: [ + "token1", + "token2" + ] + } +} +``` + +## 2.2 Limitations + +1. A Named Map will not allow to have `named` type layers inside their templates layergroup's layers definition. +2. A `named` layer does not allow Named Maps form other accounts, it's only possible to use Named Maps from the very +same user account. + + +# History + +## 1.0.0 + + - Initial version diff --git a/lib/cartodb/api/query_tables_api.js b/lib/cartodb/api/query_tables_api.js index bdcf488e..f4795b05 100644 --- a/lib/cartodb/api/query_tables_api.js +++ b/lib/cartodb/api/query_tables_api.js @@ -1,7 +1,10 @@ -var sqlApi = require('../sql/sql_api'), - PSQL = require('cartodb-psql'); +var sqlApi = require('../sql/sql_api'); +var PSQL = require('cartodb-psql'); +var Step = require('step'); -function QueryTablesApi() { +function QueryTablesApi(pgConnection, metadataBackend) { + this.pgConnection = pgConnection; + this.metadataBackend = metadataBackend; } var affectedTableRegexCache = { @@ -14,11 +17,11 @@ var affectedTableRegexCache = { module.exports = QueryTablesApi; -QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) { +QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) { var query = 'SELECT CDB_QueryTables($windshaft$' + prepareSql(sql) + '$windshaft$)'; - runQuery(username, options, query, handleAffectedTablesInQueryRows, callback); + this.runQuery(username, query, handleAffectedTablesInQueryRows, callback); }; function handleAffectedTablesInQueryRows(err, rows, callback) { @@ -33,7 +36,7 @@ function handleAffectedTablesInQueryRows(err, rows, callback) { callback(null, tableNames); } -QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) { +QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) { var query = [ 'WITH querytables AS (', @@ -44,7 +47,7 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam 'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])' ].join(' '); - runQuery(username, options, query, handleAffectedTablesAndLastUpdatedTimeRows, callback); + this.runQuery(username, query, handleAffectedTablesAndLastUpdatedTimeRows, callback); }; function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { @@ -68,20 +71,60 @@ function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) { } -function runQuery(username, options, query, queryHandler, callback) { +QueryTablesApi.prototype.runQuery = function(username, query, queryHandler, callback) { + var self = this; + if (shouldQueryPostgresDirectly()) { - var psql = new PSQL(options); - psql.query(query, function(err, resultSet) { - resultSet = resultSet || {}; - var rows = resultSet.rows || []; - queryHandler(err, rows, callback); - }); + + var params = {}; + + Step( + function setAuth() { + self.pgConnection.setDBAuth(username, params, this); + }, + function setConn(err) { + if (err) { + throw err; + } + self.pgConnection.setDBConn(username, params, this); + }, + function executeQuery(err) { + if (err) { + throw err; + } + var psql = new PSQL({ + user: params.dbuser, + pass: params.dbpass, + host: params.dbhost, + port: params.dbport, + dbname: params.dbname + }); + psql.query(query, function(err, resultSet) { + resultSet = resultSet || {}; + var rows = resultSet.rows || []; + queryHandler(err, rows, callback); + }); + } + ); + } else { - sqlApi.query(username, options.api_key, query, function(err, rows) { - queryHandler(err, rows, callback); - }); + + Step( + function getApiKey() { + self.metadataBackend.getUserMapKey(username, this); + }, + function executeQuery(err, apiKey) { + if (err) { + throw err; + } + sqlApi.query(username, apiKey, query, function(err, rows) { + queryHandler(err, rows, callback); + }); + } + ); + } -} +}; function prepareSql(sql) { diff --git a/lib/cartodb/backends/pg_connection.js b/lib/cartodb/backends/pg_connection.js new file mode 100644 index 00000000..37209b98 --- /dev/null +++ b/lib/cartodb/backends/pg_connection.js @@ -0,0 +1,96 @@ +var Step = require('step'); +var _ = require('underscore'); + +function PgConnection(metadataBackend) { + this.metadataBackend = metadataBackend; +} + +module.exports = PgConnection; + + +// Set db authentication parameters to those of the given username +// +// @param username the cartodb username, mapped to a database username +// via CartodbRedis metadata records +// +// @param params the parameters to set auth options into +// added params are: "dbuser" and "dbpassword" +// +// @param callback function(err) +// +PgConnection.prototype.setDBAuth = function(username, params, callback) { + var self = this; + + var user_params = {}; + var auth_user = global.environment.postgres_auth_user; + var auth_pass = global.environment.postgres_auth_pass; + Step( + function getId() { + self.metadataBackend.getUserId(username, this); + }, + function(err, user_id) { + if (err) throw err; + user_params['user_id'] = user_id; + var dbuser = _.template(auth_user, user_params); + _.extend(params, {dbuser:dbuser}); + + // skip looking up user_password if postgres_auth_pass + // doesn't contain the "user_password" label + if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null; + + self.metadataBackend.getUserDBPass(username, this); + }, + function(err, user_password) { + if (err) throw err; + user_params['user_password'] = user_password; + if ( auth_pass ) { + var dbpass = _.template(auth_pass, user_params); + _.extend(params, {dbpassword:dbpass}); + } + return true; + }, + function finish(err) { + callback(err); + } + ); +}; + +// Set db connection parameters to those for the given username +// +// @param dbowner cartodb username of database owner, +// mapped to a database username +// via CartodbRedis metadata records +// +// @param params the parameters to set connection options into +// added params are: "dbname", "dbhost" +// +// @param callback function(err) +// +PgConnection.prototype.setDBConn = function(dbowner, params, callback) { + var self = this; + // Add default database connection parameters + // if none given + _.defaults(params, { + dbuser: global.environment.postgres.user, + dbpassword: global.environment.postgres.password, + dbhost: global.environment.postgres.host, + dbport: global.environment.postgres.port + }); + Step( + function getConnectionParams() { + self.metadataBackend.getUserDBConnectionParams(dbowner, this); + }, + function extendParams(err, dbParams){ + if (err) throw err; + // we don't want null values or overwrite a non public user + if (params.dbuser != 'publicuser' || !dbParams.dbuser) { + delete dbParams.dbuser; + } + if ( dbParams ) _.extend(params, dbParams); + return null; + }, + function finish(err) { + callback(err); + } + ); +}; diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js index b0040668..bff4d0b1 100644 --- a/lib/cartodb/cartodb_windshaft.js +++ b/lib/cartodb/cartodb_windshaft.js @@ -1,11 +1,9 @@ -var _ = require('underscore') - , Step = require('step') - , Windshaft = require('windshaft') - , TemplateMaps = require('./template_maps.js') - , Cache = require('./cache_validator') - , os = require('os') - , HealthCheck = require('./monitoring/health_check') -; +var _ = require('underscore'); +var Step = require('step'); +var Windshaft = require('windshaft'); +var Cache = require('./cache_validator'); +var os = require('os'); +var HealthCheck = require('./monitoring/health_check'); if ( ! process.env['PGAPPNAME'] ) process.env['PGAPPNAME']='cartodb_tiler'; @@ -25,18 +23,14 @@ var CartodbWindshaft = function(serverOptions) { var cartoData = require('cartodb-redis')({pool: redisPool}); + var templateMaps = serverOptions.templateMaps; + // This is for Templated maps // // "named" is the official, "template" is for backward compatibility up to 1.6.x // var template_baseurl = global.environment.base_url_templated || '(?:/maps/named|/tiles/template)'; - var templateMapsOpts = { - max_user_templates: global.environment.maxUserTemplates - }; - var templateMaps = new TemplateMaps(redisPool, templateMapsOpts); - serverOptions.templateMaps = templateMaps; - var SurrogateKeysCache = require('./cache/surrogate_keys_cache'), NamedMapsCacheEntry = require('./cache/model/named_maps_entry'), VarnishHttpCacheBackend = require('./cache/backend/varnish_http'), @@ -141,7 +135,14 @@ var CartodbWindshaft = function(serverOptions) { var TemplateMapsController = require('./controllers/template_maps'), templateMapsController = new TemplateMapsController( - ws, serverOptions, templateMaps, cartoData, template_baseurl, surrogateKeysCache, NamedMapsCacheEntry + ws, + serverOptions, + templateMaps, + cartoData, + template_baseurl, + surrogateKeysCache, + NamedMapsCacheEntry, + serverOptions.pgConnection ); templateMapsController.register(ws); diff --git a/lib/cartodb/controllers/template_maps.js b/lib/cartodb/controllers/template_maps.js index db525c7c..5b61ae56 100644 --- a/lib/cartodb/controllers/template_maps.js +++ b/lib/cartodb/controllers/template_maps.js @@ -2,7 +2,7 @@ var Step = require('step'); var _ = require('underscore'); function TemplateMapsController(app, serverOptions, templateMaps, metadataBackend, templateBaseUrl, surrogateKeysCache, - NamedMapsCacheEntry) { + NamedMapsCacheEntry, pgConnection) { this.app = app; this.serverOptions = serverOptions; this.templateMaps = templateMaps; @@ -10,6 +10,7 @@ function TemplateMapsController(app, serverOptions, templateMaps, metadataBacken this.templateBaseUrl = templateBaseUrl; this.surrogateKeysCache = surrogateKeysCache; this.NamedMapsCacheEntry = NamedMapsCacheEntry; + this.pgConnection = pgConnection; } module.exports = TemplateMapsController; @@ -465,11 +466,11 @@ TemplateMapsController.prototype.setDBParams = function(cdbuser, params, callbac var self = this; Step( function setAuth() { - self.serverOptions.setDBAuth(cdbuser, params, this); + self.pgConnection.setDBAuth(cdbuser, params, this); }, function setConn(err) { if ( err ) throw err; - self.serverOptions.setDBConn(cdbuser, params, this); + self.pgConnection.setDBConn(cdbuser, params, this); }, function finish(err) { callback(err); diff --git a/lib/cartodb/models/mapconfig_named_layers_adapter.js b/lib/cartodb/models/mapconfig_named_layers_adapter.js new file mode 100644 index 00000000..ceb9c429 --- /dev/null +++ b/lib/cartodb/models/mapconfig_named_layers_adapter.js @@ -0,0 +1,120 @@ +var queue = require('queue-async'); +var _ = require('underscore'); +var Datasource = require('windshaft').Datasource; + +function MapConfigNamedLayersAdapter(templateMaps) { + this.templateMaps = templateMaps; +} + +module.exports = MapConfigNamedLayersAdapter; + +MapConfigNamedLayersAdapter.prototype.getLayers = function(username, layers, dbMetadata, callback) { + var self = this; + + var adaptLayersQueue = queue(layers.length); + + function adaptLayer(layer, done) { + if (isNamedTypeLayer(layer)) { + + if (!layer.options.name) { + return done(new Error('Missing Named Map `name` in layer options')); + } + + var templateName = layer.options.name; + var templateConfigParams = layer.options.config || {}; + var templateAuthTokens = layer.options.auth_tokens; + + self.templateMaps.getTemplate(username, templateName, function(err, template) { + if (err || !template) { + return done(new Error("Template '" + templateName + "' of user '" + username + "' not found")); + } + + if (self.templateMaps.isAuthorized(template, templateAuthTokens)) { + var nestedNamedLayers = template.layergroup.layers.filter(function(layer) { + return layer.type === 'named'; + }); + + if (nestedNamedLayers.length > 0) { + var nestedNamedMapsError = new Error('Nested named layers are not allowed'); + // nestedNamedMapsError.http_status = 400; + return done(nestedNamedMapsError); + } + + try { + var templateLayergroupConfig = self.templateMaps.instance(template, templateConfigParams); + return done(null, { + datasource: true, + layers: templateLayergroupConfig.layers + }); + } catch (err) { + return done(err); + } + } else { + var unauthorizedError = new Error("Unauthorized '" + templateName + "' template instantiation"); + unauthorizedError.http_status = 403; + return done(unauthorizedError); + } + }); + + } else { + return done(null, { + datasource: false, + layers: [layer] + }); + } + } + + var datasourceBuilder = new Datasource.Builder(); + + function layersAdaptQueueFinish(err, layersResults) { + if (err) { + return callback(err); + } + + if (!layersResults || layersResults.length === 0) { + return callback(new Error('Missing layers array from layergroup config')); + } + + var layers = [], + currentLayerIndex = 0; + + layersResults.forEach(function(layersResult) { + + layersResult.layers.forEach(function(layer) { + layers.push(layer); + if (layersResult.datasource) { + datasourceBuilder.withLayerDatasource(currentLayerIndex, { + user: dbAuth.dbuser + }); + } + currentLayerIndex++; + }); + + }); + + return callback(null, layers, datasourceBuilder.build()); + } + + + var dbAuth = {}; + + if (_.some(layers, isNamedTypeLayer)) { + // Lazy load dbAuth + dbMetadata.setDBAuth(username, dbAuth, function(err) { + if (err) { + return callback(err); + } + layers.forEach(function(layer) { + adaptLayersQueue.defer(adaptLayer, layer); + }); + adaptLayersQueue.awaitAll(layersAdaptQueueFinish); + }); + } else { + return callback(null, layers, datasourceBuilder.build()); + } + +}; + +function isNamedTypeLayer(layer) { + return layer.type === 'named'; +} diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 5620a9ae..807024a8 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,8 +1,11 @@ var _ = require('underscore'); var Step = require('step'); var QueryTablesApi = require('./api/query_tables_api'); +var PgConnection = require('./backends/pg_connection'); var crypto = require('crypto'); var LZMA = require('lzma').LZMA; +var TemplateMaps = require('./template_maps.js'); +var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter'); // This is for backward compatibility with 1.3.3 if ( _.isUndefined(global.environment.sqlapi.domain) ) { @@ -32,10 +35,13 @@ var REQUEST_QUERY_PARAMS_WHITELIST = [ ]; module.exports = function(redisPool) { - var redisOpts = redisPool ? {pool: redisPool} : global.environment.redis; - var cartoData = require('cartodb-redis')(redisOpts), + redisPool = redisPool + || require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:server_options'})); + + var cartoData = require('cartodb-redis')({ pool: redisPool }), lzmaWorker = new LZMA(), - queryTablesApi = new QueryTablesApi(); + pgConnection = new PgConnection(cartoData), + queryTablesApi = new QueryTablesApi(pgConnection, cartoData); var rendererConfig = _.defaults(global.environment.renderer || {}, { cache_ttl: 60000, // milliseconds @@ -102,6 +108,16 @@ module.exports = function(redisPool) { // Re-use redisPool me.redis.pool = redisPool; + // Re-use pgConnection + me.pgConnection = pgConnection; + + var templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + me.templateMaps = templateMaps; + + var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + /* This whole block is about generating X-Cache-Channel { */ // TODO: review lifetime of elements of this cache @@ -192,38 +208,8 @@ module.exports = function(redisPool) { } return [req.params.table]; } - var user, key; - var next = this; - Step ( - function findUserKey() { - if ( req.params.hasOwnProperty('_authorizedBySigner') ) { - user = req.params._authorizedBySigner; - cartoData.getUserMapKey(user, this); - } else { - user = that.userByReq(req); - key = req.params.map_key || req.params.api_key; - return null; - } - }, - function getAffected(err, data) { - if ( err ) throw err; - if ( data ) { - if ( req.profiler ) req.profiler.done('getSignerMapKey'); - key = data; - } - queryTablesApi.getAffectedTablesInQuery(user, { - user: req.params.dbuser, - pass: req.params.dbpass, - host: req.params.dbhost, - port: req.params.dbport, - dbname: req.params.dbname, - api_key: key - }, sql, this); // in addCacheChannel - }, - function finish(err, data) { - next(err,data); - } - ); + + queryTablesApi.getAffectedTablesInQuery(that.userByReq(req), sql, this); // in addCacheChannel }, function buildCacheChannel(err, tableNames) { if ( err ) throw err; @@ -294,6 +280,17 @@ module.exports = function(redisPool) { }); }; + me.beforeLayergroupCreate = function(req, requestMapConfig, callback) { + mapConfigNamedLayersAdapter.getLayers(this.userByReq(req), requestMapConfig.layers, pgConnection, function(err, layers, datasource) { + if (err) { + return callback(err); + } + + requestMapConfig.layers = layers; + return callback(null, requestMapConfig, datasource) + }); + }; + me.afterLayergroupCreate = function(req, mapconfig, response, callback) { var token = response.layergroupid; @@ -329,28 +326,16 @@ module.exports = function(redisPool) { done(); }); - var sql = []; - _.each(mapconfig.layers, function(lyr) { - sql.push(lyr.options.sql); - }); - sql = sql.join(';'); + var sql = mapconfig.layers.map(function(layer) { + return layer.options.sql; + }).join(';'); var dbName = req.params.dbname; - var usr = this.userByReq(req); - var key = req.params.map_key || req.params.api_key; - var cacheKey = dbName + ':' + token; Step( function getAffectedTablesAndLastUpdatedTime() { - queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, { - user: req.params.dbuser, - pass: req.params.dbpass, - host: req.params.dbhost, - port: req.params.dbport, - dbname: req.params.dbname, - api_key: key - }, sql, this); + queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); }, function handleAffectedTablesAndLastUpdatedTime(err, result) { if (req.profiler) req.profiler.done('queryTablesAndLastUpdated'); @@ -405,92 +390,6 @@ module.exports = function(redisPool) { return mat[1]; }; - // Set db authentication parameters to those of the given username - // - // @param username the cartodb username, mapped to a database username - // via CartodbRedis metadata records - // - // @param params the parameters to set auth options into - // added params are: "dbuser" and "dbpassword" - // - // @param callback function(err) - // - me.setDBAuth = function(username, params, callback) { - - var user_params = {}; - var auth_user = global.environment.postgres_auth_user; - var auth_pass = global.environment.postgres_auth_pass; - Step( - function getId() { - cartoData.getUserId(username, this); - }, - function(err, user_id) { - if (err) throw err; - user_params['user_id'] = user_id; - var dbuser = _.template(auth_user, user_params); - _.extend(params, {dbuser:dbuser}); - - // skip looking up user_password if postgres_auth_pass - // doesn't contain the "user_password" label - if (!auth_pass || ! auth_pass.match(/\buser_password\b/) ) return null; - - cartoData.getUserDBPass(username, this); - }, - function(err, user_password) { - if (err) throw err; - user_params['user_password'] = user_password; - if ( auth_pass ) { - var dbpass = _.template(auth_pass, user_params); - _.extend(params, {dbpassword:dbpass}); - } - return true; - }, - function finish(err) { - callback(err); - } - ); - }; - - // Set db connection parameters to those for the given username - // - // @param dbowner cartodb username of database owner, - // mapped to a database username - // via CartodbRedis metadata records - // - // @param params the parameters to set connection options into - // added params are: "dbname", "dbhost" - // - // @param callback function(err) - // - me.setDBConn = function(dbowner, params, callback) { - // Add default database connection parameters - // if none given - _.defaults(params, { - dbuser: global.environment.postgres.user, - dbpassword: global.environment.postgres.password, - dbhost: global.environment.postgres.host, - dbport: global.environment.postgres.port - }); - Step( - function getConnectionParams() { - cartoData.getUserDBConnectionParams(dbowner, this); - }, - function extendParams(err, dbParams){ - if (err) throw err; - // we don't want null values or overwrite a non public user - if (params.dbuser != 'publicuser' || !dbParams.dbuser) { - delete dbParams.dbuser; - } - if ( dbParams ) _.extend(params, dbParams); - return null; - }, - function finish(err) { - callback(err); - } - ); - }; - - // Check if a request is authorized by a signer // // @param req express request object @@ -589,7 +488,7 @@ module.exports = function(redisPool) { _.extend(req.params, { _authorizedByApiKey: true }); // authorized by api key, login as the given username and stop - that.setDBAuth(user, req.params, function(err) { + pgConnection.setDBAuth(user, req.params, function(err) { callback(err, true); // authorized (or error) }); }, @@ -624,7 +523,7 @@ module.exports = function(redisPool) { // Authorized by "signed_by" ! _.extend(req.params, { _authorizedBySigner: signed_by }); - that.setDBAuth(signed_by, req.params, function(err) { + pgConnection.setDBAuth(signed_by, req.params, function(err) { if (req.profiler) req.profiler.done('setDBAuth'); callback(err, true); // authorized (or error) }); @@ -738,7 +637,7 @@ module.exports = function(redisPool) { }, function getDatabase(err){ if(err) throw err; - that.setDBConn(user, req.params, this); + pgConnection.setDBConn(user, req.params, this); }, function finishSetup(err) { if ( err ) { callback(err, req); return; } diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index f2f6d72e..2e52be20 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,10 +1,10 @@ { "name": "windshaft-cartodb", - "version": "1.26.3", + "version": "1.28.5", "dependencies": { "cartodb-psql": { "version": "0.4.0", - "from": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0", + "from": "cartodb-psql@~0.4.0", "resolved": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0", "dependencies": { "pg": { @@ -18,8 +18,7 @@ }, "buffer-writer": { "version": "1.0.0", - "from": "buffer-writer@1.0.0", - "resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz" + "from": "buffer-writer@1.0.0" } } } @@ -27,7 +26,7 @@ }, "cartodb-redis": { "version": "0.11.0", - "from": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0", + "from": "cartodb-redis@~0.11.0", "resolved": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0", "dependencies": { "redis-mpool": { @@ -88,8 +87,7 @@ }, "isarray": { "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "from": "isarray@0.0.1" }, "string_decoder": { "version": "0.10.31", @@ -113,9 +111,14 @@ "from": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0", "resolved": "https://github.com/Vizzuality/node-varnish/tarball/0.3.0" }, + "queue-async": { + "version": "1.0.7", + "from": "queue-async@~1.0.7", + "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz" + }, "redis-mpool": { "version": "0.3.0", - "from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0", + "from": "redis-mpool@~0.3.0", "resolved": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0", "dependencies": { "generic-pool": { @@ -178,24 +181,17 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "0.35.1", - "from": "https://github.com/CartoDB/Windshaft/tarball/0.35.1", - "resolved": "https://github.com/CartoDB/Windshaft/tarball/0.35.1", + "version": "0.37.4", + "from": "windshaft@~0.37.4", "dependencies": { "chronograph": { "version": "0.1.0", "from": "chronograph@git://github.com/CartoDB/chronographjs.git#0.1.0", "resolved": "git://github.com/CartoDB/chronographjs.git#0b8c35eee510cfa14a16be24d70533b38ecc1d2d" }, - "queue-async": { - "version": "1.0.7", - "from": "queue-async@~1.0.7", - "resolved": "https://registry.npmjs.org/queue-async/-/queue-async-1.0.7.tgz" - }, "grainstore": { - "version": "0.22.1", - "from": "https://github.com/CartoDB/grainstore/tarball/0.22.1", - "resolved": "https://github.com/CartoDB/grainstore/tarball/0.22.1", + "version": "0.23.0", + "from": "grainstore@~0.23.0", "dependencies": { "carto": { "version": "0.9.5-cdb2", @@ -1637,9 +1633,9 @@ "resolved": "https://registry.npmjs.org/connect/-/connect-1.9.2.tgz", "dependencies": { "formidable": { - "version": "1.0.16", + "version": "1.0.17", "from": "formidable@1.0.x", - "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.16.tgz" + "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz" } } }, @@ -2213,8 +2209,7 @@ }, "isarray": { "version": "0.0.1", - "from": "isarray@0.0.1", - "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + "from": "isarray@0.0.1" }, "string_decoder": { "version": "0.10.31", diff --git a/package.json b/package.json index 660646b4..a45a697a 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "1.26.3", + "version": "1.28.5", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -27,10 +27,11 @@ "dot": "~1.0.2", "windshaft": "https://github.com/CartoDB/Windshaft/tarball/259-remove-old-api", "step": "~0.0.5", + "queue-async": "~1.0.7", "request": "~2.9.203", - "cartodb-redis": "https://github.com/CartoDB/node-cartodb-redis/tarball/0.11.0", - "cartodb-psql": "https://github.com/CartoDB/node-cartodb-psql/tarball/0.4.0", - "redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.3.0", + "cartodb-redis": "~0.11.0", + "cartodb-psql": "~0.4.0", + "redis-mpool": "~0.3.0", "lzma": "~1.3.7", "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb", "rollbar": "~0.3.13" diff --git a/test/acceptance/cache/surrogate_keys_invalidation.js b/test/acceptance/cache/surrogate_keys_invalidation.js index c4bb3818..118c6a63 100644 --- a/test/acceptance/cache/surrogate_keys_invalidation.js +++ b/test/acceptance/cache/surrogate_keys_invalidation.js @@ -16,9 +16,13 @@ var serverOptions = ServerOptions(); suite('templates surrogate keys', function() { - var redisClient, - sqlApiServer, - server; + var sqlApiServer; + var redisClient = redis.createClient(global.environment.redis.port); + + // Enable Varnish purge for tests + serverOptions.varnish_purge_enabled = true; + + var server = new CartodbWindshaft(serverOptions); var templateOwner = 'localhost', templateName = 'acceptance', @@ -45,19 +49,12 @@ suite('templates surrogate keys', function() { expectedBody = { template_id: expectedTemplateId }; suiteSetup(function(done) { - // Enable Varnish purge for tests - serverOptions.varnish_purge_enabled = true; - - server = new CartodbWindshaft(serverOptions); - sqlApiServer = new SqlApiEmulator(global.environment.sqlapi.port, done); - - redisClient = redis.createClient(global.environment.redis.port); }); var surrogateKeysCacheInvalidateFn = SurrogateKeysCache.prototype.invalidate; - beforeEach(function(done) { + function createTemplate(callback) { var postTemplateRequest = { url: '/tiles/template?api_key=1234', method: 'POST', @@ -90,10 +87,10 @@ suite('templates surrogate keys', function() { return true; }, function finish(err) { - done(err); + callback(err); } ); - }); + } test("update template calls surrogate keys invalidation", function(done) { var cacheEntryKey; @@ -104,7 +101,13 @@ suite('templates surrogate keys', function() { }; Step( - function putValidTemplate() { + function createTemplateToUpdate() { + createTemplate(this); + }, + function putValidTemplate(err) { + if (err) { + throw err; + } var updateTemplateRequest = { url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234', method: 'PUT', @@ -163,7 +166,13 @@ suite('templates surrogate keys', function() { }; Step( - function putValidTemplate() { + function createTemplateToDelete() { + createTemplate(this); + }, + function deleteValidTemplate(err) { + if (err) { + throw err; + } var deleteTemplateRequest = { url: '/tiles/template/' + expectedTemplateId + '/?api_key=1234', method: 'DELETE', @@ -199,12 +208,10 @@ suite('templates surrogate keys', function() { ); }); - afterEach(function(done) { - SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn; - done(); - }); - suiteTeardown(function(done) { + SurrogateKeysCache.prototype.invalidate = surrogateKeysCacheInvalidateFn; + // Enable Varnish purge for tests + serverOptions.varnish_purge_enabled = false; sqlApiServer.close(done); }); diff --git a/test/acceptance/health_check.js b/test/acceptance/health_check.js index af8c361e..b6409574 100644 --- a/test/acceptance/health_check.js +++ b/test/acceptance/health_check.js @@ -7,7 +7,7 @@ var server = new CartodbWindshaft(serverOptions); suite('health checks', function () { - beforeEach(function (done) { + function resetHealthConfig() { global.environment.health = { enabled: true, username: 'localhost', @@ -15,8 +15,7 @@ suite('health checks', function () { x: 0, y: 0 }; - done(); - }); + } var healthCheckRequest = { url: '/health', @@ -27,13 +26,14 @@ suite('health checks', function () { }; test('returns 200 and ok=true with enabled configuration', function (done) { + resetHealthConfig(); + assert.response(server, healthCheckRequest, { status: 200 }, function (res, err) { - console.log(res.body); assert.ok(!err); var parsed = JSON.parse(res.body); @@ -47,6 +47,8 @@ suite('health checks', function () { }); test('fails for invalid user because it is not in redis', function (done) { + resetHealthConfig(); + global.environment.health.username = 'invalid'; assert.response(server, diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index aff420d4..84b1e220 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -24,8 +24,6 @@ server.setMaxListeners(0); [true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) { -global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue}; - suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { var redis_client = redis.createClient(global.environment.redis.port); @@ -38,6 +36,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( var test_database = test_user + '_db'; suiteSetup(function(done){ + global.environment.enabledFeatures = { cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue }; sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); }); @@ -112,15 +111,17 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( assert.ok(cc); var dbname = test_database; assert.equal(cc.substring(0, dbname.length), dbname); - var jsonquery = cc.substring(dbname.length+1); - var sentquery = JSON.parse(jsonquery); - var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join(''); - assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$' - + expectedQuery - + '$windshaft$) as tablenames )' - + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max' - + ' FROM CDB_TableMetadata m' - + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API + var jsonquery = cc.substring(dbname.length + 1); + var sentquery = JSON.parse(jsonquery); + var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join(''); + assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$' + + expectedQuery + + '$windshaft$) as tablenames )' + + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max' + + ' FROM CDB_TableMetadata m' + + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'); + } assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL, function(err, similarity) { @@ -337,7 +338,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( layers: [ { options: { sql: 'select 1 as cartodb_id, ' - + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', + + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator from test_table limit 1', cartocss: '#layer { polygon-fill:red; }', cartocss_version: '2.0.1', interactivity: 'cartodb_id' @@ -388,21 +389,23 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( // Check X-Cache-Channel var cc = res.headers['x-cache-channel']; - assert.ok(cc); + assert.ok(cc); var dbname = test_database; assert.equal(cc.substring(0, dbname.length), dbname); - var jsonquery = cc.substring(dbname.length+1); - var sentquery = JSON.parse(jsonquery); - var expectedQuery = layergroup.layers[0].options.sql - .replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)') - .replace(/!pixel_width!/g, '1') - .replace(/!pixel_height!/g, '1'); - assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$' - + expectedQuery - + '$windshaft$) as tablenames )' - + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max' - + ' FROM CDB_TableMetadata m' - + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API + var jsonquery = cc.substring(dbname.length + 1); + var sentquery = JSON.parse(jsonquery); + var expectedQuery = layergroup.layers[0].options.sql + .replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)') + .replace(/!pixel_width!/g, '1') + .replace(/!pixel_height!/g, '1'); + assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$' + + expectedQuery + + '$windshaft$) as tablenames )' + + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max' + + ' FROM CDB_TableMetadata m' + + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'); + } assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err, similarity) { @@ -428,18 +431,20 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( assert.ok(cc); var dbname = test_database; assert.equal(cc.substring(0, dbname.length), dbname); - var jsonquery = cc.substring(dbname.length+1); - var sentquery = JSON.parse(jsonquery); - var expectedQuery = layergroup.layers[0].options.sql - .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)') - .replace('!pixel_width!', '1') - .replace('!pixel_height!', '1'); - assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$' - + expectedQuery - + '$windshaft$) as tablenames )' - + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max' - + ' FROM CDB_TableMetadata m' - + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API + var jsonquery = cc.substring(dbname.length + 1); + var sentquery = JSON.parse(jsonquery); + var expectedQuery = layergroup.layers[0].options.sql + .replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)') + .replace('!pixel_width!', '1') + .replace('!pixel_height!', '1'); + assert.equal(sentquery.q, 'WITH querytables AS ( SELECT * FROM CDB_QueryTables($windshaft$' + + expectedQuery + + '$windshaft$) as tablenames )' + + ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max' + + ' FROM CDB_TableMetadata m' + + ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'); + } assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL, function(err, similarity) { @@ -1218,8 +1223,10 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( var parsedBody = JSON.parse(res.body); var token_components = parsedBody.layergroupid.split(':'); expected_token = token_components[0]; - var last_request = sqlapi_server.getLastRequest(); - assert.equal(last_request.method, 'POST'); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API + var last_request = sqlapi_server.getLastRequest(); + assert.equal(last_request.method, 'POST'); + } return null; }, function cleanup(err) { @@ -1283,6 +1290,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( ); }); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API // See https://github.com/CartoDB/Windshaft-cartodb/issues/167 test("lack of response from sql-api will result in a timeout", function(done) { @@ -1323,6 +1331,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( } ); }); + } var layergroupTtlRequest = { url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify({ diff --git a/test/acceptance/named_layers.js b/test/acceptance/named_layers.js new file mode 100644 index 00000000..3b52f982 --- /dev/null +++ b/test/acceptance/named_layers.js @@ -0,0 +1,655 @@ +var test_helper = require('../support/test_helper'); + +var assert = require('../support/assert'); +var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft'); +var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')(); +var server = new CartodbWindshaft(serverOptions); + +var RedisPool = require('redis-mpool'); +var TemplateMaps = require('../../lib/cartodb/template_maps.js'); + +var Step = require('step'); +var _ = require('underscore'); + +suite('named_layers', function() { + // configure redis pool instance to use in tests + var redisPool = RedisPool(global.environment.redis); + + var templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + + var wadusLayer = { + type: 'cartodb', + options: { + sql: 'select 1 cartodb_id, null::geometry the_geom_webmercator', + cartocss: '#layer { marker-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } + }; + + var username = 'localhost'; + + var templateName = 'valid_template'; + var template = { + version: '0.0.1', + name: templateName, + auth: { + method: 'open' + }, + "placeholders": { + "color": { + "type": "css_color", + "default": "#cc3300" + } + }, + layergroup: { + layers: [ + wadusLayer + ] + } + }; + + var tokenAuthTemplateName = 'auth_valid_template'; + var tokenAuthTemplate = { + version: '0.0.1', + name: tokenAuthTemplateName, + auth: { + method: 'token', + valid_tokens: ['valid1', 'valid2'] + }, + placeholders: { + color: { + "type": "css_color", + "default": "#cc3300" + } + }, + layergroup: { + layers: [ + wadusLayer + ] + } + }; + + + var namedMapLayer = { + type: 'named', + options: { + name: templateName, + config: {}, + auth_tokens: [] + } + }; + + var nestedNamedMapTemplateName = 'nested_template'; + var nestedNamedMapTemplate = { + version: '0.0.1', + name: nestedNamedMapTemplateName, + auth: { + method: 'open' + }, + layergroup: { + layers: [ + namedMapLayer + ] + } + }; + + suiteSetup(function(done) { + global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true}; + templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) { + if (err) { + return done(err); + } + templateMaps.addTemplate(username, tokenAuthTemplate, function(err) { + if (err) { + return done(err); + } + templateMaps.addTemplate(username, template, function(err) { + return done(err); + }); + }); + }); + }); + + test('should fail for non-existing template name', function(done) { + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'named', + options: { + name: 'nonexistent' + } + } + ] + }; + + Step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 400 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.deepEqual(parsedBody, { errors: ["Template 'nonexistent' of user 'localhost' not found"] }); + + return null; + }, + function finish(err) { + done(err); + } + ); + }); + + test('should return 403 if not properly authorized', function(done) { + + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'named', + options: { + name: tokenAuthTemplateName, + config: {}, + auth_tokens: ['token1'] + } + } + ] + }; + + Step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 403 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.deepEqual( + parsedBody, + { errors: [ "Unauthorized 'auth_valid_template' template instantiation" ] } + ); + + return null; + }, + function finish(err) { + done(err); + } + ); + + }); + + test('should return 200 and layergroup if properly authorized', function(done) { + + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'named', + options: { + name: tokenAuthTemplateName, + config: {}, + auth_tokens: ['valid1'] + } + } + ] + }; + + Step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 200 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.ok(parsedBody.layergroupid); + assert.ok(parsedBody.last_updated); + + return null; + }, + function finish(err) { + done(err); + } + ); + + }); + + test('should return 400 for nested named map layers', function(done) { + + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'named', + options: { + name: nestedNamedMapTemplateName + } + } + ] + }; + + Step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 400 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.deepEqual(parsedBody, { errors: [ 'Nested named layers are not allowed' ] }); + + return null; + }, + function finish(err) { + done(err); + } + ); + + }); + + test('should return 200 and layergroup with private tables', function(done) { + + var privateTableTemplateName = 'private_table_template'; + var privateTableTemplate = { + version: '0.0.1', + name: privateTableTemplateName, + auth: { + method: 'open' + }, + layergroup: { + layers: [ + { + type: 'cartodb', + options: { + sql: 'select * from test_table_private_1', + cartocss: '#layer { marker-fill: #cc3300; }', + cartocss_version: '2.3.0' + } + } + ] + } + }; + + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'named', + options: { + name: privateTableTemplateName + } + } + ] + }; + + Step( + function createTemplate() { + templateMaps.addTemplate(username, privateTableTemplate, this); + }, + function createLayergroup(err) { + if (err) { + throw err; + } + + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 200 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.ok(parsedBody.layergroupid); + assert.ok(parsedBody.last_updated); + + return parsedBody.layergroupid; + }, + function requestTile(err, layergroupId) { + if (err) { + throw err; + } + + var next = this; + assert.response(server, + { + url: '/tiles/layergroup/' + layergroupId + '/0/0/0.png', + method: 'GET', + headers: { + host: 'localhost' + }, + encoding: 'binary' + }, + { + status: 200, + headers: { + 'content-type': 'image/png' + } + }, + function(res, err) { + next(err, res); + } + ); + }, + function handleTileResponse(err, res) { + if (err) { + throw err; + } + test_helper.checkCache(res); + return true; + }, + function deleteTemplate(err) { + var next = this; + templateMaps.delTemplate(username, privateTableTemplate, function(/*delErr*/) { + // ignore deletion error + next(err); + }); + }, + function finish(err) { + done(err); + } + ); + + }); + + test('should return 200 and layergroup with private tables and interactivity', function(done) { + + var privateTableTemplateNameInteractivity = 'private_table_template_interactivity'; + var privateTableTemplate = { + "version": "0.0.1", + "auth": { + "method": "open" + }, + "name": privateTableTemplateNameInteractivity, + "layergroup": { + "layers": [ + { + "type": "cartodb", + "options": { + "attributes": { + "columns": [ + "name" + ], + "id": "cartodb_id" + }, + "cartocss": "#layer { marker-fill: #cc3300; }", + "cartocss_version": "2.3.0", + "interactivity": "cartodb_id", + "sql": "select * from test_table_private_1" + } + } + ] + } + }; + + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'named', + options: { + name: privateTableTemplateNameInteractivity + } + } + ] + }; + + Step( + function createTemplate() { + templateMaps.addTemplate(username, privateTableTemplate, this); + }, + function createLayergroup(err) { + if (err) { + throw err; + } + + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 200 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.ok(parsedBody.layergroupid); + assert.ok(parsedBody.last_updated); + + return parsedBody.layergroupid; + }, + function requestTile(err, layergroupId) { + if (err) { + throw err; + } + + var next = this; + assert.response(server, + { + url: '/tiles/layergroup/' + layergroupId + '/0/0/0.png', + method: 'GET', + headers: { + host: 'localhost' + }, + encoding: 'binary' + }, + { + status: 200, + headers: { + 'content-type': 'image/png' + } + }, + function(res, err) { + next(err, res); + } + ); + }, + function handleTileResponse(err, res) { + if (err) { + throw err; + } + test_helper.checkCache(res); + return true; + }, + function deleteTemplate(err) { + var next = this; + templateMaps.delTemplate(username, privateTableTemplate, function(/*delErr*/) { + // ignore deletion error + next(err); + }); + }, + function finish(err) { + done(err); + } + ); + + }); + + test('should return 403 when private table is accessed from non named layer', function(done) { + + var layergroup = { + version: '1.3.0', + layers: [ + { + type: 'cartodb', + options: { + sql: 'select * from test_table_private_1', + cartocss: '#layer { marker-fill: #cc3300; }', + cartocss_version: '2.3.0' + } + }, + { + type: 'named', + options: { + name: templateName + } + } + ] + }; + + Step( + function createLayergroup() { + var next = this; + assert.response(server, + { + url: '/tiles/layergroup', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 403 + }, + function(res, err) { + next(err, res); + } + ); + }, + function checkLayergroup(err, response) { + if (err) { + throw err; + } + + var parsedBody = JSON.parse(response.body); + assert.ok(parsedBody.errors[0].match(/permission denied for relation test_table_private_1/)); + + return null; + }, + function finish(err) { + done(err); + } + ); + + }); + + + suiteTeardown(function(done) { + global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false}; + templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) { + if (err) { + return done(err); + } + templateMaps.delTemplate(username, tokenAuthTemplateName, function(err) { + if (err) { + return done(err); + } + templateMaps.delTemplate(username, templateName, function(err) { + return done(err); + }); + }); + }); + }); +}); diff --git a/test/acceptance/server.js b/test/acceptance/server.js index e02991a2..e31333c9 100644 --- a/test/acceptance/server.js +++ b/test/acceptance/server.js @@ -20,10 +20,7 @@ server.setMaxListeners(0); [true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) { -global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue}; - -suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { - +suite('server:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { var redis_client = redis.createClient(global.environment.redis.port); var sqlapi_server; @@ -47,8 +44,9 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( // A couple of styles to use during testing var test_style_black_200 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:10}"; var test_style_black_210 = "#test_table{marker-fill:black;marker-line-color:red;marker-width:20}"; - + suiteSetup(function(done){ + global.environment.enabledFeatures = { cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue }; sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); }); @@ -1069,10 +1067,12 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( assert(cc, 'Missing X-Cache-Channel'); var dbname = test_database; assert.equal(cc.substring(0, dbname.length), dbname); - var jsonquery = cc.substring(dbname.length+1); - var sentquery = JSON.parse(jsonquery); - assert.equal(sentquery.api_key, qo.map_key); - assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)'); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API + var jsonquery = cc.substring(dbname.length + 1); + var sentquery = JSON.parse(jsonquery); + assert.equal(sentquery.api_key, qo.map_key); + assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$' + qo.sql + '$windshaft$)'); + } return null; }, function finish(err) { @@ -1081,6 +1081,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( ); }); + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was a post if it's using the SQL API test("passes hostname header to sqlapi", function(done){ var qo = { sql: "SELECT * from gadm4", @@ -1112,7 +1113,9 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( } ); }); + } + if (!cdbQueryTablesFromPostgresEnabledValue) { // only test if it was using the SQL API test("requests to skip cache on sqlapi error", function(done){ var qo = { sql: "SELECT g.cartodb_id, g.codineprov, t.the_geom_webmercator " @@ -1145,6 +1148,7 @@ suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function( } ); }); + } // Zoom is a special variable test("Specifying zoom level in CartoCSS does not need a 'zoom' variable in SQL output", function(done){ diff --git a/test/acceptance/templates.js b/test/acceptance/templates.js index 46c872e2..f0df7414 100644 --- a/test/acceptance/templates.js +++ b/test/acceptance/templates.js @@ -27,9 +27,8 @@ server.setMaxListeners(0); [true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) { -global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue}; - suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() { + serverOptions.channelCache = {}; var redis_client = redis.createClient(global.environment.redis.port); var sqlapi_server; @@ -37,6 +36,7 @@ suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, functio var expected_last_updated = new Date(expected_last_updated_epoch).toISOString(); suiteSetup(function(done){ + global.environment.enabledFeatures = { cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue }; sqlapi_server = new SQLAPIEmu(global.environment.sqlapi.port, done); // TODO: check redis is clean ? }); diff --git a/test/integration/mapconfig_named_layers_datasource.js b/test/integration/mapconfig_named_layers_datasource.js new file mode 100644 index 00000000..14fbc29c --- /dev/null +++ b/test/integration/mapconfig_named_layers_datasource.js @@ -0,0 +1,298 @@ +var test_helper = require('../support/test_helper'); + +var assert = require('assert'); +var RedisPool = require('redis-mpool'); +var TemplateMaps = require('../../lib/cartodb/template_maps.js'); +var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); +var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter'); + +var Step = require('step'); +var _ = require('underscore'); + +// configure redis pool instance to use in tests +var redisPool = RedisPool(global.environment.redis); +var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool })); + +var templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates +}); + +var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + +var wadusSql = 'select 1 wadusLayer, null::geometry the_geom_webmercator'; +var wadusLayer = { + type: 'cartodb', + options: { + sql: wadusSql, + cartocss: '#layer { marker-fill: black; }', + cartocss_version: '2.3.0' + } +}; + +var wadusTemplateSql = 'select 1 wadusTemplateLayer, null::geometry the_geom_webmercator'; +var wadusTemplateLayer = { + type: 'cartodb', + options: { + sql: wadusTemplateSql, + cartocss: '#layer { marker-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } +}; + +var wadusMapnikSql = 'select 1 wadusMapnikLayer, null::geometry the_geom_webmercator'; +var wadusMapnikLayer = { + type: 'mapnik', + options: { + sql: wadusMapnikSql, + cartocss: '#layer { polygon-fill: <%= polygon_color %>; }', + cartocss_version: '2.3.0' + } +}; + +var username = 'localhost'; + +var templateName = 'valid_template'; +var template = { + version: '0.0.1', + name: templateName, + auth: { + method: 'open' + }, + "placeholders": { + "color": { + "type": "css_color", + "default": "#cc3300" + } + }, + layergroup: { + layers: [ + wadusTemplateLayer + ] + } +}; + +var multipleLayersTemplateName = 'multiple_valid_template'; +var multipleLayersTemplate = { + version: '0.0.1', + name: multipleLayersTemplateName, + auth: { + method: 'token', + valid_tokens: ['valid1', 'valid2'] + }, + "placeholders": { + "polygon_color": { + "type": "css_color", + "default": "green" + }, + "color": { + "type": "css_color", + "default": "red" + } + }, + layergroup: { + layers: [ + wadusMapnikLayer, + wadusTemplateLayer + ] + } +}; + +suite('named_layers datasources', function() { + suiteSetup(function(done) { + templateMaps.addTemplate(username, template, function(err) { + if (err) { + return done(err); + } + templateMaps.addTemplate(username, multipleLayersTemplate, done); + }); + }); + + function makeNamedMapLayerConfig(layers) { + return { + version: '1.3.0', + layers: layers + }; + } + + var simpleNamedLayer = { + type: 'named', + options: { + name: templateName + } + }; + + var multipleLayersNamedLayer = { + type: 'named', + options: { + name: multipleLayersTemplateName, + auth_tokens: ['valid2'] + } + }; + + var testScenarios = [ + { + desc: 'without datasource for non-named layers', + config: makeNamedMapLayerConfig([wadusLayer]), + test: function(err, layers, datasource, done) { + assert.ok(!err); + assert.equal(layers.length, 1); + + assert.equal(layers[0].type, 'cartodb'); + assert.equal(layers[0].options.sql, wadusSql); + assert.equal(datasource.getLayerDatasource(0), undefined); + + done(); + } + }, + + { + desc: 'with datasource for the named layer but not for the normal', + config: makeNamedMapLayerConfig([wadusLayer, simpleNamedLayer]), + test: function(err, layers, datasource, done) { + assert.ok(!err); + assert.equal(layers.length, 2); + + assert.equal(layers[0].type, 'cartodb'); + assert.equal(layers[0].options.sql, wadusSql); + assert.equal(datasource.getLayerDatasource(0), undefined); + + assert.equal(layers[1].type, 'cartodb'); + assert.equal(layers[1].options.sql, wadusTemplateSql); + var layerDatasource = datasource.getLayerDatasource(1); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + done(); + } + }, + + { + desc: 'with datasource for the multiple layers in the named but not for the normal', + config: makeNamedMapLayerConfig([wadusLayer, multipleLayersNamedLayer]), + test: function(err, layers, datasource, done) { + assert.ok(!err); + assert.equal(layers.length, 3); + + assert.equal(layers[0].type, 'cartodb'); + assert.equal(layers[0].options.sql, wadusSql); + assert.equal(datasource.getLayerDatasource(0), undefined); + + assert.equal(layers[1].type, 'mapnik'); + assert.equal(layers[1].options.sql, wadusMapnikSql); + var layerDatasource = datasource.getLayerDatasource(1); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[2].type, 'cartodb'); + assert.equal(layers[2].options.sql, wadusTemplateSql); + layerDatasource = datasource.getLayerDatasource(2); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + done(); + } + }, + + { + desc: 'all with datasource because all are named', + config: makeNamedMapLayerConfig([multipleLayersNamedLayer, simpleNamedLayer]), + test: function(err, layers, datasource, done) { + assert.ok(!err); + assert.equal(layers.length, 3); + + assert.equal(layers[0].type, 'mapnik'); + assert.equal(layers[0].options.sql, wadusMapnikSql); + var layerDatasource = datasource.getLayerDatasource(0); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[1].type, 'cartodb'); + assert.equal(layers[1].options.sql, wadusTemplateSql); + layerDatasource = datasource.getLayerDatasource(1); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[2].type, 'cartodb'); + assert.equal(layers[2].options.sql, wadusTemplateSql); + layerDatasource = datasource.getLayerDatasource(2); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + done(); + } + }, + + { + desc: 'with a mix of datasource and no datasource depending if layers are named or not', + config: makeNamedMapLayerConfig([simpleNamedLayer, multipleLayersNamedLayer, wadusLayer, simpleNamedLayer, wadusLayer, multipleLayersNamedLayer]), + test: function(err, layers, datasource, done) { + + assert.ok(!err); + assert.equal(layers.length, 8); + + assert.equal(layers[0].type, 'cartodb'); + assert.equal(layers[0].options.sql, wadusTemplateSql); + var layerDatasource = datasource.getLayerDatasource(0); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[1].type, 'mapnik'); + assert.equal(layers[1].options.sql, wadusMapnikSql); + layerDatasource = datasource.getLayerDatasource(1); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[2].type, 'cartodb'); + assert.equal(layers[2].options.sql, wadusTemplateSql); + layerDatasource = datasource.getLayerDatasource(2); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[3].type, 'cartodb'); + assert.equal(layers[3].options.sql, wadusSql); + assert.equal(datasource.getLayerDatasource(3), undefined); + + assert.equal(layers[4].type, 'cartodb'); + assert.equal(layers[4].options.sql, wadusTemplateSql); + layerDatasource = datasource.getLayerDatasource(4); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[5].type, 'cartodb'); + assert.equal(layers[5].options.sql, wadusSql); + assert.equal(datasource.getLayerDatasource(5), undefined); + + assert.equal(layers[6].type, 'mapnik'); + assert.equal(layers[6].options.sql, wadusMapnikSql); + layerDatasource = datasource.getLayerDatasource(6); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + assert.equal(layers[7].type, 'cartodb'); + assert.equal(layers[7].options.sql, wadusTemplateSql); + layerDatasource = datasource.getLayerDatasource(7); + assert.notEqual(layerDatasource, undefined); + assert.ok(layerDatasource.user); + + done(); + } + } + ]; + + testScenarios.forEach(function(testScenario) { + test('should return a list of layers ' + testScenario.desc, function(done) { + mapConfigNamedLayersAdapter.getLayers(username, testScenario.config.layers, pgConnection, function(err, layers, datasource) { + testScenario.test(err, layers, datasource, done); + }); + }); + }); + + suiteTeardown(function(done) { + templateMaps.delTemplate(username, templateName, function(err) { + if (err) { + return done(err); + } + templateMaps.delTemplate(username, multipleLayersTemplateName, done); + }); + }); +}); diff --git a/test/integration/mapconfig_named_layers_expanded.js b/test/integration/mapconfig_named_layers_expanded.js new file mode 100644 index 00000000..72ab67cf --- /dev/null +++ b/test/integration/mapconfig_named_layers_expanded.js @@ -0,0 +1,312 @@ +var testHelper = require('../support/test_helper'); + +var assert = require('assert'); +var RedisPool = require('redis-mpool'); +var TemplateMaps = require('../../lib/cartodb/template_maps.js'); +var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); +var MapConfigNamedLayersAdapter = require('../../lib/cartodb/models/mapconfig_named_layers_adapter'); + +var Step = require('step'); +var _ = require('underscore'); + +suite('mapconfig_named_layers_adapter', function() { + + // configure redis pool instance to use in tests + var redisPool = RedisPool(global.environment.redis); + var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool })); + + var templateMaps = new TemplateMaps(redisPool, { + max_user_templates: global.environment.maxUserTemplates + }); + + var mapConfigNamedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + + var wadusLayer = { + type: 'cartodb', + options: { + sql: 'select 1 cartodb_id, null::geometry the_geom_webmercator', + cartocss: '#layer { marker-fill: <%= color %>; }', + cartocss_version: '2.3.0' + } + }; + + var wadusMapnikLayer = { + type: 'mapnik', + options: { + sql: 'select 1 cartodb_id, null::geometry the_geom_webmercator', + cartocss: '#layer { polygon-fill: <%= polygon_color %>; }', + cartocss_version: '2.3.0' + } + }; + + var username = 'localhost'; + + var templateName = 'valid_template'; + var template = { + version: '0.0.1', + name: templateName, + auth: { + method: 'open' + }, + "placeholders": { + "color": { + "type": "css_color", + "default": "#cc3300" + } + }, + layergroup: { + layers: [ + wadusLayer + ] + } + }; + + var tokenAuthTemplateName = 'auth_valid_template'; + var tokenAuthTemplate = { + version: '0.0.1', + name: tokenAuthTemplateName, + auth: { + method: 'token', + valid_tokens: ['valid1', 'valid2'] + }, + layergroup: { + layers: [ + wadusLayer + ] + } + }; + + var multipleLayersTemplateName = 'multiple_valid_template'; + var multipleLayersTemplate = { + version: '0.0.1', + name: multipleLayersTemplateName, + auth: { + method: 'token', + valid_tokens: ['valid1', 'valid2'] + }, + "placeholders": { + "polygon_color": { + "type": "css_color", + "default": "green" + }, + "color": { + "type": "css_color", + "default": "red" + } + }, + layergroup: { + layers: [ + wadusMapnikLayer, + wadusLayer + ] + } + }; + + var namedMapLayer = { + type: 'named', + options: { + name: templateName, + config: {}, + auth_tokens: [] + } + }; + + var nestedNamedMapTemplateName = 'nested_template'; + var nestedNamedMapTemplate = { + version: '0.0.1', + name: nestedNamedMapTemplateName, + auth: { + method: 'open' + }, + layergroup: { + layers: [ + namedMapLayer + ] + } + }; + + function makeNamedMapLayerConfig(options) { + return { + version: '1.3.0', + layers: [ + { + type: 'named', + options: options + } + ] + }; + } + + + suiteSetup(function(done) { + templateMaps.addTemplate(username, template, done); + }); + + test('should fail for named map layer with missing name', function(done) { + var missingNamedMapLayerConfig = makeNamedMapLayerConfig({ + config: {} + }); + mapConfigNamedLayersAdapter.getLayers(username, missingNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(err); + assert.ok(!layers); + assert.ok(!datasource); + assert.equal(err.message, 'Missing Named Map `name` in layer options'); + + done(); + }); + }); + + test('should fail for non-existing template name', function(done) { + var missingTemplateName = 'wadus'; + var nonExistentNamedMapLayerConfig = makeNamedMapLayerConfig({ + name: missingTemplateName + }); + mapConfigNamedLayersAdapter.getLayers(username, nonExistentNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(err); + assert.ok(!layers); + assert.ok(!datasource); + assert.equal(err.message, "Template '" + missingTemplateName + "' of user '" + username + "' not found"); + + done(); + }); + }); + + test('should fail if not properly authorized', function(done) { + templateMaps.addTemplate(username, tokenAuthTemplate, function(err) { + if (err) { + return done(err); + } + + var nonAuthTokensNamedMapLayerConfig = makeNamedMapLayerConfig({ + name: tokenAuthTemplateName + }); + mapConfigNamedLayersAdapter.getLayers(username, nonAuthTokensNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(err); + assert.ok(!layers); + assert.ok(!datasource); + assert.equal(err.message, "Unauthorized '" + tokenAuthTemplateName + "' template instantiation"); + + templateMaps.delTemplate(username, tokenAuthTemplateName, done); + }); + }); + }); + + test('should fail for nested named map layers', function(done) { + templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) { + if (err) { + return done(err); + } + + var nestedNamedMapLayerConfig = makeNamedMapLayerConfig({ + name: nestedNamedMapTemplateName + }); + mapConfigNamedLayersAdapter.getLayers(username, nestedNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(err); + assert.ok(!layers); + assert.ok(!datasource); + assert.equal(err.message, 'Nested named layers are not allowed'); + + templateMaps.delTemplate(username, nestedNamedMapTemplateName, done); + }); + }); + }); + + test('should return an expanded list of layers for a named map layer', function(done) { + var validNamedMapMapLayerConfig = makeNamedMapLayerConfig({ + name: templateName + }); + mapConfigNamedLayersAdapter.getLayers(username, validNamedMapMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(!err); + assert.ok(layers.length, 1); + assert.ok(layers[0].type, 'cartodb'); + assert.notEqual(datasource.getLayerDatasource(0), undefined); + + done(); + }); + }); + + test('should return on auth=token with valid tokens provided', function(done) { + templateMaps.addTemplate(username, tokenAuthTemplate, function(err) { + if (err) { + return done(err); + } + + var validAuthTokensNamedMapLayerConfig = makeNamedMapLayerConfig({ + name: tokenAuthTemplateName, + auth_tokens: ['valid1'] + }); + mapConfigNamedLayersAdapter.getLayers(username, validAuthTokensNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(!err); + assert.equal(layers.length, 1); + assert.notEqual(datasource.getLayerDatasource(0), undefined); + + templateMaps.delTemplate(username, tokenAuthTemplateName, done); + }); + }); + }); + + test('should return an expanded list of layers for a named map layer, multiple layers version', function(done) { + templateMaps.addTemplate(username, multipleLayersTemplate, function(err) { + if (err) { + return done(err); + } + + var multipleLayersNamedMapLayerConfig = makeNamedMapLayerConfig({ + name: multipleLayersTemplateName, + auth_tokens: ['valid2'] + }); + mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(!err); + assert.equal(layers.length, 2); + + assert.equal(layers[0].type, 'mapnik'); + assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: green; }'); + assert.notEqual(datasource.getLayerDatasource(0), undefined); + + assert.equal(layers[1].type, 'cartodb'); + assert.equal(layers[1].options.cartocss, '#layer { marker-fill: red; }'); + assert.notEqual(datasource.getLayerDatasource(1), undefined); + + templateMaps.delTemplate(username, multipleLayersTemplateName, done); + }); + }); + }); + + test('should replace template params with the given config', function(done) { + templateMaps.addTemplate(username, multipleLayersTemplate, function(err) { + if (err) { + return done(err); + } + + var color = '#cc3300', + polygonColor = '#ff9900'; + + var multipleLayersNamedMapLayerConfig = makeNamedMapLayerConfig({ + name: multipleLayersTemplateName, + config: { + polygon_color: polygonColor, + color: color + }, + auth_tokens: ['valid2'] + }); + mapConfigNamedLayersAdapter.getLayers(username, multipleLayersNamedMapLayerConfig.layers, pgConnection, function(err, layers, datasource) { + assert.ok(!err); + assert.equal(layers.length, 2); + + assert.equal(layers[0].type, 'mapnik'); + assert.equal(layers[0].options.cartocss, '#layer { polygon-fill: ' + polygonColor + '; }'); + assert.notEqual(datasource.getLayerDatasource(0), undefined); + + assert.equal(layers[1].type, 'cartodb'); + assert.equal(layers[1].options.cartocss, '#layer { marker-fill: ' + color + '; }'); + assert.notEqual(datasource.getLayerDatasource(1), undefined); + + templateMaps.delTemplate(username, multipleLayersTemplateName, done); + }); + }); + }); + + suiteTeardown(function(done) { + templateMaps.delTemplate(username, templateName, done); + }); +}); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 8ec0c08a..8185da4d 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -78,6 +78,12 @@ if test x"$PREPARE_PGSQL" = xyes; then sed "s/:TESTPASS/${TESTPASS}/" | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + psql -c "CREATE EXTENSION plpythonu;" ${TEST_DB} + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql + cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql | + psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + fi if test x"$PREPARE_REDIS" = xyes; then diff --git a/test/support/sql/CDB_QueryStatements.sql b/test/support/sql/CDB_QueryStatements.sql new file mode 100644 index 00000000..bb4d10cf --- /dev/null +++ b/test/support/sql/CDB_QueryStatements.sql @@ -0,0 +1,14 @@ +-- Return an array of statements found in the given query text +-- +-- Regexp curtesy of Hubert Lubaczewski (depesz) +-- Implemented in plpython for performance reasons +-- +CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text) +RETURNS SETOF TEXT AS $$ + import re + pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL ) + for match in pat.findall(query): + cleaned = match[0].strip() + if ( cleaned ): + yield cleaned +$$ language 'plpythonu' IMMUTABLE STRICT; diff --git a/test/support/sql/CDB_QueryTables.sql b/test/support/sql/CDB_QueryTables.sql new file mode 100644 index 00000000..cd8b51b4 --- /dev/null +++ b/test/support/sql/CDB_QueryTables.sql @@ -0,0 +1,67 @@ +-- Return an array of table names scanned by a given query +-- +-- Requires PostgreSQL 9.x+ +-- +CREATE OR REPLACE FUNCTION CDB_QueryTables(query text) +RETURNS name[] +AS $$ +DECLARE + exp XML; + tables NAME[]; + rec RECORD; + rec2 RECORD; +BEGIN + + tables := '{}'; + + FOR rec IN SELECT CDB_QueryStatements(query) q LOOP + + IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN + --RAISE WARNING 'Skipping %', rec.q; + CONTINUE; + END IF; + + BEGIN + EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp; + EXCEPTION WHEN others THEN + -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as + -- the affected table ? + RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM; + RAISE EXCEPTION '%', SQLERRM; + CONTINUE; + END; + + -- Now need to extract all values of + + -- RAISE DEBUG 'Explain: %', exp; + + FOR rec2 IN WITH + inp AS ( + SELECT + xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x, + xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s + ) + SELECT unnest(x)::name as p, unnest(s)::name as sc from inp + LOOP + -- RAISE DEBUG 'tab: %', rec2.p; + -- RAISE DEBUG 'sc: %', rec2.sc; + tables := array_append(tables, (rec2.sc || '.' || rec2.p)::name); + END LOOP; + + -- RAISE DEBUG 'Tables: %', tables; + + END LOOP; + + -- RAISE DEBUG 'Tables: %', tables; + + -- Remove duplicates and sort by name + IF array_upper(tables, 1) > 0 THEN + WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p ) + SELECT array_agg(p) from dist into tables; + END IF; + + --RAISE DEBUG 'Tables: %', tables; + + return tables; +END +$$ LANGUAGE 'plpgsql' VOLATILE STRICT; diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index 4ae82bfe..e174d6b4 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -177,3 +177,15 @@ CREATE TABLE test_table_private_1 ( INSERT INTO test_table_private_1 SELECT * from test_table; GRANT ALL ON TABLE test_table_private_1 TO :TESTUSER; + +CREATE TABLE IF NOT EXISTS + CDB_TableMetadata ( + tabname regclass not null primary key, + updated_at timestamp with time zone not null default now() + ); + +INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('test_table'::regclass, '2009-02-13T23:31:30.123Z'); +INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('test_table_private_1'::regclass, '2009-02-13T23:31:30.123Z'); + +-- GRANT SELECT ON CDB_TableMetadata TO :PUBLICUSER; +GRANT SELECT ON CDB_TableMetadata TO :TESTUSER;