diff --git a/NEWS.md b/NEWS.md index 44ce75d1..1d8676b3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,18 @@ # Changelog -## 2.22.1 +## 2.23.1 Released 2016-mm-dd +## 2.23.0 + +Released 2016-02-10 + +Improvements: +- Support for overviews + + ## 2.22.0 Released 2016-02-08 diff --git a/lib/cartodb/api/overviews_metadata_api.js b/lib/cartodb/api/overviews_metadata_api.js new file mode 100644 index 00000000..f3568f5c --- /dev/null +++ b/lib/cartodb/api/overviews_metadata_api.js @@ -0,0 +1,44 @@ +function OverviewsMetadataApi(pgQueryRunner) { + this.pgQueryRunner = pgQueryRunner; +} + +module.exports = OverviewsMetadataApi; + +// TODO: share this with QueryTablesApi? ... or maintain independence? +var affectedTableRegexCache = { + bbox: /!bbox!/g, + scale_denominator: /!scale_denominator!/g, + pixel_width: /!pixel_width!/g, + pixel_height: /!pixel_height!/g +}; + +function prepareSql(sql) { + return sql + .replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)') + .replace(affectedTableRegexCache.scale_denominator, '0') + .replace(affectedTableRegexCache.pixel_width, '1') + .replace(affectedTableRegexCache.pixel_height, '1') + ; +} + +OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) { + var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))'; + this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) { + if (err){ + var msg = err.message ? err.message : err; + callback(new Error('could not get overviews metadata: ' + msg)); + return; + } + var metadata = {}; + rows.forEach(function(row) { + var table = row.base_table; + var table_metadata = metadata[table]; + if ( !table_metadata ) { + table_metadata = metadata[table] = {}; + } + table_metadata[row.z] = { table: row.overview_table }; + }); + return callback(null, metadata); + }); + +}; diff --git a/lib/cartodb/controllers/base.js b/lib/cartodb/controllers/base.js index ac3ac11c..9945ff5a 100644 --- a/lib/cartodb/controllers/base.js +++ b/lib/cartodb/controllers/base.js @@ -239,6 +239,7 @@ module.exports.findStatusCode = findStatusCode; function statusFromErrorMessage(errMsg) { // Find an appropriate statusCode based on message + // jshint maxcomplexity:7 var statusCode = 400; if ( -1 !== errMsg.indexOf('permission denied') ) { statusCode = 403; @@ -252,6 +253,8 @@ function statusFromErrorMessage(errMsg) { else if ( -1 !== errMsg.indexOf('does not exist') ) { if ( -1 !== errMsg.indexOf(' role ') ) { statusCode = 403; // role 'xxx' does not exist + } else if ( errMsg.match(/function .* does not exist/) ) { + statusCode = 400; // invalid SQL (SQL function does not exist) } else { statusCode = 404; } diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index dad77dbd..a7613c2d 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -18,6 +18,7 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry'); var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_layergroup_provider'); +var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter'); /** * @param {AuthApi} authApi @@ -26,12 +27,14 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/create_laye * @param {MapBackend} mapBackend * @param metadataBackend * @param {QueryTablesApi} queryTablesApi + * @param {OverviewsMetadataApi} overviewsMetadataApi * @param {SurrogateKeysCache} surrogateKeysCache * @param {UserLimitsApi} userLimitsApi * @param {LayergroupAffectedTables} layergroupAffectedTables * @constructor */ -function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, queryTablesApi, +function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend, + queryTablesApi, overviewsMetadataApi, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) { BaseController.call(this, authApi, pgConnection); @@ -41,11 +44,13 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.mapBackend = mapBackend; this.metadataBackend = metadataBackend; this.queryTablesApi = queryTablesApi; + this.overviewsMetadataApi = overviewsMetadataApi; this.surrogateKeysCache = surrogateKeysCache; this.userLimitsApi = userLimitsApi; this.layergroupAffectedTables = layergroupAffectedTables; this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); + this.overviewsAdapter = new MapConfigOverviewsAdapter(this.overviewsMetadataApi); } util.inherits(MapController, BaseController); @@ -148,6 +153,22 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { } ); }, + function addOverviewsInformation(err, requestMapConfig, datasource) { + assert.ifError(err); + var next = this; + self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, + function(err, layers) { + if (err) { + return next(err); + } + + if (layers) { + requestMapConfig.layers = layers; + } + return next(null, requestMapConfig, datasource); + } + ); + }, function createLayergroup(err, requestMapConfig, datasource) { assert.ifError(err); mapConfig = new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource()); @@ -203,7 +224,23 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn ); mapConfigProvider.getMapConfig(this); }, - function createLayergroup(err, mapConfig_, rendererParams/*, context*/) { + function addOverviewsInformation(err, requestMapConfig, rendererParams/*, context*/) { + assert.ifError(err); + var next = this; + self.overviewsAdapter.getLayers(req.context.user, requestMapConfig.layers, + function(err, layers) { + if (err) { + return next(err); + } + + if (layers) { + requestMapConfig.layers = layers; + } + return next(null, requestMapConfig, rendererParams); + } + ); + }, + function createLayergroup(err, mapConfig_, rendererParams) { assert.ifError(err); mapConfig = mapConfig_; self.mapBackend.createLayergroup( diff --git a/lib/cartodb/models/mapconfig_overviews_adapter.js b/lib/cartodb/models/mapconfig_overviews_adapter.js new file mode 100644 index 00000000..70a02dc5 --- /dev/null +++ b/lib/cartodb/models/mapconfig_overviews_adapter.js @@ -0,0 +1,53 @@ +var queue = require('queue-async'); +var _ = require('underscore'); + +function MapConfigOverviewsAdapter(overviewsMetadataApi) { + this.overviewsMetadataApi = overviewsMetadataApi; +} + +module.exports = MapConfigOverviewsAdapter; + +MapConfigOverviewsAdapter.prototype.getLayers = function(username, layers, callback) { + var self = this; + + if (!layers || layers.length === 0) { + return callback(null, layers); + } + + var augmentLayersQueue = queue(layers.length); + + function augmentLayer(layer, done) { + if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) { + return done(null, layer); + } + self.overviewsMetadataApi.getOverviewsMetadata(username, layer.options.sql, function(err, metadata){ + if (err) { + done(err, layer); + } else { + if ( !_.isEmpty(metadata) ) { + layer = _.extend({}, layer); + layer.options = _.extend({}, layer.options, { query_rewrite_data: { overviews: metadata } }); + } + done(null, layer); + } + }); + } + + function layersAugmentQueueFinish(err, layers) { + if (err) { + return callback(err); + } + + if (!layers || layers.length === 0) { + return callback(new Error('Missing layers array from layergroup config')); + } + + return callback(null, layers); + } + + layers.forEach(function(layer) { + augmentLayersQueue.defer(augmentLayer, layer); + }); + augmentLayersQueue.awaitAll(layersAugmentQueueFinish); + +}; diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index f4205676..9f9d9f69 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -20,6 +20,7 @@ var mapnik = windshaft.mapnik; var TemplateMaps = require('./backends/template_maps.js'); var QueryTablesApi = require('./api/query_tables_api'); +var OverviewsMetadataApi = require('./api/overviews_metadata_api'); var UserLimitsApi = require('./api/user_limits_api'); var AuthApi = require('./api/auth_api'); var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables'); @@ -52,6 +53,7 @@ module.exports = function(serverOptions) { var pgConnection = new PgConnection(metadataBackend); var pgQueryRunner = new PgQueryRunner(pgConnection); var queryTablesApi = new QueryTablesApi(pgQueryRunner); + var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); var userLimitsApi = new UserLimitsApi(metadataBackend, { limits: { cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false, @@ -175,6 +177,7 @@ module.exports = function(serverOptions) { mapBackend, metadataBackend, queryTablesApi, + overviewsMetadataApi, surrogateKeysCache, userLimitsApi, layergroupAffectedTablesCache diff --git a/lib/cartodb/server_options.js b/lib/cartodb/server_options.js index 2a859d2e..f1a70889 100644 --- a/lib/cartodb/server_options.js +++ b/lib/cartodb/server_options.js @@ -1,5 +1,10 @@ var os = require('os'); var _ = require('underscore'); +var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter'); + +var overviewsQueryRewriter = new OverviewsQueryRewriter({ + zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' +}); var rendererConfig = _.defaults(global.environment.renderer || {}, { cache_ttl: 60000, // milliseconds @@ -15,6 +20,8 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, { http: {} }); +rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter; + // Perform keyword substitution in statsd // See https://github.com/CartoDB/Windshaft-cartodb/issues/153 if ( global.environment.statsd ) { diff --git a/lib/cartodb/utils/overviews_query_rewriter.js b/lib/cartodb/utils/overviews_query_rewriter.js new file mode 100644 index 00000000..0b6c592a --- /dev/null +++ b/lib/cartodb/utils/overviews_query_rewriter.js @@ -0,0 +1,182 @@ +var TableNameParser = require('./table_name_parser'); + +function OverviewsQueryRewriter(options) { + + this.options = options; +} + +module.exports = OverviewsQueryRewriter; + +// TODO: some names are introudced in the queries, and the +// '_vovw_' (for vector overviews) is used in them, but no check +// is performed for conflicts with existing identifiers in the query. + +// Build UNION expression to replace table, using overviews metadata +// overviews metadata: { 1: 'table_ov1', ... } +// assume table and overview names include schema if necessary and are quoted as needed +function overviews_view_for_table(table, overviews_metadata, indent) { + var condition, i, len, ov_table, overview_layers, selects, z_hi, z_lo; + var parsed_table = TableNameParser.parse(table); + + var sorted_overviews = []; // [[1, 'table_ov1'], ...] + + indent = indent || ' '; + for (var z in overviews_metadata) { + if (overviews_metadata.hasOwnProperty(z)) { + sorted_overviews.push([z, overviews_metadata[z].table]); + } + } + sorted_overviews.sort(function(a, b){ return a[0]-b[0]; }); + + overview_layers = []; + z_lo = null; + for (i = 0, len = sorted_overviews.length; i < len; i++) { + z_hi = parseInt(sorted_overviews[i][0]); + ov_table = sorted_overviews[i][1]; + overview_layers.push([overview_z_condition(z_lo, z_hi), ov_table]); + z_lo = z_hi; + } + overview_layers.push(["_vovw_z > " + z_lo, table]); + + selects = overview_layers.map(function(condition_table) { + condition = condition_table[0]; + ov_table = TableNameParser.parse(condition_table[1]); + ov_table.schema = ov_table.schema || parsed_table.schema; + var ov_identifier = TableNameParser.table_identifier(ov_table); + return indent + "SELECT * FROM " + ov_identifier + ", _vovw_scale WHERE " + condition; + }); + + return selects.join("\n"+indent+"UNION ALL\n"); +} + +function overview_z_condition(z_lo, z_hi) { + if (z_lo !== null) { + if (z_lo === z_hi - 1) { + return "_vovw_z = " + z_hi; + } else { + return "_vovw_z > " + z_lo + " AND _vovw_z <= " + z_hi; + } + } else { + if (z_hi === 0) { + return "_vovw_z = " + z_hi; + } else { + return "_vovw_z <= " + z_hi; + } + } +} + +// name to be used for the view of the table using overviews +function overviews_view_name(table) { + var parsed_table = TableNameParser.parse(table); + parsed_table.table = '_vovw_' + parsed_table.table; + parsed_table.schema = null; + return TableNameParser.table_identifier(parsed_table); +} + +// replace a table name in a query by anoter name +function replace_table_in_query(sql, old_table_name, new_table_name) { + var old_table = TableNameParser.parse(old_table_name); + var new_table = TableNameParser.parse(new_table_name); + var old_table_ident = TableNameParser.table_identifier(old_table); + var new_table_ident = TableNameParser.table_identifier(new_table); + + // text that will be substituted by the table name pattern + var replacement = new_table_ident; + + // regular expression prefix (beginning) to match a table name + function pattern_prefix(schema, identifier) { + if ( schema ) { + // to match a table name including schema prefix + // name should not be part of another name, so we require + // to start a at a word boundary + if ( identifier[0] !== '"' ) { + return '\\b'; + } else { + return ''; + } + } else { + // to match a table name without schema + // name should not begin right after a dot (i.e. have a explicit schema) + // nor be part of another name + // since the pattern matches the first character of the table + // it must be put back in the replacement text + replacement = '$01'+replacement; + return '([^\.a-z0-9_]|^)'; + } + } + + // regular expression suffix (ending) to match a table name + function pattern_suffix(identifier) { + // name shouldn't be the prefix of a longer name + if ( identifier[identifier.length-1] !== '"' ) { + return '\\b'; + } else { + return ''; + } + } + + // regular expression to match a table name + var regexp = pattern_prefix(old_table.schema, old_table_ident) + + old_table_ident + + pattern_suffix(old_table_ident); + + // replace all occurrences of the table pattern + return sql.replace(new RegExp(regexp, 'g'), replacement); +} + +function overviews_query(query, overviews, zoom_level_expression) { + var replaced_query = query; + var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )"; + for ( var table in overviews ) { + if (overviews.hasOwnProperty(table)) { + var table_overviews = overviews[table]; + var table_view = overviews_view_name(table); + replaced_query = replace_table_in_query(replaced_query, table, table_view); + sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )"; + } + } + if ( replaced_query !== query ) { + sql += "\n"; + sql += replaced_query; + } else { + sql = query; + } + return sql; +} + +// Transform an SQL query so that it uses overviews. +// overviews contains metadata about the overviews to be used: +// { 'table-name': {1: { table: 'overview-table-1' }, ... }, ... } +// +// For a given query `SELECT * FROM table`, if any of tables in it +// has overviews as defined by the provided metadat, the query will +// be transform into something similar to this: +// +// WITH _vovw_scale AS ( ... ), -- define scale level +// WITH _vovw_table AS ( ... ), -- define union of overviews and base table +// SELECT * FROM _vovw_table -- query with table replaced by _vovw_table +// +// This transformation can in principle be applied to arbitrary queries +// (except for the case of queries that include the name of tables with +// overviews inside text literals: at the current table name substitution +// doesnn't prevent substitution inside literals). +// But the transformation will currently only be applied to simple queries +// of the form detected by the overviews_supported_query function. +OverviewsQueryRewriter.prototype.query = function(query, data) { + var overviews = this.overviews_metadata(data); + if ( !overviews || !this.is_supported_query(query)) { + return query; + } + var zoom_level_expression = this.options.zoom_level || '0'; + return overviews_query(query, overviews, zoom_level_expression); +}; + +OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { + return !!sql.match( + /^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i + ); +}; + +OverviewsQueryRewriter.prototype.overviews_metadata = function(data) { + return data && data.overviews; +}; diff --git a/lib/cartodb/utils/table_name_parser.js b/lib/cartodb/utils/table_name_parser.js new file mode 100644 index 00000000..f71c3f34 --- /dev/null +++ b/lib/cartodb/utils/table_name_parser.js @@ -0,0 +1,106 @@ +// Quote an PostgreSQL identifier if ncecessary +function quote_identifier_if_needed(txt) { + if ( txt && !txt.match(/^[a-z_][a-z_0-9]*$/)) { + return '"' + txt.replace(/\"/g, '""') + '"'; + } else { + return txt; + } +} + +// Parse PostgreSQL table name (possibly quoted and with optional schema).+ +// Returns { schema: 'schema_name', table: 'table_name' } +function parse_table_name(table) { + + function split_as_quoted_parts(table_name) { + // parse table into 'parts' that may be quoted, each part + // in the parts array being an object { part: 'text', quoted: false/true } + var parts = []; + var splitted = table_name.split(/\"/); + for (var i=0; i 0 && i < splitted.length-1 ) { + i++; + parts[parts.length - 1].part += '"' + splitted[i]; + } + } + else { + var is_quoted = (i > 0 && splitted[i-1] === '') || + (i < splitted.length - 1 && splitted[i+1] === ''); + parts.push({ part: splitted[i], quoted: is_quoted }); + } + } + return parts; + } + + var parts = split_as_quoted_parts(table); + + function split_single_part(part) { + var schema_part = null; + var table_part = null; + if ( part.quoted ) { + table_part = part.part; + } else { + var parts = part.part.split('.'); + if ( parts.length === 1 ) { + schema_part = null; + table_part = parts[0]; + } else if ( parts.length === 2 ) { + schema_part = parts[0]; + table_part = parts[1]; + } // else invalid table name + } + return { + schema: schema_part, + table: table_part + }; + } + + function split_two_parts(part1, part2) { + var schema_part = null; + var table_part = null; + if ( part1.quoted && !part2.quoted ) { + if ( part2.part[0] === '.' ) { + schema_part = part1.part; + table_part = part2.part.slice(1); + } // else invalid table name (missing dot) + } else if ( !part1.quoted && part2.quoted ) { + if ( part1.part[part1.part.length - 1] === '.' ) { + schema_part = part1.part.slice(0, -1); + table_part = part2.part; + } // else invalid table name (missing dot) + } // else invalid table name (missing dot) + return { + schema: schema_part, + table: table_part + }; + } + + if ( parts.length === 1 ) { + return split_single_part(parts[0]); + } else if ( parts.length === 2 ) { + return split_two_parts(parts[0], parts[1]); + } else if ( parts.length === 3 && parts[1].part === '.' ) { + return { + schema: parts[0].part, + table: parts[2].part + }; + } // else invalid table name +} + +function table_identifier(parsed_name) { + if ( parsed_name && parsed_name.table ) { + if ( parsed_name.schema ) { + return quote_identifier_if_needed(parsed_name.schema) + '.' + quote_identifier_if_needed(parsed_name.table); + } else { + return quote_identifier_if_needed(parsed_name.table); + } + } else { + return null; + } +} + +module.exports = { + parse: parse_table_name, + quote: quote_identifier_if_needed, + table_identifier: table_identifier +}; diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index b545d61d..70e7c50e 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -1,6 +1,6 @@ { "name": "windshaft-cartodb", - "version": "2.22.0", + "version": "2.23.1", "dependencies": { "body-parser": { "version": "1.14.2", @@ -85,7 +85,7 @@ }, "mime-types": { "version": "2.1.9", - "from": "mime-types@>=2.1.2 <2.2.0", + "from": "mime-types@>=2.1.9 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", "dependencies": { "mime-db": { @@ -193,7 +193,7 @@ "dependencies": { "mime-types": { "version": "2.1.9", - "from": "mime-types@>=2.1.9 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", "dependencies": { "mime-db": { @@ -382,7 +382,7 @@ }, "mime-types": { "version": "2.1.9", - "from": "mime-types@>=2.1.9 <2.2.0", + "from": "mime-types@>=2.1.6 <2.2.0", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.9.tgz", "dependencies": { "mime-db": { @@ -536,7 +536,7 @@ }, "inherits": { "version": "2.0.1", - "from": "inherits@>=2.0.0 <3.0.0", + "from": "inherits@>=2.0.1 <2.1.0", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz" }, "isarray": { @@ -830,9 +830,8 @@ "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz" }, "windshaft": { - "version": "1.8.3", - "from": "windshaft@1.8.3", - "resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.8.3.tgz", + "version": "1.9.0", + "from": "windshaft@1.9.0", "dependencies": { "mapnik": { "version": "1.4.15-cdb6", diff --git a/package.json b/package.json index bb9c996a..e31e26ff 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "windshaft-cartodb", - "version": "2.22.1", + "version": "2.23.1", "description": "A map tile server for CartoDB", "keywords": [ "cartodb" @@ -26,7 +26,7 @@ "node-statsd": "~0.0.7", "underscore" : "~1.6.0", "dot": "~1.0.2", - "windshaft": "1.8.3", + "windshaft": "1.9.0", "step": "~0.0.6", "queue-async": "~1.0.7", "request": "~2.62.0", diff --git a/test/acceptance/multilayer.js b/test/acceptance/multilayer.js index 2611bb7f..d4e06e52 100644 --- a/test/acceptance/multilayer.js +++ b/test/acceptance/multilayer.js @@ -576,7 +576,7 @@ describe(suiteName, function() { version: '1.0.0', layers: [ { options: { - sql: 'select 1 as cartodb_id, !pixel_height! as h' + + sql: 'select 1 as cartodb_id, !pixel_height! as h,' + 'ST_Buffer(!bbox!, -32*greatest(!pixel_width!,!pixel_height!)) as the_geom_webmercator', cartocss: '#layer { polygon-fit:red; }', cartocss_version: '2.0.1' @@ -617,7 +617,7 @@ describe(suiteName, function() { headers: {host: 'localhost', 'Content-Type': 'application/json' }, data: JSON.stringify(layergroup) }, {}, function(res) { - assert.equal(res.statusCode, 404, res.statusCode + ": " + res.body); + assert.equal(res.statusCode, 400, res.statusCode + ": " + res.body); var parsed = JSON.parse(res.body); var msg = parsed.errors[0]; assert.ok(msg.match(/bogus.*exist/), msg); @@ -1319,6 +1319,39 @@ describe(suiteName, function() { ); }); + it('should response to empty layers mapconfig', function(done) { + var layergroup = { + layers: [] + }; + + assert.response( + server, + { + url: '/api/v1/map', + method: 'POST', + headers: { + host: 'localhost', + 'Content-Type': 'application/json' + }, + data: JSON.stringify(layergroup) + }, + { + status: 200 + }, + function(res, err) { + assert.ok(!err); + + var parsedBody = JSON.parse(res.body); + assert.ok(parsedBody.layergroupid); + + keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + + done(); + } + ); + }); + }); }); diff --git a/test/acceptance/multilayer_server.js b/test/acceptance/multilayer_server.js index e43070f1..544aea6c 100644 --- a/test/acceptance/multilayer_server.js +++ b/test/acceptance/multilayer_server.js @@ -332,13 +332,9 @@ describe('tests from old api translated to multilayer', function() { assert.ok(!res.headers.hasOwnProperty('x-cache-channel')); - // TODO when affected tables query makes the request to fail layergroup should be removed - keysToDelete['map_cfg|4fb7bd7008322ce66f22d20aebba1ab0'] = 0; - keysToDelete['user:localhost:mapviews:global'] = 5; - var parsed = JSON.parse(res.body); assert.deepEqual(parsed, { - errors: ["Error: could not fetch affected tables or last updated time: fake error message"] + errors: ["could not get overviews metadata: fake error message"] }); done(); diff --git a/test/acceptance/overviews_metadata.js b/test/acceptance/overviews_metadata.js new file mode 100644 index 00000000..0fa99bf9 --- /dev/null +++ b/test/acceptance/overviews_metadata.js @@ -0,0 +1,110 @@ +var test_helper = require('../support/test_helper'); + +var assert = require('../support/assert'); +var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server'); +var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); +var server = new CartodbWindshaft(serverOptions); + +var LayergroupToken = require('../../lib/cartodb/models/layergroup_token'); + +var RedisPool = require('redis-mpool'); + +var step = require('step'); + +var windshaft = require('windshaft'); + + +describe('overviews metadata', function() { + // configure redis pool instance to use in tests + var redisPool = new RedisPool(global.environment.redis); + + var overviews_layer = { + type: 'cartodb', + options: { + sql: 'SELECT * FROM test_table_overviews', + cartocss: '#layer { marker-fill: black; }', + cartocss_version: '2.3.0' + } + }; + + var non_overviews_layer = { + type: 'cartodb', + options: { + sql: 'SELECT * FROM test_table', + cartocss: '#layer { marker-fill: black; }', + cartocss_version: '2.3.0' + } + }; + + var keysToDelete; + + beforeEach(function() { + keysToDelete = {}; + }); + + afterEach(function(done) { + test_helper.deleteRedisKeys(keysToDelete, done); + }); + + it("layers with and without overviews", function(done) { + + var layergroup = { + version: '1.0.0', + layers: [overviews_layer, non_overviews_layer] + }; + + var layergroup_url = '/api/v1/map'; + + var expected_token; + step( + function do_post() + { + var next = this; + assert.response(server, { + url: layergroup_url, + method: 'POST', + headers: {host: 'localhost', 'Content-Type': 'application/json' }, + data: JSON.stringify(layergroup) + }, {}, function(res) { + assert.equal(res.statusCode, 200, res.body); + var parsedBody = JSON.parse(res.body); + assert.equal(res.headers['x-layergroup-id'], parsedBody.layergroupid); + expected_token = parsedBody.layergroupid; + next(null, res); + }); + }, + function do_get_mapconfig(err) + { + assert.ifError(err); + var next = this; + + var mapStore = new windshaft.storage.MapStore({ + pool: redisPool, + expire_time: 500000 + }); + mapStore.load(LayergroupToken.parse(expected_token).token, function(err, mapConfig) { + assert.ifError(err); + assert.deepEqual(non_overviews_layer, mapConfig._cfg.layers[1]); + assert.equal(mapConfig._cfg.layers[0].type, 'cartodb'); + assert.ok(mapConfig._cfg.layers[0].options.query_rewrite_data); + var expected_data = { + overviews: { + test_table_overviews: { + 1: { table: '_vovw_1_test_table_overviews' }, + 2: { table: '_vovw_2_test_table_overviews' } + } + } + }; + assert.deepEqual(mapConfig._cfg.layers[0].options.query_rewrite_data, expected_data); + }); + + next(err); + }, + function finish(err) { + keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0; + keysToDelete['user:localhost:mapviews:global'] = 5; + done(err); + } + ); + }); +}); diff --git a/test/acceptance/overviews_queries.js b/test/acceptance/overviews_queries.js new file mode 100644 index 00000000..8024ba72 --- /dev/null +++ b/test/acceptance/overviews_queries.js @@ -0,0 +1,72 @@ +var testHelper = require('../support/test_helper'); +var assert = require('../support/assert'); + +var cartodbServer = require('../../lib/cartodb/server'); +var ServerOptions = require('./ported/support/ported_server_options'); +var testClient = require('./ported/support/test_client'); +var BaseController = require('../../lib/cartodb/controllers/base'); + +describe('overviews_queries', function() { + + var server = cartodbServer(ServerOptions); + server.setMaxListeners(0); + + var IMAGE_EQUALS_TOLERANCE_PER_MIL = 2; + + var req2paramsFn; + before(function() { + req2paramsFn = BaseController.prototype.req2params; + BaseController.prototype.req2params = ServerOptions.req2params; + }); + + after(function() { + BaseController.prototype.req2params = req2paramsFn; + + testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir); + }); + + function imageCompareFn(fixture, done) { + return function(err, tile) { + if (err) { + return done(err); + } + assert.imageEqualsFile(tile.body, './test/fixtures/' + fixture, IMAGE_EQUALS_TOLERANCE_PER_MIL, done); + }; + } + + it("should not use overview for tables without overviews", function(done){ + testClient.getTile(testClient.defaultTableMapConfig('test_table'), 1, 0, 0, + imageCompareFn('test_table_1_0_0.png', done) + ); + }); + + it("should not use overview for tables without overviews at z=2", function(done){ + testClient.getTile(testClient.defaultTableMapConfig('test_table'), 2, 1, 1, + imageCompareFn('test_table_2_1_1.png', done) + ); + }); + + it("should not use overview for tables without overviews at z=2", function(done){ + testClient.getTile(testClient.defaultTableMapConfig('test_table'), 3, 3, 3, + imageCompareFn('test_table_3_3_3.png', done) + ); + }); + + it("should use overview for zoom level 1", function(done){ + testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 1, 0, 0, + imageCompareFn('_vovw_1_test_table_1_0_0.png', done) + ); + }); + + it("should use overview for zoom level 1", function(done){ + testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 2, 1, 1, + imageCompareFn('_vovw_2_test_table_2_1_1.png', done) + ); + }); + + it("should not use overview for zoom level 3", function(done){ + testClient.getTile(testClient.defaultTableMapConfig('test_table_overviews'), 3, 3, 3, + imageCompareFn('test_table_3_3_3.png', done) + ); + }); +}); diff --git a/test/acceptance/ported/support/ported_server_options.js b/test/acceptance/ported/support/ported_server_options.js index 7894c0a9..1cc5331f 100644 --- a/test/acceptance/ported/support/ported_server_options.js +++ b/test/acceptance/ported/support/ported_server_options.js @@ -2,6 +2,10 @@ var _ = require('underscore'); var serverOptions = require('../../../../lib/cartodb/server_options'); var LayergroupToken = require('../../../../lib/cartodb/models/layergroup_token'); var mapnik = require('windshaft').mapnik; +var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter'); +var overviewsQueryRewriter = new OverviewsQueryRewriter({ + zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)' +}); module.exports = _.extend({}, serverOptions, { base_url: '/database/:dbname/table/:table', @@ -26,7 +30,8 @@ module.exports = _.extend({}, serverOptions, { limits: { render: 0, cacheOnTimeout: true - } + }, + queryRewriter: overviewsQueryRewriter }, http: { timeout: 5000, diff --git a/test/fixtures/_vovw_1_test_table_1_0_0.png b/test/fixtures/_vovw_1_test_table_1_0_0.png new file mode 100644 index 00000000..1dd602df Binary files /dev/null and b/test/fixtures/_vovw_1_test_table_1_0_0.png differ diff --git a/test/fixtures/_vovw_2_test_table_2_1_1.png b/test/fixtures/_vovw_2_test_table_2_1_1.png new file mode 100644 index 00000000..23aadb78 Binary files /dev/null and b/test/fixtures/_vovw_2_test_table_2_1_1.png differ diff --git a/test/fixtures/test_table_1_0_0.png b/test/fixtures/test_table_1_0_0.png new file mode 100644 index 00000000..53028337 Binary files /dev/null and b/test/fixtures/test_table_1_0_0.png differ diff --git a/test/fixtures/test_table_2_1_1.png b/test/fixtures/test_table_2_1_1.png new file mode 100644 index 00000000..361c52b9 Binary files /dev/null and b/test/fixtures/test_table_2_1_1.png differ diff --git a/test/fixtures/test_table_3_3_3.png b/test/fixtures/test_table_3_3_3.png new file mode 100644 index 00000000..35a0059e Binary files /dev/null and b/test/fixtures/test_table_3_3_3.png differ diff --git a/test/integration/mapconfig_overviews_adapter.js b/test/integration/mapconfig_overviews_adapter.js new file mode 100644 index 00000000..079575a7 --- /dev/null +++ b/test/integration/mapconfig_overviews_adapter.js @@ -0,0 +1,87 @@ +require('../support/test_helper'); + +var assert = require('assert'); +var RedisPool = require('redis-mpool'); +var cartodbRedis = require('cartodb-redis'); +var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection'); +var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); +var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); +var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig_overviews_adapter'); + +// configure redis pool instance to use in tests +var redisPool = new RedisPool(global.environment.redis); +var pgConnection = new PgConnection(require('cartodb-redis')({ pool: redisPool })); + +var redisPool = new RedisPool(global.environment.redis); +var metadataBackend = cartodbRedis({pool: redisPool}); +var pgConnection = new PgConnection(metadataBackend); +var pgQueryRunner = new PgQueryRunner(pgConnection); +var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + + +var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi); + +describe('MapConfigOverviewsAdapter', function() { + + it('should not modify layers for which no overviews are available', function(done) { + var sql = 'SELECT * FROM test_table'; + var cartocss = '#layer { marker-fill: black; }'; + var cartocss_version = '2.3.0'; + var layer_without_overviews = { + type: 'cartodb', + options: { + sql: sql, + cartocss: cartocss, + cartocss_version: cartocss_version + } + }; + + mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) { + assert.ok(!err); + assert.equal(layers.length, 1); + assert.equal(layers[0].type, 'cartodb'); + assert.equal(layers[0].options.sql, sql); + assert.equal(layers[0].options.cartocss, cartocss); + assert.equal(layers[0].options.cartocss_version, cartocss_version); + assert.equal(layers[0].options.overviews, undefined); + done(); + }); + }); +}); + +describe('MapConfigOverviewsAdapter', function() { + + it('should add overviews metadata for layers using tables with overviews', function(done) { + var sql = 'SELECT * FROM test_table_overviews'; + var cartocss = '#layer { marker-fill: black; }'; + var cartocss_version = '2.3.0'; + var layer_without_overviews = { + type: 'cartodb', + options: { + sql: sql, + cartocss: cartocss, + cartocss_version: cartocss_version + } + }; + + mapConfigOverviewsAdapter.getLayers('localhost', [layer_without_overviews], function(err, layers) { + assert.ok(!err); + assert.equal(layers.length, 1); + assert.equal(layers[0].type, 'cartodb'); + assert.equal(layers[0].options.sql, sql); + assert.equal(layers[0].options.cartocss, cartocss); + assert.equal(layers[0].options.cartocss_version, cartocss_version); + assert.ok(layers[0].options.query_rewrite_data); + var expected_data = { + overviews: { + test_table_overviews: { + 1: { table: '_vovw_1_test_table_overviews' }, + 2: { table: '_vovw_2_test_table_overviews' } + } + } + }; + assert.deepEqual(layers[0].options.query_rewrite_data, expected_data); + done(); + }); + }); +}); diff --git a/test/integration/overviews-metadata-api.js b/test/integration/overviews-metadata-api.js new file mode 100644 index 00000000..6c054d1e --- /dev/null +++ b/test/integration/overviews-metadata-api.js @@ -0,0 +1,54 @@ +require('../support/test_helper'); + +var assert = require('assert'); + +var RedisPool = require('redis-mpool'); +var cartodbRedis = require('cartodb-redis'); + +var PgConnection = require('../../lib/cartodb/backends/pg_connection'); +var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner'); +var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api'); +var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api'); + + +describe('OverviewsMetadataApi', function() { + + var queryTablesApi, overviewsMetadataApi; + + before(function() { + var redisPool = new RedisPool(global.environment.redis); + var metadataBackend = cartodbRedis({pool: redisPool}); + var pgConnection = new PgConnection(metadataBackend); + var pgQueryRunner = new PgQueryRunner(pgConnection); + queryTablesApi = new QueryTablesApi(pgQueryRunner); + overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner); + }); + + it('should return an empty relation for tables that have no overviews', function(done) { + var query = 'select * from test_table'; + overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + assert.ok(!err, err); + + assert.deepEqual(result, {}); + + done(); + }); + }); + + it('should return overviews metadata', function(done) { + var query = 'select * from test_table_overviews'; + overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) { + assert.ok(!err, err); + + assert.deepEqual(result, { + 'test_table_overviews': { + 1: { table: '_vovw_1_test_table_overviews' }, + 2: { table: '_vovw_2_test_table_overviews' } + } + }); + + done(); + }); + }); + +}); diff --git a/test/support/prepare_db.sh b/test/support/prepare_db.sh index 8ade522f..10b34830 100755 --- a/test/support/prepare_db.sh +++ b/test/support/prepare_db.sh @@ -79,19 +79,12 @@ if test x"$PREPARE_PGSQL" = xyes; then psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB} - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_TableMetadata.sql -o sql/CDB_TableMetadata.sql - curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/CDB_ForeignTable.sql -o sql/CDB_ForeignTable.sql - cat sql/CDB_QueryStatements.sql | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - cat sql/CDB_QueryTables.sql | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - cat sql/CDB_TableMetadata.sql| - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 - cat sql/CDB_ForeignTable.sql|sed -e 's/cartodb\./public./g' \ - -e "s/''cartodb''/''public''/g" | - psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews + do + curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql + cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \ + | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1 + done fi diff --git a/test/support/sql/CDB_QueryStatements.sql b/test/support/sql/CDB_QueryStatements.sql deleted file mode 100644 index bb4d10cf..00000000 --- a/test/support/sql/CDB_QueryStatements.sql +++ /dev/null @@ -1,14 +0,0 @@ --- Return an array of statements found in the given query text --- --- Regexp curtesy of Hubert Lubaczewski (depesz) --- Implemented in plpython for performance reasons --- -CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text) -RETURNS SETOF TEXT AS $$ - import re - pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL ) - for match in pat.findall(query): - cleaned = match[0].strip() - if ( cleaned ): - yield cleaned -$$ language 'plpythonu' IMMUTABLE STRICT; diff --git a/test/support/sql/CDB_QueryTables.sql b/test/support/sql/CDB_QueryTables.sql deleted file mode 100644 index c7cfa64b..00000000 --- a/test/support/sql/CDB_QueryTables.sql +++ /dev/null @@ -1,78 +0,0 @@ --- Return an array of table names scanned by a given query --- --- Requires PostgreSQL 9.x+ --- -CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text) -RETURNS text[] -AS $$ -DECLARE - exp XML; - tables text[]; - rec RECORD; - rec2 RECORD; -BEGIN - - tables := '{}'; - - FOR rec IN SELECT CDB_QueryStatements(query) q LOOP - - IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN - --RAISE WARNING 'Skipping %', rec.q; - CONTINUE; - END IF; - - BEGIN - EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp; - EXCEPTION WHEN others THEN - -- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as - -- the affected table ? - RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM; - RAISE EXCEPTION '%', SQLERRM; - CONTINUE; - END; - - -- Now need to extract all values of - - -- RAISE DEBUG 'Explain: %', exp; - - FOR rec2 IN WITH - inp AS ( - SELECT - xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x, - xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s - ) - SELECT unnest(x)::text as p, unnest(s)::text as sc from inp - LOOP - -- RAISE DEBUG 'tab: %', rec2.p; - -- RAISE DEBUG 'sc: %', rec2.sc; - tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p))); - END LOOP; - - -- RAISE DEBUG 'Tables: %', tables; - - END LOOP; - - -- RAISE DEBUG 'Tables: %', tables; - - -- Remove duplicates and sort by name - IF array_upper(tables, 1) > 0 THEN - WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p ) - SELECT array_agg(p) from dist into tables; - END IF; - - --RAISE DEBUG 'Tables: %', tables; - - return tables; -END -$$ LANGUAGE 'plpgsql' VOLATILE STRICT; - - --- Keep CDB_QueryTables with same signature for backwards compatibility. --- It should probably be removed in the future. -CREATE OR REPLACE FUNCTION CDB_QueryTables(query text) -RETURNS name[] -AS $$ -BEGIN - RETURN CDB_QueryTablesText(query)::name[]; -END -$$ LANGUAGE 'plpgsql' VOLATILE STRICT; diff --git a/test/support/sql/windshaft.test.sql b/test/support/sql/windshaft.test.sql index d72c748e..8a01903b 100644 --- a/test/support/sql/windshaft.test.sql +++ b/test/support/sql/windshaft.test.sql @@ -238,3 +238,97 @@ CREATE INDEX test_big_poly_the_geom_webmercator_idx ON test_big_poly USING gist GRANT ALL ON TABLE test_big_poly TO :TESTUSER; GRANT SELECT ON TABLE test_big_poly TO :PUBLICUSER; + +-- table with overviews + +CREATE TABLE test_table_overviews ( + updated_at timestamp without time zone DEFAULT now(), + created_at timestamp without time zone DEFAULT now(), + cartodb_id integer NOT NULL, + name character varying, + address character varying, + the_geom geometry, + the_geom_webmercator geometry, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE test_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER; + +CREATE SEQUENCE test_table_overviews_cartodb_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE test_table_overviews_cartodb_id_seq OWNED BY test_table_overviews.cartodb_id; + +SELECT pg_catalog.setval('test_table_overviews_cartodb_id_seq', 60, true); + +ALTER TABLE test_table_overviews ALTER COLUMN cartodb_id SET DEFAULT nextval('test_table_overviews_cartodb_id_seq'::regclass); + +INSERT INTO test_table_overviews VALUES +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241'); + +ALTER TABLE ONLY test_table_overviews ADD CONSTRAINT test_table_overviews_pkey PRIMARY KEY (cartodb_id); + +CREATE INDEX test_table_overviews_the_geom_idx ON test_table_overviews USING gist (the_geom); +CREATE INDEX test_table_overviews_the_geom_webmercator_idx ON test_table_overviews USING gist (the_geom_webmercator); + +GRANT ALL ON TABLE test_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE test_table_overviews TO :PUBLICUSER; + +CREATE TABLE _vovw_1_test_table_overviews ( + updated_at timestamp without time zone DEFAULT now(), + created_at timestamp without time zone DEFAULT now(), + cartodb_id integer NOT NULL, + name character varying, + address character varying, + the_geom geometry, + the_geom_webmercator geometry, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE _vovw_1_test_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE _vovw_1_test_table_overviews TO :PUBLICUSER; + +CREATE TABLE _vovw_2_test_table_overviews ( + updated_at timestamp without time zone DEFAULT now(), + created_at timestamp without time zone DEFAULT now(), + cartodb_id integer NOT NULL, + name character varying, + address character varying, + the_geom geometry, + the_geom_webmercator geometry, + CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)), + CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)), + CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))), + CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))), + CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)), + CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857)) +); + +GRANT ALL ON TABLE _vovw_2_test_table_overviews TO :TESTUSER; +GRANT SELECT ON TABLE _vovw_2_test_table_overviews TO :PUBLICUSER; + +INSERT INTO _vovw_2_test_table_overviews VALUES +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'), +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E610000000000000009431C026043C75E7224340', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'); + +INSERT INTO _vovw_1_test_table_overviews VALUES +('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E610000000000000000020C00000000000004440', '0101000020110F000076491621312319C122D4663F1DCC5241'); diff --git a/test/unit/cartodb/overviews_query_rewriter.js b/test/unit/cartodb/overviews_query_rewriter.js new file mode 100644 index 00000000..9adbe031 --- /dev/null +++ b/test/unit/cartodb/overviews_query_rewriter.js @@ -0,0 +1,429 @@ +require('../../support/test_helper'); + +var assert = require('assert'); +var OverviewsQueryRewriter = require('../../../lib/cartodb/utils/overviews_query_rewriter'); +var overviewsQueryRewriter = new OverviewsQueryRewriter({ + zoom_level: 'ZoomLevel()' +}); + +function normalize_whitespace(txt) { + return txt.replace(/\s+/g, " ").trim(); +} + +// compare SQL statements ignoring whitespace +function assertSameSql(sql1, sql2) { + assert.equal(normalize_whitespace(sql1), normalize_whitespace(sql2)); +} + +describe('Overviews query rewriter', function() { + + it('does not alter queries if no overviews data is present', function(done){ + var sql = "SELECT * FROM table1"; + var overviews_sql = overviewsQueryRewriter.query(sql); + assert.equal(overviews_sql, sql); + overviews_sql = overviewsQueryRewriter.query(sql, {}); + assert.equal(overviews_sql, sql); + overviews_sql = overviewsQueryRewriter.query(sql, { overviews: {} }); + assert.equal(overviews_sql, sql); + done(); + }); + + + it('does not alter queries which don\'t use overviews', function(done){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table2: { + 0: { table: 'table2_ov0' }, + 1: { table: 'table2_ov1' }, + 4: { table: 'table2_ov4' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + done(); + }); + + // jshint multistr:true + + it('generates query with single overview layer for level 0', function(done){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\ + )\ + SELECT * FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query with single overview layer for level >0', function(done){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT * FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query with multiple overview layers for all levels up to N', function(done){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 1: { table: 'table1_ov1' }, + 2: { table: 'table1_ov2' }, + 3: { table: 'table1_ov3' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ + UNION ALL\ + SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ + UNION ALL\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2\ + UNION ALL\ + SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\ + )\ + SELECT * FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query with multiple overview layers for random levels', function(done){ + var sql = "SELECT * FROM table1"; + var data = { + overviews: { + table1: { + 0: { table: 'table1_ov0' }, + 1: { table: 'table1_ov1' }, + 6: { table: 'table1_ov6' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ + UNION ALL\ + SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ + UNION ALL\ + SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\ + )\ + SELECT * FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query for a table with explicit schema', function(done){ + var sql = "SELECT * FROM public.table1"; + var data = { + overviews: { + 'public.table1': { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT * FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query for a table with explicit schema in the overviews info', function(done){ + var sql = "SELECT * FROM public.table1"; + var data = { + overviews: { + 'public.table1': { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT * FROM _vovw_table1\ + "; + + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query for a table that needs quoting with explicit schema', function(done){ + var sql = "SELECT * FROM public.\"table 1\""; + var data = { + overviews: { + 'public."table 1"': { + 2: { table: '"table 1_ov2"' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + \"_vovw_table 1\" AS (\ + SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT * FROM \"_vovw_table 1\"\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query for a table with explicit schema that needs quoting', function(done){ + var sql = "SELECT * FROM \"user-1\".table1"; + var data = { + overviews: { + '"user-1".table1': { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT * FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + + it('generates query for a table with explicit schema both needing quoting', function(done){ + var sql = "SELECT * FROM \"user-1\".\"table 1\""; + var data = { + overviews: { + '"user-1"."table 1"': { + 2: { table: '"table 1_ov2"' } + + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + \"_vovw_table 1\" AS (\ + SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT * FROM \"_vovw_table 1\"\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + + it('generates query using overviews for queries with selected columns', function(done){ + var sql = "SELECT column1, column2, column3 FROM table1"; + var data = { + overviews: { + table1: { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT column1, column2, column3 FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query using overviews for queries with selected columns and all columns', function(done){ + var sql = "SELECT table1.*, column1, column2, column3 FROM table1"; + var data = { + overviews: { + table1: { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query using overviews for queries with a semicolon', function(done){ + var sql = "SELECT table1.*, column1, column2, column3 FROM table1;"; + var data = { + overviews: { + table1: { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('generates query using overviews for queries with extra whitespace', function(done){ + var sql = " SELECT table1.* , column1,column2, column3 FROM table1 "; + var data = { + overviews: { + table1: { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ + WITH\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ + _vovw_table1 AS (\ + SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ + UNION ALL\ + SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ + )\ + SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\ + "; + assertSameSql(overviews_sql, expected_sql); + done(); + }); + + it('does not alter queries which have not the simple supported form', function(done){ + var sql = "SELECT * FROM table1 WHERE column1='x'"; + var data = { + overviews: { + table1: { + 2: { table: 'table1_ov2' } + } + } + }; + var overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "SELECT * FROM table1 JOIN table2 ON (table1.col1=table2.col1)"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "SELECT a+b AS c FROM table1"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "SELECT f(a) AS b FROM table1"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "SELECT * FROM table1 AS x"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "WITH a AS (1) SELECT * FROM table1"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "SELECT * FROM table1 WHERE a=1"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + sql = "\ + SELECT table1.* FROM table1 \ + JOIN areas ON ST_Intersects(table1.the_geom, areas.the_geom) \ + WHERE areas.name='A' \ + "; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + + done(); + }); + + +}); diff --git a/test/unit/cartodb/table_name_parser.js b/test/unit/cartodb/table_name_parser.js new file mode 100644 index 00000000..c08ebd8a --- /dev/null +++ b/test/unit/cartodb/table_name_parser.js @@ -0,0 +1,60 @@ +require('../../support/test_helper'); + +var assert = require('assert'); +var TableNameParser = require('../../../lib/cartodb/utils/table_name_parser'); + +describe('TableNameParser', function() { + + it('parses table names with scheme and quotes', function(done){ + + var test_cases = [ + ['xyz', { schema: null, table: 'xyz' }], + ['"xyz"', { schema: null, table: 'xyz' }], + ['"xy z"', { schema: null, table: 'xy z' }], + ['"xy.z"', { schema: null, table: 'xy.z' }], + ['"x.y.z"', { schema: null, table: 'x.y.z' }], + ['abc.xyz', { schema: 'abc', table: 'xyz' }], + ['"abc".xyz', { schema: 'abc', table: 'xyz' }], + ['abc."xyz"', { schema: 'abc', table: 'xyz' }], + ['"abc"."xyz"', { schema: 'abc', table: 'xyz' }], + ['"a bc"."x yz"', { schema: 'a bc', table: 'x yz' }], + ['"a bc".xyz', { schema: 'a bc', table: 'xyz' }], + ['"a.bc".xyz', { schema: 'a.bc', table: 'xyz' }], + ['"a.b.c".xyz', { schema: 'a.b.c', table: 'xyz' }], + ['"a.b.c.".xyz', { schema: 'a.b.c.', table: 'xyz' }], + ['"a""bc".xyz', { schema: 'a"bc', table: 'xyz' }], + ['"a""bc"."x""yz"', { schema: 'a"bc', table: 'x"yz' }], + ]; + + test_cases.forEach(function(test_case) { + var table_name = test_case[0]; + var expected_result = test_case[1]; + var result = TableNameParser.parse(table_name); + assert.deepEqual(result, expected_result); + }); + done(); + }); + + it('quotes identifiers that need quoting', function(done){ + assert.equal(TableNameParser.quote('x yz'), '"x yz"'); + assert.equal(TableNameParser.quote('x-yz'), '"x-yz"'); + assert.equal(TableNameParser.quote('x.yz'), '"x.yz"'); + done(); + }); + + it('doubles quotes', function(done){ + assert.equal(TableNameParser.quote('x"yz'), '"x""yz"'); + assert.equal(TableNameParser.quote('x"y"z'), '"x""y""z"'); + assert.equal(TableNameParser.quote('x""y"z'), '"x""""y""z"'); + assert.equal(TableNameParser.quote('x "yz'), '"x ""yz"'); + assert.equal(TableNameParser.quote('x"y-y"z'), '"x""y-y""z"'); + done(); + }); + + it('does not quote identifiers that don\'t need to be quoted', function(done){ + assert.equal(TableNameParser.quote('xyz'), 'xyz'); + assert.equal(TableNameParser.quote('x_z123'), 'x_z123'); + done(); + }); + +});