From 579cabdc1ac997f86509e91e1616abb910830374 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Fri, 10 Jul 2015 01:30:38 +0200 Subject: [PATCH] Initial refactor of layergroup creation --- lib/cartodb/controllers/map.js | 58 +++++--- lib/cartodb/controllers/named_maps.js | 136 +++++++----------- lib/cartodb/server.js | 190 ++++++++++++-------------- 3 files changed, 175 insertions(+), 209 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 585cf4f8..4e3ef210 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -4,27 +4,35 @@ var step = require('step'); var cors = require('../middleware/cors'); -var MapStoreMapConfigProvider = require('windshaft').model.provider.MapStoreMapConfig; +var windshaft = require('windshaft'); +var MapStoreMapConfigProvider = windshaft.model.provider.MapStoreMapConfig; +var MapConfig = windshaft.model.MapConfig; +var Datasource = windshaft.model.Datasource; + +var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter'); /** * @param app + * @param {PgConnection} pgConnection * @param {MapStore} mapStore + * @param {TemplateMaps} templateMaps * @param {MapBackend} mapBackend * @param {TileBackend} tileBackend * @param {PreviewBackend} previewBackend * @param {AttributesBackend} attributesBackend - * @param layergroupRequestDecorator * @constructor */ -function MapController(app, mapStore, mapBackend, tileBackend, previewBackend, attributesBackend, - layergroupRequestDecorator) { +function MapController(app, pgConnection, mapStore, templateMaps, mapBackend, tileBackend, previewBackend, + attributesBackend) { this._app = app; + this.pgConnection = pgConnection; this.mapStore = mapStore; + this.templateMaps = templateMaps; this.mapBackend = mapBackend; this.tileBackend = tileBackend; this.previewBackend = previewBackend; this.attributesBackend = attributesBackend; - this._layergroupRequestDecorator = layergroupRequestDecorator; + this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); } module.exports = MapController; @@ -79,31 +87,39 @@ MapController.prototype.attributes = function(req, res) { MapController.prototype.create = function(req, res, prepareConfigFn) { var self = this; - var layergroupDecorator = { - beforeLayergroupCreate: function(requestMapConfig, callback) { - self._layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback); - }, - afterLayergroupCreate: function(layergroup, response, callback) { - self._layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback); - } - }; - step( function setupParams(){ self._app.req2params(req, this); }, prepareConfigFn, - function initLayergroup(err, requestMapConfig) { + function beforeLayergroupCreate(err, requestMapConfig) { assert.ifError(err); - self.mapBackend.createLayergroup(requestMapConfig, req.params, layergroupDecorator, this); + var next = this; + self.namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, self.pgConnection, + function(err, layers, datasource) { + if (err) { + return next(err); + } + + if (layers) { + requestMapConfig.layers = layers; + } + return next(null, requestMapConfig, datasource); + } + ); }, - function finish(err, response){ + function createLayergroup(err, requestMapConfig, datasource) { + assert.ifError(err); + self.mapBackend.createLayergroup( + new MapConfig(requestMapConfig, datasource || Datasource.EmptyDatasource()), req.params, this + ); + }, + function finish(err, layergroup) { if (err) { - response = { errors: [ err.message ] }; var statusCode = self._app.findStatusCode(err); - self._app.sendError(res, response, statusCode, 'GET LAYERGROUP', err); + self._app.sendError(res, { errors: [ err.message ] }, statusCode, 'ANONYMOUS LAYERGROUP', err); } else { - self._app.sendResponse(res, [response, 200]); + self._app.sendResponse(res, [layergroup, 200]); } } ); @@ -127,7 +143,7 @@ MapController.prototype.createPost = function(req, res) { this.create(req, res, function createPost$prepareConfig(err, req) { assert.ifError(err); - if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] !== 'application/json' ) { + if (!req.is('application/json')) { throw new Error('layergroup POST data must be of type application/json'); } return req.body; diff --git a/lib/cartodb/controllers/named_maps.js b/lib/cartodb/controllers/named_maps.js index 4509dec7..e75100de 100644 --- a/lib/cartodb/controllers/named_maps.js +++ b/lib/cartodb/controllers/named_maps.js @@ -7,8 +7,7 @@ var cors = require('../middleware/cors'); var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider'); function NamedMapsController(app, pgConnection, mapStore, templateMaps, metadataBackend, mapBackend, tileBackend, - previewBackend, templateBaseUrl, surrogateKeysCache, layergroupRequestDecorator, - tablesExtentApi) { + previewBackend, templateBaseUrl, surrogateKeysCache, tablesExtentApi) { this.app = app; this.mapStore = mapStore; this.pgConnection = pgConnection; @@ -19,7 +18,6 @@ function NamedMapsController(app, pgConnection, mapStore, templateMaps, metadata this.previewBackend = previewBackend; this.templateBaseUrl = templateBaseUrl; this.surrogateKeysCache = surrogateKeysCache; - this.layergroupRequestDecorator = layergroupRequestDecorator; this.tablesExtentApi = tablesExtentApi; } @@ -35,50 +33,40 @@ NamedMapsController.prototype.register = function(app) { }; NamedMapsController.prototype.instantiate = function(req, res) { - var self = this; - if (req.profiler) { req.profiler.start('windshaft-cartodb.instance_template_post'); } - step( - function instantiateTemplate() { - ifInvalidContentType(req, 'template POST data must be of type application/json'); - self.instantiateTemplate(req, res, req.body, this); - }, function finishInstantiation(err, response) { - self.finish_instantiation(err, response, res); + this.instantiateTemplate(req, res, function prepareTemplateParams(callback) { + if (!req.is('application/json')) { + return callback(new Error('Template POST data must be of type application/json')); } - ); + return callback(null, req.body); + }); }; -/** - * jsonp endpoint, allows to instantiate a template with a json call. - * callback query argument is mandatory - */ NamedMapsController.prototype.jsonp = function(req, res) { - var self = this; - if (req.profiler) { req.profiler.start('windshaft-cartodb.instance_template_get'); } - step( - function jsonp$instantiateTemplate() { - if ( req.query.callback === undefined || req.query.callback.length === 0) { - throw new Error('callback parameter should be present and be a function name'); - } - var config = {}; - if(req.query.config) { - try { - config = JSON.parse(req.query.config); - } catch(e) { - throw new Error('Invalid config parameter, should be a valid JSON'); - } - } - self.instantiateTemplate(req, res, config, this); - }, function finishInstantiation(err, response) { - self.finish_instantiation(err, response, res); + + this.instantiateTemplate(req, res, function prepareJsonTemplateParams(callback) { + var err = null; + if ( req.query.callback === undefined || req.query.callback.length === 0) { + err = new Error('callback parameter should be present and be a function name'); } - ); + + var templateParams = {}; + if (req.query.config) { + try { + templateParams = JSON.parse(req.query.config); + } catch(e) { + err = new Error('Invalid config parameter, should be a valid JSON'); + } + } + + return callback(err, templateParams); + }); }; NamedMapsController.prototype.tile = function(req, res) { @@ -221,71 +209,51 @@ NamedMapsController.prototype.staticMap = function(req, res) { // Instantiate a template -NamedMapsController.prototype.instantiateTemplate = function(req, res, template_params, callback) { +NamedMapsController.prototype.instantiateTemplate = function(req, res, prepareParamsFn) { var self = this; - var layergroupDecorator = { - beforeLayergroupCreate: function(requestMapConfig, callback) { - self.layergroupRequestDecorator.beforeLayergroupCreate(req, requestMapConfig, callback); - }, - afterLayergroupCreate: function(layergroup, response, callback) { - self.layergroupRequestDecorator.afterLayergroupCreate(req, layergroup, response, callback); - } - }; - var cdbuser = req.context.user; - var mapConfigProvider = new NamedMapMapConfigProvider( - this.templateMaps, - this.pgConnection, - cdbuser, - req.params.template_id, - template_params, - req.query.auth_token, - req.params - ); + + var mapConfigProvider; step( - function getTemplate(){ + function getTemplateParams() { + prepareParamsFn(this); + }, + function getTemplate(err, templateParams) { + assert.ifError(err); + mapConfigProvider = new NamedMapMapConfigProvider( + this.templateMaps, + this.pgConnection, + cdbuser, + req.params.template_id, + templateParams, + req.query.auth_token, + req.params + ); mapConfigProvider.getMapConfig(this); }, function createLayergroup(err, mapConfig, rendererParams/*, context*/) { assert.ifError(err); - self.mapBackend.createLayergroup(mapConfig.obj(), rendererParams, layergroupDecorator, this); + self.mapBackend.createLayergroup(mapConfig, rendererParams, this); }, - function prepareResponse(err, layergroup) { + function finishTemplateInstantiation(err, layergroup) { if (err) { - return callback(err, { errors: [''+err] }); + var statusCode = this._app.findStatusCode(err); + this.app.sendError(res, { errors: [ err.message ] }, statusCode, 'NAMED MAP LAYERGROUP', err); + } else { + var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0, 8); + layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; + + res.header('X-Layergroup-Id', layergroup.layergroupid); + self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.getTemplateName())); + + this.app.sendResponse(res, [layergroup, 200]); } - var templateHash = self.templateMaps.fingerPrint(mapConfigProvider.template).substring(0,8); - layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; - res.header('X-Layergroup-Id', layergroup.layergroupid); - - self.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(cdbuser, mapConfigProvider.template.name)); - - callback(null, layergroup); } ); }; -NamedMapsController.prototype.finish_instantiation = function(err, response, res) { - if (err) { - var statusCode = 400; - response = { errors: [''+err] }; - if ( ! _.isUndefined(err.http_status) ) { - statusCode = err.http_status; - } - this.app.sendError(res, response, statusCode, 'POST INSTANCE TEMPLATE', err); - } else { - this.app.sendResponse(res, [response, 200]); - } -}; - -function ifInvalidContentType(req, description) { - if ( ! req.headers['content-type'] || req.headers['content-type'].split(';')[0] != 'application/json' ) { - throw new Error(description); - } -} - function getStaticImageOptions(template, callback) { if (template.view) { var zoomCenter = templateZoomCenter(template.view); diff --git a/lib/cartodb/server.js b/lib/cartodb/server.js index aa55f39a..725c1483 100644 --- a/lib/cartodb/server.js +++ b/lib/cartodb/server.js @@ -13,9 +13,6 @@ var NamedMapsCacheEntry = require('./cache/model/named_maps_entry'); var VarnishHttpCacheBackend = require('./cache/backend/varnish_http'); var FastlyCacheBackend = require('./cache/backend/fastly'); -var MapConfigNamedLayersAdapter = require('./models/mapconfig_named_layers_adapter'); - - var windshaft = require('windshaft'); var mapnik = windshaft.mapnik; @@ -198,104 +195,89 @@ module.exports = function(serverOptions) { return statusCode; }; - var namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps); - var layergroupRequestDecorator = { - beforeLayergroupCreate: function(req, requestMapConfig, callback) { - namedLayersAdapter.getLayers(req.context.user, requestMapConfig.layers, pgConnection, - function(err, layers, datasource) { - if (err) { - return callback(err); - } - - if (layers) { - requestMapConfig.layers = layers; - } - return callback(null, requestMapConfig, datasource); - } - ); - }, - afterLayergroupCreate: function(req, mapconfig, response, callback) { - var token = response.layergroupid; - - var username = req.context.user; - - var tasksleft = 2; // redis key and affectedTables - var errors = []; - - var done = function(err) { - if ( err ) { - errors.push('' + err); - } - if ( ! --tasksleft ) { - err = errors.length ? new Error(errors.join('\n')) : null; - callback(err); - } - }; - - // include in layergroup response the variables in serverMedata - // those variables are useful to send to the client information - // about how to reach this server or information about it - var serverMetadata = global.environment.serverMetadata; - if (serverMetadata) { - _.extend(response, serverMetadata); - } - - // Don't wait for the mapview count increment to - // take place before proceeding. Error will be logged - // asyncronously - metadataBackend.incMapviewCount(username, mapconfig.stat_tag, function(err) { - if (req.profiler) { - req.profiler.done('incMapviewCount'); - } - if ( err ) { - console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); - } - done(); - }); - - var sql = mapconfig.layers.map(function(layer) { - return layer.options.sql; - }).join(';'); - - var dbName = req.params.dbname; - var cacheKey = dbName + ':' + token; - - step( - function getAffectedTablesAndLastUpdatedTime() { - queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); - }, - function handleAffectedTablesAndLastUpdatedTime(err, result) { - if (req.profiler) { - req.profiler.done('queryTablesAndLastUpdated'); - } - assert.ifError(err); - var cacheChannel = app.buildCacheChannel(dbName, result.affectedTables); - app.channelCache[cacheKey] = cacheChannel; - - // last update for layergroup cache buster - response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime; - response.last_updated = new Date(result.lastUpdatedTime).toISOString(); - - var res = req.res; - if (res) { - if (req.method === 'GET') { - var ttl = global.environment.varnish.layergroupTtl || 86400; - res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); - res.header('Last-Modified', (new Date()).toUTCString()); - res.header('X-Cache-Channel', cacheChannel); - } - - res.header('X-Layergroup-Id', response.layergroupid); - } - - return null; - }, - function finish(err) { - done(err); - } - ); - } - }; +// var layergroupRequestDecorator = { +// afterLayergroupCreate: function(req, mapconfig, layergroup, callback) { +// var token = layergroup.layergroupid; +// +// var username = req.context.user; +// +// var tasksleft = 2; // redis key and affectedTables +// var errors = []; +// +// var done = function(err) { +// if ( err ) { +// errors.push('' + err); +// } +// if ( ! --tasksleft ) { +// err = errors.length ? new Error(errors.join('\n')) : null; +// callback(err); +// } +// }; +// +// // include in layergroup response the variables in serverMedata +// // those variables are useful to send to the client information +// // about how to reach this server or information about it +// var serverMetadata = global.environment.serverMetadata; +// if (serverMetadata) { +// _.extend(layergroup, serverMetadata); +// } +// +// // Don't wait for the mapview count increment to +// // take place before proceeding. Error will be logged +// // asyncronously +// metadataBackend.incMapviewCount(username, mapconfig.stat_tag, function(err) { +// if (req.profiler) { +// req.profiler.done('incMapviewCount'); +// } +// if ( err ) { +// console.log("ERROR: failed to increment mapview count for user '" + username + "': " + err); +// } +// done(); +// }); +// +// var sql = mapconfig.layers.map(function(layer) { +// return layer.options.sql; +// }).join(';'); +// +// var dbName = req.params.dbname; +// var cacheKey = dbName + ':' + token; +// +// step( +// function getAffectedTablesAndLastUpdatedTime() { +// queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this); +// }, +// function handleAffectedTablesAndLastUpdatedTime(err, result) { +// if (req.profiler) { +// req.profiler.done('queryTablesAndLastUpdated'); +// } +// assert.ifError(err); +// var cacheChannel = app.buildCacheChannel(dbName, result.affectedTables); +// app.channelCache[cacheKey] = cacheChannel; +// +// // last update for layergroup cache buster +// layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime; +// layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString(); +// +// var res = req.res; +// if (res) { +// if (req.method === 'GET') { +// var ttl = global.environment.varnish.layergroupTtl || 86400; +// res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate'); +// res.header('Last-Modified', (new Date()).toUTCString()); +// res.header('X-Cache-Channel', cacheChannel); +// } +// +// res.header('X-Layergroup-Id', layergroup.layergroupid); +// } +// +// return null; +// }, +// function finish(err) { +// done(err); +// } +// ); +// } +// }; var TablesExtentApi = require('./api/tables_extent_api'); var tablesExtentApi = new TablesExtentApi(pgQueryRunner); @@ -311,12 +293,13 @@ module.exports = function(serverOptions) { new controller.Map( app, + pgConnection, mapStore, + templateMaps, mapBackend, tileBackend, previewBackend, - attributesBackend, - layergroupRequestDecorator + attributesBackend ).register(app); new controller.NamedMaps( @@ -330,7 +313,6 @@ module.exports = function(serverOptions) { previewBackend, template_baseurl, surrogateKeysCache, - layergroupRequestDecorator, tablesExtentApi ).register(app);