cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #515 from CartoDB/turbo-carto-datasource-improvements

Browse Source 
Use psql client instead of pg query runner to use proper params
This commit is contained in:
Raul Ochoa
2016-06-21 13:24:36 +02:00
committed by GitHub
parent 51eb8eb67f 863128013d
commit 7cee0f3ee3
7 changed files with 297 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/cartodb/utils/style/postgres-datasource.js → lib/cartodb/backends/turbo-carto-postgres-datasource.js
+7 -5
View File
@@ -54,9 +54,8 @@ var method2strategy = {
category: STRATEGY.EXACT
};
function PostgresDatasource (pgQueryRunner, username, query) {
this.pgQueryRunner = pgQueryRunner;
this.username = username;
function PostgresDatasource (psql, query) {
this.psql = psql;
this.query = query;
}
@@ -75,11 +74,14 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
var query = template({ _column: column, _sql: this.query, _buckets: buckets });
this.pgQueryRunner.run(this.username, query, function (err, result) {
this.psql.query(query, function (err, resultSet) {
if (err) {
return callback(err);
}
resultSet = resultSet || {};
var result = resultSet.rows || [];
var strategy = method2strategy[methodName];
var ramp = result[0][methodName] || [];
if (strategy !== STRATEGY.EXACT) {
@@ -89,7 +91,7 @@ PostgresDatasource.prototype.getRamp = function (column, buckets, method, callba
}
return callback(null, { ramp: ramp, strategy: strategy });
});
}, true); // use read-only transaction
};
module.exports = PostgresDatasource;
lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js
+36 -6
View File
@@ -3,10 +3,13 @@
var dot = require('dot');
dot.templateSettings.strip = false;
var queue = require('queue-async');
var SubstitutionTokens = require('../../../utils/substitution-tokens');
var PSQL = require('cartodb-psql');
var turboCarto = require('turbo-carto');
function TurboCartoAdapter(turboCartoParser) {
this.turboCartoParser = turboCartoParser;
var SubstitutionTokens = require('../../../utils/substitution-tokens');
var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasource');
function TurboCartoAdapter() {
}
module.exports = TurboCartoAdapter;
@@ -23,7 +26,7 @@ TurboCartoAdapter.prototype.getMapConfig = function (user, requestMapConfig, par
var parseCartoQueue = queue(layers.length);
layers.forEach(function(layer, index) {
parseCartoQueue.defer(self._parseCartoCss.bind(self), user, layer, index);
parseCartoQueue.defer(self._parseCartoCss.bind(self), user, params, layer, index);
});
parseCartoQueue.awaitAll(function (err, layers) {
@@ -51,7 +54,7 @@ var pixelSizeTemplate = dot.template('40075017 * cos(ST_Y(ST_Centroid({{=it._bbo
var scaleDenominatorTemplate = dot.template('({{=it._pixelSize}} / 0.00028)::numeric');
TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, index, callback) {
TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer, index, callback) {
if (!shouldParseLayerCartocss(layer)) {
return callback(null, layer);
}
@@ -81,7 +84,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, index, c
var sql = SubstitutionTokens.replace(layerSql, tokens);
this.turboCartoParser.process(username, layer.options.cartocss, sql, function (err, cartocss) {
var pg = new PSQL(dbParamsFromReqParams(params));
this.process(pg, layer.options.cartocss, sql, function (err, cartocss) {
// Only return turbo-carto errors
if (err && err.name === 'TurboCartoError') {
var error = new Error('turbo-carto: ' + err.message);
@@ -104,6 +109,31 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, layer, index, c
});
};
TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) {
var datasource = new PostgresDatasource(psql, sql);
turboCarto(cartocss, datasource, callback);
};
function shouldParseLayerCartocss(layer) {
return layer && layer.options && layer.options.cartocss && layer.options.sql;
}
function dbParamsFromReqParams(params) {
var dbParams = {};
if ( params.dbuser ) {
dbParams.user = params.dbuser;
}
if ( params.dbpassword ) {
dbParams.pass = params.dbpassword;
}
if ( params.dbhost ) {
dbParams.host = params.dbhost;
}
if ( params.dbport ) {
dbParams.port = params.dbport;
}
if ( params.dbname ) {
dbParams.dbname = params.dbname;
}
return dbParams;
}
lib/cartodb/server.js
+1 -5
View File
@@ -33,8 +33,6 @@ var AnalysisBackend = require('./backends/analysis');
var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
var TurboCartoParser = require('./utils/style/turbo-carto-parser');
var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
@@ -154,15 +152,13 @@ module.exports = function(serverOptions) {
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var turboCartoParser = new TurboCartoParser(pgQueryRunner);
var mapConfigAdapter = new MapConfigAdapter(
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
new SqlWrapMapConfigAdapter(),
new DataviewsWidgetsAdapter(),
new AnalysisMapConfigAdapter(analysisBackend),
new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
new TurboCartoAdapter(turboCartoParser)
new TurboCartoAdapter()
);
var namedMapProviderCache = new NamedMapProviderCache(
lib/cartodb/utils/style/turbo-carto-parser.js
-15
View File
@@ -1,15 +0,0 @@
'use strict';
var turboCarto = require('turbo-carto');
var PostgresDatasource = require('./postgres-datasource');
function TurboCartoParser (pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = TurboCartoParser;
TurboCartoParser.prototype.process = function (username, cartocss, sql, callback) {
var datasource = new PostgresDatasource(this.pgQueryRunner, username, sql);
turboCarto(cartocss, datasource, callback);
};
test/acceptance/turbo-cartocss/named-maps.js
+193 -177
View File
@@ -22,215 +22,231 @@ describe('turbo-carto for named maps', function() {
var templateId = 'turbo-carto-template-1';
var template = {
version: '0.0.1',
name: templateId,
auth: { method: 'open' },
placeholders: {
color: {
type: "css_color",
default: "Reds"
}
},
layergroup: {
version: '1.0.0',
layers: [{
options: {
sql: [
'SELECT test_table.*, _prices.price FROM test_table JOIN (' +
' SELECT 1 AS cartodb_id, 10.00 AS price',
' UNION',
' SELECT 2, 10.50',
' UNION',
' SELECT 3, 11.00',
' UNION',
' SELECT 4, 12.00',
' UNION',
' SELECT 5, 21.00',
') _prices ON _prices.cartodb_id = test_table.cartodb_id'
].join('\n'),
cartocss: [
'#layer {',
' marker-fill: ramp([price], colorbrewer(<%= color %>));',
' marker-allow-overlap:true;',
'}'
].join('\n'),
cartocss_version: '2.0.2'
}
function template(table) {
return {
version: '0.0.1',
name: templateId,
auth: { method: 'open' },
placeholders: {
color: {
type: "css_color",
default: "Reds"
}
]
}
};
},
layergroup: {
version: '1.0.0',
layers: [
{
options: {
sql: [
'SELECT ' + table + '.*, _prices.price FROM ' + table + ' JOIN (' +
' SELECT 1 AS cartodb_id, 10.00 AS price',
' UNION',
' SELECT 2, 10.50',
' UNION',
' SELECT 3, 11.00',
' UNION',
' SELECT 4, 12.00',
' UNION',
' SELECT 5, 21.00',
') _prices ON _prices.cartodb_id = ' + table + '.cartodb_id'
].join('\n'),
cartocss: [
'#layer {',
' marker-fill: ramp([price], colorbrewer(<%= color %>));',
' marker-allow-overlap:true;',
'}'
].join('\n'),
cartocss_version: '2.0.2'
}
}
]
}
};
}
var templateParamsReds = { color: 'Reds' };
var templateParamsBlues = { color: 'Blues' };
it('should create a template with turbo-carto parsed properly', function (done) {
step(
function postTemplate() {
var next = this;
var scenarios = [
{
desc: 'with public tables',
table: 'test_table'
},
{
desc: 'with private tables',
table: 'test_table_private_1'
}
];
assert.response(server, {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(template)
}, {},
function (res, err) {
next(err, res);
});
},
function checkTemplate(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
assert.deepEqual(JSON.parse(res.body), {
template_id: templateId
});
scenarios.forEach(function(scenario) {
it('should create a template with turbo-carto parsed properly: ' + scenario.desc, function (done) {
step(
function postTemplate() {
var next = this;
return null;
},
function instantiateTemplateWithReds(err) {
assert.ifError(err);
assert.response(server, {
url: '/api/v1/map/named?api_key=1234',
method: 'POST',
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(template(scenario.table))
}, {},
function (res, err) {
next(err, res);
});
},
function checkTemplate(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
assert.deepEqual(JSON.parse(res.body), {
template_id: templateId
});
var next = this;
assert.response(server, {
url: '/api/v1/map/named/' + templateId,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(templateParamsReds)
}, {},
function(res, err) {
return next(err, res);
});
},
function checkInstanciationWithReds(err, res) {
assert.ifError(err);
return null;
},
function instantiateTemplateWithReds(err) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
var next = this;
assert.response(server, {
url: '/api/v1/map/named/' + templateId,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(templateParamsReds)
}, {},
function(res, err) {
return next(err, res);
});
},
function checkInstanciationWithReds(err, res) {
assert.ifError(err);
var parsedBody = JSON.parse(res.body);
assert.equal(res.statusCode, 200);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var parsedBody = JSON.parse(res.body);
assert.ok(parsedBody.layergroupid);
assert.ok(parsedBody.last_updated);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
return parsedBody.layergroupid;
},
function requestTileReds(err, layergroupId) {
assert.ifError(err);
assert.ok(parsedBody.layergroupid);
assert.ok(parsedBody.last_updated);
var next = this;
return parsedBody.layergroupid;
},
function requestTileReds(err, layergroupId) {
assert.ifError(err);
assert.response(server, {
url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
method: 'GET',
headers: { host: 'localhost' },
encoding: 'binary'
}, {},
function(res, err) {
next(err, res);
});
},
function checkTileReds(err, res) {
assert.ifError(err);
var next = this;
var next = this;
assert.response(server, {
url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
method: 'GET',
headers: { host: 'localhost' },
encoding: 'binary'
}, {},
function(res, err) {
next(err, res);
});
},
function checkTileReds(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
assert.equal(res.headers['content-type'], 'image/png');
var next = this;
var fixturePath = './test/fixtures/turbo-carto-named-maps-reds.png';
var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
assert.equal(res.statusCode, 200);
assert.equal(res.headers['content-type'], 'image/png');
assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
},
function instantiateTemplateWithBlues(err) {
assert.ifError(err);
var fixturePath = './test/fixtures/turbo-carto-named-maps-reds.png';
var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
var next = this;
assert.response(server, {
url: '/api/v1/map/named/' + templateId,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(templateParamsBlues)
}, {},
function(res, err) {
return next(err, res);
});
},
function checkInstanciationWithBlues(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
},
function instantiateTemplateWithBlues(err) {
assert.ifError(err);
var parsedBody = JSON.parse(res.body);
var next = this;
assert.response(server, {
url: '/api/v1/map/named/' + templateId,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(templateParamsBlues)
}, {},
function(res, err) {
return next(err, res);
});
},
function checkInstanciationWithBlues(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var parsedBody = JSON.parse(res.body);
assert.ok(parsedBody.layergroupid);
assert.ok(parsedBody.last_updated);
keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
return parsedBody.layergroupid;
},
function requestTileBlues(err, layergroupId) {
assert.ifError(err);
assert.ok(parsedBody.layergroupid);
assert.ok(parsedBody.last_updated);
var next = this;
return parsedBody.layergroupid;
},
function requestTileBlues(err, layergroupId) {
assert.ifError(err);
assert.response(server, {
url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
method: 'GET',
headers: { host: 'localhost' },
encoding: 'binary'
}, {},
function(res, err) {
next(err, res);
});
},
function checkTileBlues(err, res) {
assert.ifError(err);
var next = this;
var next = this;
assert.response(server, {
url: '/api/v1/map/' + layergroupId + '/0/0/0.png',
method: 'GET',
headers: { host: 'localhost' },
encoding: 'binary'
}, {},
function(res, err) {
next(err, res);
});
},
function checkTileBlues(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200);
assert.equal(res.headers['content-type'], 'image/png');
var next = this;
var fixturePath = './test/fixtures/turbo-carto-named-maps-blues.png';
var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
assert.equal(res.statusCode, 200);
assert.equal(res.headers['content-type'], 'image/png');
assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
},
function deleteTemplate(err) {
assert.ifError(err);
var fixturePath = './test/fixtures/turbo-carto-named-maps-blues.png';
var image = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
var next = this;
assert.imageIsSimilarToFile(image, fixturePath, IMAGE_TOLERANCE_PER_MIL, next);
},
function deleteTemplate(err) {
assert.ifError(err);
assert.response(server, {
url: '/api/v1/map/named/' + templateId + '?api_key=1234',
method: 'DELETE',
headers: { host: 'localhost' }
}, {}, function (res, err) {
next(err, res);
});
},
function checkDeleteTemplate(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 204);
assert.ok(!res.body);
var next = this;
return null;
},
function finish(err) {
done(err);
}
);
assert.response(server, {
url: '/api/v1/map/named/' + templateId + '?api_key=1234',
method: 'DELETE',
headers: { host: 'localhost' }
}, {}, function (res, err) {
next(err, res);
});
},
function checkDeleteTemplate(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 204);
assert.ok(!res.body);
return null;
},
function finish(err) {
done(err);
}
);
});
});
});
test/acceptance/turbo-cartocss/regressions.js
+51
View File
@@ -66,6 +66,57 @@ describe('turbo-carto regressions', function() {
});
});
it('should fail for private tables', function(done) {
var cartocss = [
"#private_table {",
" marker-placement: point;",
" marker-allow-overlap: true;",
" marker-line-width: 0;",
" marker-fill-opacity: 1.0;",
" marker-width: ramp([cartodb_id], 10, 20);",
" marker-fill: red;",
"}"
].join('\n');
this.testClient = new TestClient(makeMapconfig('SELECT * FROM test_table_private_1', cartocss));
this.testClient.getLayergroup(TestClient.RESPONSE.ERROR, function(err, layergroup) {
assert.ok(!err, err);
assert.ok(!layergroup.hasOwnProperty('layergroupid'));
assert.ok(layergroup.hasOwnProperty('errors'));
var turboCartoError = layergroup.errors_with_context[0];
assert.ok(turboCartoError);
assert.equal(turboCartoError.type, 'turbo-carto');
assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\srelation\stest_table_private_1/));
done();
});
});
it('should work for private tables with api key', function(done) {
var cartocss = [
"#private_table {",
" marker-placement: point;",
" marker-allow-overlap: true;",
" marker-line-width: 0;",
" marker-fill-opacity: 1.0;",
" marker-width: ramp([cartodb_id], 10, 20);",
" marker-fill: red;",
"}"
].join('\n');
this.testClient = new TestClient(makeMapconfig('SELECT * FROM test_table_private_1', cartocss), 1234);
this.testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err, err);
assert.ok(layergroup.hasOwnProperty('layergroupid'));
assert.ok(!layergroup.hasOwnProperty('errors'));
done();
});
});
it('should work with mapnik substitution tokens', function(done) {
var cartocss = [
"#layer {",
test/support/test-client.js
+9
View File
@@ -24,6 +24,15 @@ function TestClient(mapConfig, apiKey) {
module.exports = TestClient;
module.exports.RESPONSE = {
ERROR: {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
TestClient.prototype.getWidget = function(widgetName, params, callback) {
var self = this;