Compare commits

..

2 Commits
3.9.4 ... 3.6.7

Author SHA1 Message Date
Raul Ochoa
47217909f6 Merge pull request #741 from CartoDB/3.6.6-fix-pg-vulnerability
3.6.6 fix pg vulnerability
2017-09-01 17:13:35 +02:00
Simon
3a0a366230 Adding a new cartodb-psql fixing a pg vulnerability 2017-09-01 13:30:30 +02:00
44 changed files with 482 additions and 1963 deletions

4
.gitignore vendored
View File

@@ -8,6 +8,6 @@ tools/munin/windshaft.conf
logs/
pids/
redis.pid
*.log
test.log
npm-debug.log
coverage/
.DS_Store

View File

@@ -40,7 +40,7 @@
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
// "eqnull" : false, // true: Tolerate use of `== null`
// "es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
// "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`)
// "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// // (ex: `for each`, multiple try/catch, function expression…)
// "evil" : false, // true: Tolerate use of `eval` and `new Function()`

View File

@@ -1,7 +1,7 @@
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. If there are modified dependencies in package.json, update them with `yarn upgrade {{package_name}}@{{version}}`
4. Recreate yarn.lock with: `yarn upgrade`
5. Commit package.json, yarn.lock, NEWS
6. git tag -a Major.Minor.Patch # use NEWS section as content
7. Stub NEWS/package for next version

57
NEWS.md
View File

@@ -1,61 +1,10 @@
# Changelog
## 3.9.4
Released 2017-06-22
## 3.6.7
Released 2017-09-01
Announcements:
- Upgrades camshaft to [0.55.6](https://github.com/CartoDB/camshaft/releases/tag/0.55.6).
## 3.9.3
Released 2017-06-16
Announcements:
- Upgrades camshaft to [0.55.5](https://github.com/CartoDB/camshaft/releases/tag/0.55.5).
## 3.9.2
Released 2017-06-16
Announcements:
- Upgrades camshaft to [0.55.4](https://github.com/CartoDB/camshaft/releases/tag/0.55.4).
## 3.9.1
Released 2017-06-06
Announcements:
- Upgrades camshaft to [0.55.3](https://github.com/CartoDB/camshaft/releases/tag/0.55.3).
## 3.9.0
Released 2017-05-31
Announcements:
- Upgrades windshaft to [3.2.1](https://github.com/CartoDB/windshaft/releases/tag/3.2.1).
- Add support to retrieve info about layer stats in map instantiation.
- Upgrades camshaft to [0.55.2](https://github.com/CartoDB/camshaft/releases/tag/0.55.2).
- Remove promise polyfill from turbo-carto adapter
## 3.8.0
Released 2017-05-22
Announcements:
- Upgrades camshaft to [0.55.0](https://github.com/CartoDB/camshaft/releases/tag/0.55.0).
- Upgrades turbo-carto to [0.19.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.19.1)
## 3.7.1
Released 2017-05-18
Bug fixes:
- Fix buffersize assignment when is not defined in requested mapconfig.
## 3.7.0
Released 2017-05-18
Announcements:
- Manage multiple values of buffer-size for different formats
- Upgrades windshaft to [3.2.0](https://github.com/CartoDB/windshaft/releases/tag/3.2.0).
- Adding a new cartodb-psql fixing a pg vulnerability.
## 3.6.6

View File

@@ -43,19 +43,53 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
ownFilter = !!ownFilter;
var query = (ownFilter) ? dataviewDefinition.sql.own_filter_on : dataviewDefinition.sql.own_filter_off;
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
query = bboxFilter.sql(query);
if ( queryRewriteData ) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom_webmercator',
srid: 3857
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
}
var queryRewriteData = getQueryRewriteData(mapConfig, dataviewDefinition, params);
var dataviewFactory = DataviewFactoryWithOverviews.getFactory(
overviewsQueryRewriter, queryRewriteData, { bbox: params.bbox }
);
var overrideParams = _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}
);
var dataview = dataviewFactory.getDataview(query, dataviewDefinition);
dataview.getResult(pg, getOverrideParams(params, ownFilter), this);
dataview.getResult(pg, overrideParams, this);
},
function returnCallback(err, result) {
return callback(err, result);
@@ -63,49 +97,6 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
);
};
function getQueryRewriteData(mapConfig, dataviewDefinition, params) {
var sourceId = dataviewDefinition.source.id; // node.id
var layer = _.find(mapConfig.obj().layers, function(l) {
return l.options.source && (l.options.source.id === sourceId);
});
var queryRewriteData = layer && layer.options.query_rewrite_data;
if (queryRewriteData && dataviewDefinition.node.type === 'source') {
queryRewriteData = _.extend({}, queryRewriteData, {
filters: dataviewDefinition.node.filters,
unfiltered_query: dataviewDefinition.sql.own_filter_on
});
}
if (params.bbox && queryRewriteData) {
var bbox_filter_definition = {
type: 'bbox',
options: {
column: 'the_geom_webmercator',
srid: 3857
},
params: {
bbox: params.bbox
}
};
queryRewriteData = _.extend(queryRewriteData, { bbox_filter: bbox_filter_definition });
}
return queryRewriteData;
}
function getOverrideParams(params, ownFilter) {
return _.reduce(_.pick(params, 'start', 'end', 'bins'),
function castNumbers(overrides, val, k) {
if (!Number.isFinite(+val)) {
throw new Error('Invalid number format for parameter \'' + k + '\'');
}
overrides[k] = +val;
return overrides;
},
{ownFilter: ownFilter}
);
}
DataviewBackend.prototype.search = function (mapConfigProvider, user, params, callback) {
var dataviewName = params.dataviewName;

View File

@@ -1,16 +0,0 @@
function EmptyLayerStats(types) {
this._types = types || {};
}
EmptyLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
EmptyLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
setImmediate(function() {
callback(null, {});
});
};
module.exports = EmptyLayerStats;

View File

@@ -1,23 +0,0 @@
var LayerStats = require('./layer-stats');
var EmptyLayerStats = require('./empty-layer-stats');
var MapnikLayerStats = require('./mapnik-layer-stats');
var TorqueLayerStats = require('./torque-layer-stats');
module.exports = function LayerStatsFactory(type) {
var layerStatsIterator = [];
var selectedType = type || 'ALL';
if (selectedType === 'ALL') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true }));
layerStatsIterator.push(new MapnikLayerStats());
layerStatsIterator.push(new TorqueLayerStats());
} else if (selectedType === 'mapnik') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, torque: true }));
layerStatsIterator.push(new MapnikLayerStats());
} else if (selectedType === 'torque') {
layerStatsIterator.push(new EmptyLayerStats({ http: true, plain: true, mapnik: true }));
layerStatsIterator.push(new TorqueLayerStats());
}
return new LayerStats(layerStatsIterator);
};

View File

@@ -1,45 +0,0 @@
var queue = require('queue-async');
function LayerStats(layerStatsIterator) {
this.layerStatsIterator = layerStatsIterator;
}
LayerStats.prototype.getStats = function (mapConfig, dbConnection, callback) {
var self = this;
var stats = [];
if (!mapConfig.getLayers().length) {
return callback(null, stats);
}
var metaQueue = queue(mapConfig.getLayers().length);
mapConfig.getLayers().forEach(function (layer, layerId) {
var layerType = mapConfig.layerType(layerId);
for (var i = 0; i < self.layerStatsIterator.length; i++) {
if (self.layerStatsIterator[i].is(layerType)) {
var getStats = self.layerStatsIterator[i].getStats.bind(self.layerStatsIterator[i]);
metaQueue.defer(getStats, layer, dbConnection);
break;
}
}
});
metaQueue.awaitAll(function (err, results) {
if (err) {
return callback(err);
}
if (!results) {
return callback(null, null);
}
mapConfig.getLayers().forEach(function (layer, layerIndex) {
stats[layerIndex] = results[layerIndex];
});
return callback(err, stats);
});
};
module.exports = LayerStats;

View File

@@ -1,28 +0,0 @@
var queryUtils = require('../../utils/query-utils');
function MapnikLayerStats () {
this._types = {
mapnik: true,
cartodb: true
};
}
MapnikLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
MapnikLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
// This query would gather stats for postgresql table if not exists
dbConnection.query(queryRowCountSql, function (err, res) {
if (err) {
return callback(null, {estimatedFeatureCount: -1});
} else {
// We decided that the relation is 1 row == 1 feature
return callback(null, {estimatedFeatureCount: res.rows[0].rows});
}
});
};
module.exports = MapnikLayerStats;

View File

@@ -1,16 +0,0 @@
function TorqueLayerStats() {
this._types = {
torque: true
};
}
TorqueLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
TorqueLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
return callback(null, {});
};
module.exports = TorqueLayerStats;

View File

@@ -1,16 +0,0 @@
var layerStats = require('./layer-stats/factory');
function StatsBackend() {
}
module.exports = StatsBackend;
StatsBackend.prototype.getStats = function(mapConfig, dbConnection, callback) {
var enabledFeatures = global.environment.enabledFeatures;
var layerStatsEnabled = enabledFeatures ? enabledFeatures.layerStats: false;
if (layerStatsEnabled) {
layerStats().getStats(mapConfig, dbConnection, callback);
} else {
return callback(null, []);
}
};

View File

@@ -296,7 +296,7 @@ TemplateMaps.prototype.delTemplate = function(owner, tpl_id, callback) {
// @param callback function(err)
//
TemplateMaps.prototype.updTemplate = function(owner, tpl_id, template, callback) {
var self = this;
template = templateDefaults(template);
@@ -430,17 +430,13 @@ var _reNumber = /^([-+]?[\d\.]?\d+([eE][+-]?\d+)?)$/,
_reCSSColorVal = /^#[0-9a-fA-F]{3,6}$/;
function _replaceVars (str, params) {
// Construct regular expressions for each param
//return _.template(str, params); // lazy way, possibly dangerous
// Construct regular expressions for each param
Object.keys(params).forEach(function(k) {
str = str.replace(new RegExp("<%=\\s*" + k + "\\s*%>", "g"), params[k]);
});
return str;
}
function isObject(val) {
return ( _.isObject(val) && !_.isArray(val) && !_.isFunction(val));
}
TemplateMaps.prototype.instance = function(template, params) {
var all_params = {};
var phold = template.placeholders || {};
@@ -478,13 +474,6 @@ TemplateMaps.prototype.instance = function(template, params) {
// NOTE: we're deep-cloning the layergroup here
var layergroup = JSON.parse(JSON.stringify(template.layergroup));
if (layergroup.buffersize && isObject(layergroup.buffersize)) {
Object.keys(layergroup.buffersize).forEach(function(k) {
layergroup.buffersize[k] = parseInt(_replaceVars(layergroup.buffersize[k], all_params), 10);
});
}
for (var i=0; i<layergroup.layers.length; ++i) {
var lyropt = layergroup.layers[i].options;

View File

@@ -253,9 +253,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
png: true
};
var formatStat = 'invalid';

View File

@@ -20,7 +20,6 @@ var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
var NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
/**
* @param {AuthApi} authApi
* @param {PgConnection} pgConnection
@@ -31,12 +30,10 @@ var CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/cr
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
statsBackend) {
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter) {
BaseController.call(this, authApi, pgConnection);
@@ -50,8 +47,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
this.mapConfigAdapter = mapConfigAdapter;
this.resourceLocator = new ResourceLocator(global.environment);
this.statsBackend = statsBackend;
}
util.inherits(MapController, BaseController);
@@ -219,6 +214,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
var mapConfigProvider;
var mapConfig;
step(
function setupParams(){
self.req2params(req, this);
@@ -253,9 +249,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
},
function afterLayergroupCreate(err, layergroup) {
assert.ifError(err);
self.afterLayergroupCreate(req, res, mapConfig, layergroup,
mapConfigProvider.analysesResults,
this);
self.afterLayergroupCreate(req, res, mapConfig, layergroup, mapConfigProvider.analysesResults, this);
},
function finishTemplateInstantiation(err, layergroup) {
if (err) {
@@ -278,8 +272,7 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
);
};
MapController.prototype.afterLayergroupCreate =
function(req, res, mapconfig, layergroup, analysesResults, callback) {
MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, layergroup, analysesResults, callback) {
var self = this;
var username = req.context.user;
@@ -325,7 +318,6 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) {
var dbName = req.params.dbname;
var layergroupId = layergroup.layergroupid;
var dbConnection;
step(
function getPgConnection() {
@@ -333,8 +325,7 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) {
},
function getAffectedTablesAndLastUpdatedTime(err, connection) {
assert.ifError(err);
dbConnection = connection;
QueryTables.getAffectedTablesFromQuery(dbConnection, sql.join(';'), this);
QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
req.profiler.done('queryTablesAndLastUpdated');
@@ -361,21 +352,6 @@ function(req, res, mapconfig, layergroup, analysesResults, callback) {
return null;
},
function fetchLayersStats(err) {
assert.ifError(err);
var next = this;
self.statsBackend.getStats(mapconfig, dbConnection, function(err, layersStats) {
if (err) {
return next(err);
}
if (layersStats.length > 0) {
layergroup.metadata.layers.forEach(function (layer, index) {
layer.meta.stats = layersStats[index];
});
}
return next();
});
},
function finish(err) {
done(err);
}

View File

@@ -66,8 +66,7 @@ function getBoundingBoxes(west, south, east, north) {
bboxes.push([west, south, east, north]);
} else {
bboxes.push([west, south, 180, north]);
// here we assume west,east have been adjusted => west >= -180 => east > 180
bboxes.push([-180, south, east - 360, north]);
bboxes.push([-180, south, east % 180, north]);
}
return bboxes;

View File

@@ -1,25 +0,0 @@
function MapConfigBufferSizeAdapter() {
this.formats = ['png', 'png32', 'mvt', 'grid.json'];
}
module.exports = MapConfigBufferSizeAdapter;
MapConfigBufferSizeAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
if (!context.templateParams || !context.templateParams.buffersize) {
return callback(null, requestMapConfig);
}
this.formats.forEach(function (format) {
if (Number.isFinite(context.templateParams.buffersize[format])) {
if (requestMapConfig.buffersize === undefined) {
requestMapConfig.buffersize = {};
}
requestMapConfig.buffersize[format] = context.templateParams.buffersize[format];
}
});
setImmediate(function () {
callback(null, requestMapConfig);
});
};

View File

@@ -43,6 +43,7 @@ MapConfigNamedLayersAdapter.prototype.getMapConfig = function (user, requestMapC
if (nestedNamedLayers.length > 0) {
var nestedNamedMapsError = new Error('Nested named layers are not allowed');
// nestedNamedMapsError.http_status = 400;
return done(nestedNamedMapsError);
}

View File

@@ -4,6 +4,13 @@ var dot = require('dot');
dot.templateSettings.strip = false;
var queue = require('queue-async');
var PSQL = require('cartodb-psql');
/**
* cartodb-psql creates `global.Promise` as an empty constructor.
* However, `turbo-carto` relies on a polyfil that fails to create the polyfil
* as it finds `global.Promise` but it doesn't find `Promise.resolve`.
*/
global.Promise = global.Promise || function() {};
global.Promise.resolve = global.Promise.resolve || function() {};
var turboCarto = require('turbo-carto');
var SubstitutionTokens = require('../../../utils/substitution-tokens');

View File

@@ -90,7 +90,6 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
},
function instantiateTemplate(err, templateParams) {
assert.ifError(err);
context.templateParams = templateParams;
return self.templateMaps.instance(self.template, templateParams);
},
function prepareAdapterMapConfig(err, requestMapConfig) {

View File

@@ -35,15 +35,12 @@ var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encodin
var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
var MapConfigAdapter = require('./models/mapconfig/adapter');
var StatsBackend = require('./backends/stats');
module.exports = function(serverOptions) {
// Make stats client globally accessible
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@@ -153,14 +150,11 @@ module.exports = function(serverOptions) {
var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
var statsBackend = new StatsBackend();
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var mapConfigAdapter = new MapConfigAdapter(
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
new MapConfigBufferSizeAdapter(),
new SqlWrapMapConfigAdapter(),
new DataviewsWidgetsAdapter(),
new AnalysisMapConfigAdapter(analysisBackend),
@@ -213,8 +207,7 @@ module.exports = function(serverOptions) {
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
mapConfigAdapter,
statsBackend
mapConfigAdapter
).register(app);
new controller.NamedMaps(

View File

@@ -1,26 +0,0 @@
function prepareQuery(sql) {
var affectedTableRegexCache = {
bbox: /!bbox!/g,
scale_denominator: /!scale_denominator!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
return sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.scale_denominator, '0')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1');
}
module.exports.extractTableNames = function extractTableNames(query) {
return [
'SELECT * FROM CDB_QueryTablesText($windshaft$',
prepareQuery(query),
'$windshaft$) as tablenames'
].join('');
};
module.exports.getQueryRowCount = function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount(\'' + query + '\') as rows';
};

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "3.9.4",
"version": "3.6.7",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -20,8 +20,8 @@
],
"dependencies": {
"body-parser": "~1.14.0",
"camshaft": "0.55.6",
"cartodb-psql": "0.8.0",
"camshaft": "0.54.4",
"cartodb-psql": "0.10.1",
"cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.2",
"debug": "~2.2.0",
@@ -37,22 +37,21 @@
"request": "~2.79.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"turbo-carto": "0.19.1",
"turbo-carto": "0.19.0",
"underscore": "~1.6.0",
"windshaft": "3.2.1",
"windshaft": "3.1.2",
"yargs": "~5.0.0"
},
"devDependencies": {
"istanbul": "~0.4.3",
"jshint": "~2.9.4",
"mocha": "~3.4.1",
"jshint": "~2.6.0",
"mocha": "~1.21.4",
"nock": "~2.11.0",
"redis": "~0.12.1",
"semver": "~1.1.4",
"strftime": "~0.8.2"
},
"scripts": {
"lint": "jshint lib test",
"preinstall": "make pre-install",
"test": "make test-all"
},

View File

@@ -1,441 +0,0 @@
require('../support/test_helper');
var fs = require('fs');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
var mapnik = require('windshaft').mapnik;
var IMAGE_TOLERANCE_PER_MIL = 5;
var CARTOCSS_LABELS = [
'#layer {',
' polygon-fill: #374C70;',
' polygon-opacity: 0.9;',
' line-width: 1;',
' line-color: #FFF;',
' line-opacity: 0.5;',
'}',
'#layer::labels {',
' text-name: [name];',
' text-face-name: \'DejaVu Sans Book\';',
' text-size: 20;',
' text-fill: #FFFFFF;',
' text-label-position-tolerance: 0;',
' text-halo-radius: 1;',
' text-halo-fill: #6F808D;',
' text-dy: -10;',
' text-allow-overlap: true;',
' text-placement: point;',
' text-placement-type: dummy;',
'}'
].join('\n');
function createMapConfig (bufferSize, cartocss) {
cartocss = cartocss || CARTOCSS_LABELS;
return {
version: '1.6.0',
buffersize: bufferSize,
layers: [{
type: "cartodb",
options: {
sql: [
'select',
' *',
'from',
' populated_places_simple_reduced',
].join('\n'),
cartocss: cartocss,
cartocss_version: '2.3.0',
interactivity: 'cartodb_id'
}
}]
};
}
describe('buffer size per format', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
mapConfig: createMapConfig({ png: 0, 'grid.json': 0 }),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
mapConfig: createMapConfig({ png: 128, 'grid.json': 128 }),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get mvt tile using buffer-size 0',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.mvt',
mapConfig: createMapConfig({ mvt: 0 }),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
assert.equal(features.length, 1);
callback();
}
},
{
desc: 'should get mvt tile using buffer-size 128',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.mvt',
mapConfig: createMapConfig({ mvt: 128 }),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
assert.equal(features.length, 9);
callback();
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
mapConfig: createMapConfig({ 'grid.json': 0 }),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
mapConfig: createMapConfig({ 'grid.json': 128 }),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
}
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.mapConfig, 1234);
var coords = test.coords;
var options = {
format: test.format,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
// tile.save(test.fixturePath);
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
function createBufferSizeTemplate (name, buffersize, placeholders, cartocss) {
cartocss = cartocss || CARTOCSS_LABELS;
return {
"version": "0.0.1",
"name": name,
"placeholders": placeholders || {
"buffersize": {
"type": "number",
"default": 0
}
},
"layergroup": createMapConfig(buffersize)
};
}
describe('buffer size per format for named maps', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0 (default value in template)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-default-buffer-size', {png: '<%= buffersize %>'}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 (placehoder value)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-custom-buffer-size', { png: '<%= buffersize %>'}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 0 (default value in template by format)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize_png: 0 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-default-buffer-size-by-format', {
png: '<%= buffersize_png %>'
}, {
"buffersize_png": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 (placehoder value in template by format)',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: { buffersize_png: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-custom-buffer-size-by-format', {
png: '<%= buffersize_png %>'
}, {
"buffersize_png": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: { buffersize_gridjson: 0 },
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
template: createBufferSizeTemplate('named-default-buffer-size-by-format-gridjson', {
'grid.json': '<%= buffersize_gridjson %>'
}, {
"buffersize_gridjson": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: { buffersize_gridjson: 128 },
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
template: createBufferSizeTemplate('named-custom-buffer-size-by-format-gridjson', {
'grid.json': '<%= buffersize_gridjson %>'
}, {
"buffersize_gridjson": {
"type": "number",
"default": "0"
}
}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
}
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});
describe('buffer size per format for named maps w/o placeholders', function () {
var testCases = [
{
desc: 'should get png tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-no-buffer-size-png-0', {}, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get png tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.png',
template: createBufferSizeTemplate('named-no-buffer-size-png-128', {}, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
{
desc: 'should get mvt tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
placeholders: {
buffersize: {
mvt: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-0.mvt',
template: createBufferSizeTemplate('named-no-buffer-size-mvt', {}, {}),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
var dataFixture = fs.readFileSync(this.fixturePath);
var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
vtile.setDataSync(dataFixture);
var vtileJSON = vtile.toJSON();
var vtileFeatures = vtileJSON[0].features;
assert.equal(features.length, vtileFeatures.length);
callback();
}
},
{
desc: 'should get mvt tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'mvt',
placeholders: {
buffersize: {
mvt: 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-mvt-7.64.48-buffer-size-128.mvt',
template: createBufferSizeTemplate('named-no-buffer-size-mvt-128', {}, {}),
assert: function (tile, callback) {
var tileJSON = tile.toJSON();
var features = tileJSON[0].features;
var dataFixture = fs.readFileSync(this.fixturePath);
var vtile = new mapnik.VectorTile(this.coords.z, this.coords.x, this.coords.y);
vtile.setDataSync(dataFixture);
var vtileJSON = vtile.toJSON();
var vtileFeatures = vtileJSON[0].features;
assert.equal(features.length, vtileFeatures.length);
callback();
}
},
{
desc: 'should get grid.json tile using buffer-size 0 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: {
buffersize: {
'grid.json': 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-grid.json.7.64.48-buffer-size-0.grid.json',
template: createBufferSizeTemplate('named-no-buffer-size-grid-json-0', {}, {}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2,callback);
}
},
{
desc: 'should get grid.json tile using buffer-size 128 overriden by template params',
coords: { z: 7, x: 64, y: 48 },
format: 'grid.json',
layers: [0],
placeholders: {
buffersize: {
'grid.json': 128
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-128.grid.json',
template: createBufferSizeTemplate('named-no-buffer-size-grid-json-128', {}, {}),
assert: function (tile, callback) {
assert.utfgridEqualsFile(tile, this.fixturePath, 2, callback);
}
},
{
desc: 'should get png tile using buffer-size 0' +
' overriden by template params with no buffersize in mapconfig',
coords: { z: 7, x: 64, y: 48 },
format: 'png',
placeholders: {
buffersize: {
png: 0
}
},
fixturePath: './test/fixtures/buffer-size/tile-7.64.48-buffer-size-0.png',
template: createBufferSizeTemplate('named-no-buffer-size-mapconfig-png-0', undefined, {}),
assert: function (tile, callback) {
assert.imageIsSimilarToFile(tile, this.fixturePath, IMAGE_TOLERANCE_PER_MIL, callback);
}
},
];
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save(test.fixturePath);
// require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
// require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
});
});
});
});

View File

@@ -1041,7 +1041,7 @@ describe(suiteName, function() {
);
});
}
// See https://github.com/CartoDB/Windshaft-cartodb/issues/91
// and https://github.com/CartoDB/Windshaft-cartodb/issues/38
it("tiles for private tables can be fetched with api_key", function(done) {

View File

@@ -5,7 +5,6 @@ var step = require('step');
var cartodbServer = require('../../../lib/cartodb/server');
var ServerOptions = require('./support/ported_server_options');
var testClient = require('./support/test_client');
var TestClient = require('../../support/test-client');
var BaseController = require('../../../lib/cartodb/controllers/base');
@@ -24,14 +23,6 @@ describe('multilayer error cases', function() {
BaseController.prototype.req2params = req2paramsFn;
});
// var client = null;
afterEach(function(done) {
if (this.client) {
return this.client.drain(done);
}
return done();
});
it("post layergroup with wrong Content-Type", function(done) {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
@@ -162,16 +153,24 @@ describe('multilayer error cases', function() {
]
};
ServerOptions.afterLayergroupCreateCalls = 0;
this.client = new TestClient(layergroup);
this.client.getLayergroup({status: 400}, function(err, parsed) {
assert.ok(!err, err);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
assert.ok(error.match(/column "missing" does not exist/m), error);
done();
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
var parsed = JSON.parse(res.body);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
assert.ok(error.match(/column "missing" does not exist/m), error);
// TODO: check which layer introduced the problem ?
done();
} catch (err) { done(err); }
});
});

View File

@@ -16,13 +16,13 @@ describe('server_png8_format', function() {
var serverOptionsPng32 = ServerOptions;
serverOptionsPng32.grainstore = _.clone(ServerOptions.grainstore);
serverOptionsPng32.grainstore.mapnik_tile_format = 'png32';
var serverPng32 = cartodbServer(serverOptionsPng32);
var serverPng32 = new cartodbServer(serverOptionsPng32);
serverPng32.setMaxListeners(0);
var serverOptionsPng8 = ServerOptions;
serverOptionsPng8.grainstore = _.clone(ServerOptions.grainstore);
serverOptionsPng8.grainstore.mapnik_tile_format = 'png8:m=h';
var serverPng8 = cartodbServer(serverOptionsPng8);
var serverPng8 = new cartodbServer(serverOptionsPng8);
serverPng8.setMaxListeners(0);

View File

@@ -1,279 +0,0 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('Create mapnik layergroup', function() {
before(function() {
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerStats = true;
});
after(function() {
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var mapnikLayer1 = {
type: 'mapnik',
options: {
sql: 'select * from test_table limit 1',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer2 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_2 limit 2',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer3 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_3 limit 3',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer4 = {
type: 'mapnik',
options: {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id;'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var mapnikLayerGeomColumn = {
type: 'mapnik',
options: {
sql: 'select *, the_geom as my_geom from test_table_3 limit 2',
geom_column: 'my_geom',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
function typeLayerId(type, index) {
return type + '-' + mapnikBasicLayerId(index);
}
it('with one mapnik layer should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
});
});
it('with two mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
testClient.drain(done);
});
});
it('with three mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2,
mapnikLayer3
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2));
assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3);
testClient.drain(done);
});
});
it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer3,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
assert.ok(!layergroup.metadata.layers[2]);
testClient.drain(done);
});
});
it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[1].type, 'http');
testClient.drain(done);
});
});
it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss);
testClient.drain(done);
});
});
it('should work with different geom_column', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayerGeomColumn
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
testClient.drain(done);
});
});
it('should not include the stats part if the FF is disabled', function(done) {
global.environment.enabledFeatures.layerStats = false;
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.equal(layergroup.metadata.layers[2].type, 'http');
testClient.drain(done);
});
});
});

View File

@@ -1,225 +0,0 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('multilayer stats disabled', function() {
before(function () {
this.layerMetadataConfig = global.environment.enabledFeatures.layerMetadata;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerMetadata = true;
global.environment.enabledFeatures.layerStats = false;
});
after(function () {
global.environment.enabledFeatures.layerMetadata = this.layerMetadataConfig;
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
function testLayerMetadataStats(testScenario) {
it(testScenario.desc, function(done) {
var mapConfig = {
version: '1.3.0',
layers: testScenario.layers
};
var testClient = new TestClient(mapConfig);
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
layergroup.metadata.layers.forEach(function (layer) {
if (layer.type !== 'torque' && layer.type !== 'mapnik') {
assert.ok(!('stats' in layer.meta));
} else if (layer.type !== 'torque') {
assert.ok(!('stats' in layer.meta));
assert.ok('cartocss' in layer.meta);
} else {
assert.ok('cartocss' in layer.meta);
// check torque metadata at least match in number
var torqueLayers = mapConfig.layers.filter(function(layer) { return layer.type === 'torque'; });
if (torqueLayers.length) {
assert.equal(Object.keys(layergroup.metadata.torque).length, torqueLayers.length);
}
}
});
testClient.drain(done);
});
});
}
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var sql = "select 1 as i, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var sqlWadus = "select 1 as wadus, st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 4326) as the_geom, " +
"st_setsrid('LINESTRING(0 0, 1 0)'::geometry, 3857) as the_geom_webmercator";
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var torqueLayer = {
type: 'torque',
options: {
sql: "select 1 id, '1970-01-02'::date d, 'POINT(0 0)'::geometry the_geom_webmercator",
cartocss: [
"Map {",
"-torque-frame-count:2;",
"-torque-resolution:3;",
"-torque-time-attribute:d;",
"-torque-aggregation-function:'count(id)';",
"}"
].join(' '),
cartocss_version: '2.0.1'
}
};
var mapnikLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikInteractivityLayer = {
type: 'mapnik',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var cartodbInteractivityLayer = {
type: 'cartodb',
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var cartodbWadusInteractivityLayer = {
type: 'cartodb',
options: {
sql: sqlWadus,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'wadus'
}
};
var noTypeLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var noTypeInteractivityLayer = {
options: {
sql: sql,
cartocss_version: cartocssVersion,
cartocss: cartocss,
interactivity: 'i'
}
};
var testScenarios = [
{
desc: 'one layer, no interactivity',
layers: [cartodbLayer]
},
{
desc: 'two layers, different interactivity columns',
layers: [
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'torque + interactivity layers',
layers: [
torqueLayer,
cartodbWadusInteractivityLayer,
cartodbInteractivityLayer
]
},
{
desc: 'interactivity + torque + interactivity',
layers: [
cartodbInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'http + interactivity + torque + no interactivity + torque + interactivity',
layers: [
httpLayer,
cartodbInteractivityLayer,
torqueLayer,
cartodbLayer,
torqueLayer,
cartodbWadusInteractivityLayer
]
},
{
desc: 'mapnik type two layers, interactivity mix',
layers: [
mapnikLayer,
mapnikInteractivityLayer
]
},
{
desc: 'mapnik type http + interactivity + torque + interactivity',
layers: [
httpLayer,
mapnikInteractivityLayer,
torqueLayer,
cartodbInteractivityLayer
]
},
{
desc: 'no type two layers, interactivity mix',
layers: [
noTypeLayer,
noTypeInteractivityLayer
]
},
{
desc: 'no type http + interactivity + torque + interactivity',
layers: [
httpLayer,
noTypeInteractivityLayer,
torqueLayer,
noTypeInteractivityLayer
]
}
];
testScenarios.forEach(testLayerMetadataStats);
});

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

View File

@@ -1 +0,0 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[-53839,4629161]},"properties":{"name":"Alicante","cartodb_id":1200}},{"type":"Feature","geometry":{"type":"Point","coordinates":[242835,5069332]},"properties":{"name":"Barcelona","cartodb_id":5330}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-5567,4861644]},"properties":{"name":"Castello","cartodb_id":1201}},{"type":"Feature","geometry":{"type":"Point","coordinates":[272735,5092314]},"properties":{"name":"Mataro","cartodb_id":615}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-125787,4576600]},"properties":{"name":"Murcia","cartodb_id":952}},{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}},{"type":"Feature","geometry":{"type":"Point","coordinates":[139148,5030112]},"properties":{"name":"Tarragona","cartodb_id":616}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-44746,4791667]},"properties":{"name":"Valencia","cartodb_id":5942}},{"type":"Feature","geometry":{"type":"Point","coordinates":[-99072,5108695]},"properties":{"name":"Zaragoza","cartodb_id":5932}}]}

View File

@@ -1 +0,0 @@
{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !! ","!!! !!!!! ","!!!!!!! ! ","!!! !!!!! "," !! ! "," "," "," "," "," "," "," "," ### # "," ####### ###"," ####### ## ","$ ## #### ## ","$$ ","$$ ","$$ "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","9","2","1"],"data":{"1":{"cartodb_id":5942},"2":{"cartodb_id":5500},"9":{"cartodb_id":1201}}}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

View File

@@ -1 +0,0 @@
{"grid":[" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," !!! ! "," !!!!!!! !!!"," !!!!!!! !! "," !! !!!! !! "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "],"keys":["","1"],"data":{"1":{"cartodb_id":5500}}}

View File

@@ -1 +0,0 @@
{"type":"FeatureCollection","features":[{"type":"Feature","geometry":{"type":"Point","coordinates":[295469,4804267]},"properties":{"name":"Palma","cartodb_id":5500}}]}

View File

@@ -126,25 +126,22 @@ assert.response = function(server, req, res, callback) {
// Assert response body
if (res.body) {
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
if (!eql) {
return callback(response, new Error(colorize(
'[red]{Invalid response body.}\n' +
assert.ok(
eql,
colorize('[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}'))
);
}
' Got: [red]{' + response.body + '}')
);
}
// Assert response status
if (typeof status === 'number') {
if (response.statusCode != status) {
return callback(response, new Error(colorize(
'[red]{Invalid response status code.}\n' +
assert.equal(response.statusCode, status,
colorize('[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body))
);
}
' Body: ' + response.body)
);
}
// Assert response headers
@@ -155,13 +152,11 @@ assert.response = function(server, req, res, callback) {
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
if (!headerEql) {
return callback(response, new Error(colorize(
'Invalid response header [bold]{' + name + '}.\n' +
assert.ok(headerEql,
colorize('Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}'))
);
}
' Got: [red]{' + actual + '}')
);
}
}

View File

@@ -76,7 +76,7 @@ if test x"$PREPARE_PGSQL" = xyes; then
createdb -Ttemplate_postgis -EUTF8 "${TEST_DB}" || die "Could not create test database"
LOCAL_SQL_SCRIPTS='analysis_catalog windshaft.test gadm4 ported/populated_places_simple_reduced cdb_analysis_check cdb_invalidate_varnish'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ CDB_EstimateRowCount'
REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews CDB_QuantileBins CDB_JenksBins CDB_HeadsTailsBins CDB_EqualIntervalBins CDB_Hexagon CDB_XYZ'
CURL_ARGS=""
for i in ${REMOTE_SQL_SCRIPTS}

View File

@@ -649,5 +649,3 @@ CREATE OR REPLACE FUNCTION cdb_crankshaft.CDB_KMeans(query text, no_clusters int
END;
$$ LANGUAGE plpgsql;
GRANT ALL ON FUNCTION cdb_crankshaft.CDB_KMeans(text, integer, integer) TO :TESTUSER;
ANALYZE;

View File

@@ -16,23 +16,14 @@ var serverOptions = require('../../lib/cartodb/server_options');
serverOptions.analysis.batch.inlineExecution = true;
var server = new CartodbWindshaft(serverOptions);
function TestClient(config, apiKey) {
this.mapConfig = isMapConfig(config) ? config : null;
this.template = isTemplate(config) ? config : null;
function TestClient(mapConfig, apiKey) {
this.mapConfig = mapConfig;
this.apiKey = apiKey;
this.keysToDelete = {};
}
module.exports = TestClient;
function isMapConfig(config) {
return config && config.layers;
}
function isTemplate(config) {
return config && config.layergroup;
}
module.exports.RESPONSE = {
ERROR: {
status: 400,
@@ -415,7 +406,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
var url = '/api/v1/map';
var urlNamed = url + '/named';
if (this.apiKey) {
url += '?' + qs.stringify({api_key: this.apiKey});
@@ -423,60 +413,17 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
var layergroupId;
step(
function createTemplate () {
function createLayergroup() {
var next = this;
if (!self.template) {
return next();
}
if (!self.apiKey) {
return next(new Error('apiKey param is mandatory to create a new template'));
}
params.placeholders = params.placeholders || {};
assert.response(server,
{
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
url: url,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(self.template)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function (res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).template_id);
}
);
},
function createLayergroup(err, templateId) {
var next = this;
var data = templateId ? params.placeholders : self.mapConfig
var path = templateId ?
urlNamed + '/' + templateId + '?' + qs.stringify({api_key: self.apiKey}) :
url;
assert.response(server,
{
url: path,
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(data)
data: JSON.stringify(self.mapConfig)
},
{
status: 200,
@@ -538,27 +485,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
expectedResponse.headers['Content-Type'] = 'image/png';
}
var isMvt = format.match(/mvt$/);
if (isMvt) {
request.encoding = 'binary';
expectedResponse.headers['Content-Type'] = 'application/x-protobuf';
}
var isGeojson = format.match(/geojson$/);
if (isGeojson) {
request.encoding = 'utf-8';
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
var isGridJSON = format.match(/grid.json$/);
if (isGridJSON) {
request.encoding = 'utf-8';
expectedResponse.headers['Content-Type'] = 'application/json; charset=utf-8';
}
assert.response(server, request, expectedResponse, function(res, err) {
assert.ifError(err);
@@ -566,12 +492,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
if (isPng) {
obj = mapnik.Image.fromBytes(new Buffer(res.body, 'binary'));
}
else if (isMvt) {
obj = new mapnik.VectorTile(z, x, y);
obj.setDataSync(new Buffer(res.body, 'binary'));
}
else {
} else {
obj = JSON.parse(res.body);
}
@@ -617,20 +538,18 @@ TestClient.prototype.getLayergroup = function(expectedResponse, callback) {
},
expectedResponse,
function(res, err) {
// If there is a response, we are still interested in catching the created keys
// to be able to delete them on the .drain() method.
if (res) {
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
}
if (err) {
return callback(err);
}
return callback(null, JSON.parse(res.body));
var parsedBody = JSON.parse(res.body);
if (parsedBody.layergroupid) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
return callback(null, parsedBody);
}
);
};

View File

@@ -1,153 +0,0 @@
var assert = require('assert');
var MapnikLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/mapnik-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
function getDbConnectionMock () {
return {
query: function(sql, callback) {
return callback(null, {
rows: [{rows: 1}]
});
}
};
}
describe('mapnik-layer-stats', function() {
beforeEach(function () {
this.dbConnectionMock = getDbConnectionMock();
this.rendererCacheMock = {};
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
}
]
};
var testMapConfigTwoLayers = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0'
}
},
]
};
var testMapConfigOneLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
]
};
var testMapConfigTwoLayerTwoTables = {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_1', 'test_table_2']
}
},
{
type: 'mapnik',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
affected_tables: ['test_table_3', 'test_table_4']
}
},
]
};
it('should return 1 feature for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayers);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
it('should return 1 feature for one layers with two tables', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayerTwoTables);
var layer = mapConfig.getLayer(0);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer, this.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
it('should return 1 feature for two layers and two tables', function(done) {
var self = this;
var mapConfig = MapConfig.create(testMapConfigTwoLayerTwoTables);
var layer0 = mapConfig.getLayer(0);
var layer1 = mapConfig.getLayer(1);
var testSubject = new MapnikLayerStats();
testSubject.getStats(layer0, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
testSubject.getStats(layer1, self.dbConnectionMock, function (err, result) {
assert.ifError(err);
assert.equal(result.estimatedFeatureCount, 1);
done();
});
});
});
});

View File

@@ -1,36 +0,0 @@
var assert = require('assert');
var TorqueLayerStats = require('../../../../../lib/cartodb/backends/layer-stats/torque-layer-stats');
var MapConfig = require('windshaft').model.MapConfig;
describe('torque-layer-stats', function () {
beforeEach(function () {
this.params = {};
});
var testMapConfigOneLayer = {
version: '1.5.0',
layers: [
{
type: 'torque',
options: {
sql: 'select * from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.3.0',
}
},
]
};
it('should return torque stats for one layer', function(done) {
var mapConfig = MapConfig.create(testMapConfigOneLayer);
var layerId = 0;
var layer = mapConfig.getLayer(layerId);
var testSubject = new TorqueLayerStats();
testSubject.getStats(layer, {}, function (err, result) {
assert.ifError(err);
assert.deepEqual({}, result);
done();
});
});
});

View File

@@ -111,23 +111,6 @@ describe('Bounding box filter', function() {
createRef([-180, -45, 0, 45])
);
});
it('generating multiple bbox', function() {
var bbox = [90, -45, 190, 45];
var bboxFilter = createFilter(bbox);
assert.equal(bboxFilter.bboxes.length, 2);
assert.deepEqual(
bboxFilter.bboxes[0],
createRef([90, -45, 180, 45])
);
assert.deepEqual(
bboxFilter.bboxes[1],
createRef([-180, -45, -170, 45])
);
});
});
describe('out of bounds', function() {

669
yarn.lock

File diff suppressed because it is too large Load Diff