Compare commits
55 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf8ce42049 | ||
|
|
20609bc37e | ||
|
|
b56d110f50 | ||
|
|
34863765ed | ||
|
|
3cb007d147 | ||
|
|
16a7c4fa3d | ||
|
|
3979cda8c2 | ||
|
|
250d52f72c | ||
|
|
26e5b4f404 | ||
|
|
f19c1a34ec | ||
|
|
94c7bc41be | ||
|
|
df0597f12a | ||
|
|
52cb224225 | ||
|
|
baf87e90d7 | ||
|
|
6c3fde70e8 | ||
|
|
d9f6df9815 | ||
|
|
b2539f52b8 | ||
|
|
7c154dd405 | ||
|
|
47dfdf964e | ||
|
|
66aea5e10f | ||
|
|
e0d18e3c20 | ||
|
|
4b79d06ae3 | ||
|
|
4e40a61795 | ||
|
|
2a789b5a5b | ||
|
|
1789993467 | ||
|
|
604b50ffb5 | ||
|
|
2818413c5a | ||
|
|
06164af17f | ||
|
|
6131c4a66a | ||
|
|
465dde7a51 | ||
|
|
7894acf830 | ||
|
|
86c6f6040d | ||
|
|
0036056c07 | ||
|
|
dcf156ba21 | ||
|
|
f0a1e7a0e0 | ||
|
|
21f3c8a387 | ||
|
|
e491c0b825 | ||
|
|
37fcfe69c7 | ||
|
|
850f1cb7f4 | ||
|
|
e67f7b0d0e | ||
|
|
ba8e3d419e | ||
|
|
2d6ee93448 | ||
|
|
cf06ff86c2 | ||
|
|
e1732076fc | ||
|
|
587f66c23d | ||
|
|
3d0c0f34ad | ||
|
|
6ece30fa2c | ||
|
|
19596245b8 | ||
|
|
0e83420e24 | ||
|
|
119846b56b | ||
|
|
33ba629c6d | ||
|
|
9e7b288f44 | ||
|
|
a656285001 | ||
|
|
b7ff554209 | ||
|
|
95ab99be4d |
12
.travis.yml
12
.travis.yml
@@ -1,18 +1,18 @@
|
||||
sudo: false
|
||||
|
||||
addons:
|
||||
postgresql: "9.4"
|
||||
postgresql: "9.3"
|
||||
apt:
|
||||
packages:
|
||||
- postgresql-plpython-9.4
|
||||
- pkg-config
|
||||
- libcairo2-dev
|
||||
- libjpeg8-dev
|
||||
- libgif-dev
|
||||
- pkg-config
|
||||
- libcairo2-dev
|
||||
- libjpeg8-dev
|
||||
- libgif-dev
|
||||
|
||||
before_install:
|
||||
- npm install -g npm@2
|
||||
- createdb template_postgis
|
||||
- createuser publicuser
|
||||
- psql -c "CREATE EXTENSION postgis" template_postgis
|
||||
|
||||
env:
|
||||
|
||||
38
NEWS.md
38
NEWS.md
@@ -1,5 +1,43 @@
|
||||
# Changelog
|
||||
|
||||
## 2.27.0
|
||||
|
||||
Released 2016-03-09
|
||||
|
||||
New features:
|
||||
- Add [Surrogate-Key](https://github.com/CartoDB/cartodb/wiki/CartoDB-Surrogate-Keys) headers to responses
|
||||
|
||||
Enhancements:
|
||||
- Use new `node-cartodb-query-tables` library to obtain affected tables in queries
|
||||
|
||||
Announcements:
|
||||
- Remove deprecated tools directory
|
||||
|
||||
|
||||
## 2.26.3
|
||||
|
||||
Released 2016-03-03
|
||||
|
||||
Improvements:
|
||||
- Optimize overviews queries for efficient spatial filtering in PostgreSQL
|
||||
|
||||
|
||||
## 2.26.2
|
||||
|
||||
Released 2016-02-25
|
||||
|
||||
Announcements:
|
||||
- Upgrades windshaft to [1.13.2](https://github.com/CartoDB/Windshaft/releases/tag/1.13.2)
|
||||
|
||||
|
||||
## 2.26.1
|
||||
|
||||
Released 2016-02-24
|
||||
|
||||
Announcements:
|
||||
- Upgrades windshaft to [1.13.1](https://github.com/CartoDB/Windshaft/releases/tag/1.13.1)
|
||||
|
||||
|
||||
## 2.26.0
|
||||
|
||||
Released 2016-02-24
|
||||
|
||||
@@ -25,8 +25,7 @@ OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, c
|
||||
var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
|
||||
this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
|
||||
if (err){
|
||||
var msg = err.message ? err.message : err;
|
||||
callback(new Error('could not get overviews metadata: ' + msg));
|
||||
callback(err);
|
||||
return;
|
||||
}
|
||||
var metadata = {};
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
function QueryTablesApi(pgQueryRunner) {
|
||||
this.pgQueryRunner = pgQueryRunner;
|
||||
}
|
||||
|
||||
var affectedTableRegexCache = {
|
||||
bbox: /!bbox!/g,
|
||||
scale_denominator: /!scale_denominator!/g,
|
||||
pixel_width: /!pixel_width!/g,
|
||||
pixel_height: /!pixel_height!/g
|
||||
};
|
||||
|
||||
module.exports = QueryTablesApi;
|
||||
|
||||
|
||||
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, sql, callback) {
|
||||
var query = 'SELECT CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$)';
|
||||
|
||||
this.pgQueryRunner.run(username, query, function handleAffectedTablesInQueryRows (err, rows) {
|
||||
if (err){
|
||||
var msg = err.message ? err.message : err;
|
||||
callback(new Error('could not fetch source tables: ' + msg));
|
||||
return;
|
||||
}
|
||||
|
||||
// This is an Array, so no need to split into parts
|
||||
var tableNames = rows[0].cdb_querytablestext;
|
||||
return callback(null, tableNames);
|
||||
});
|
||||
};
|
||||
|
||||
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, sql, callback) {
|
||||
var query = [
|
||||
'WITH querytables AS (',
|
||||
'SELECT * FROM CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$) as tablenames',
|
||||
')',
|
||||
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
|
||||
'FROM CDB_TableMetadata m',
|
||||
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
|
||||
].join(' ');
|
||||
|
||||
this.pgQueryRunner.run(username, query, function handleAffectedTablesAndLastUpdatedTimeRows (err, rows) {
|
||||
if (err || rows.length === 0) {
|
||||
var msg = err.message ? err.message : err;
|
||||
callback(new Error('could not fetch affected tables or last updated time: ' + msg));
|
||||
return;
|
||||
}
|
||||
|
||||
var result = rows[0];
|
||||
|
||||
// This is an Array, so no need to split into parts
|
||||
var tableNames = result.tablenames;
|
||||
|
||||
var lastUpdatedTime = result.max || 0;
|
||||
|
||||
callback(null, {
|
||||
affectedTables: tableNames,
|
||||
lastUpdatedTime: lastUpdatedTime * 1000
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
QueryTablesApi.prototype.getLastUpdatedTime = function (username, tableNames, callback) {
|
||||
if (!Array.isArray(tableNames) || tableNames.length === 0) {
|
||||
return callback(null, 0);
|
||||
}
|
||||
|
||||
var query = [
|
||||
'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max',
|
||||
'FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY[',
|
||||
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(','),
|
||||
'])'
|
||||
].join(' ');
|
||||
|
||||
this.pgQueryRunner.run(username, query, function handleLastUpdatedTimeRows (err, rows) {
|
||||
if (err) {
|
||||
var msg = err.message ? err.message : err;
|
||||
return callback(new Error('could not fetch affected tables or last updated time: ' + msg));
|
||||
}
|
||||
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
|
||||
var lastUpdated = 0;
|
||||
if (rows.length !== 0) {
|
||||
lastUpdated = rows[0].max || 0;
|
||||
}
|
||||
|
||||
return callback(null, lastUpdated*1000);
|
||||
});
|
||||
};
|
||||
|
||||
function prepareSql(sql) {
|
||||
return sql
|
||||
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
|
||||
.replace(affectedTableRegexCache.scale_denominator, '0')
|
||||
.replace(affectedTableRegexCache.pixel_width, '1')
|
||||
.replace(affectedTableRegexCache.pixel_height, '1')
|
||||
;
|
||||
}
|
||||
@@ -13,13 +13,9 @@ module.exports = TablesExtentApi;
|
||||
* `table_name` format as valid input
|
||||
* @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
|
||||
*/
|
||||
TablesExtentApi.prototype.getBounds = function (username, tableNames, callback) {
|
||||
var estimatedExtentSQLs = tableNames.map(function(tableName) {
|
||||
var schemaTable = tableName.split('.');
|
||||
if (schemaTable.length > 1) {
|
||||
return "ST_EstimatedExtent('" + schemaTable[0] + "', '" + schemaTable[1] + "', 'the_geom_webmercator')";
|
||||
}
|
||||
return "ST_EstimatedExtent('" + schemaTable[0] + "', 'the_geom_webmercator')";
|
||||
TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
|
||||
var estimatedExtentSQLs = tables.map(function(table) {
|
||||
return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
|
||||
});
|
||||
|
||||
var query = [
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
var assert = require('assert');
|
||||
var step = require('step');
|
||||
var PSQL = require('cartodb-psql');
|
||||
var _ = require('underscore');
|
||||
|
||||
function PgConnection(metadataBackend) {
|
||||
@@ -99,3 +100,37 @@ PgConnection.prototype.setDBConn = function(dbowner, params, callback) {
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Returns a `cartodb-psql` object for a given username.
|
||||
* @param {String} username
|
||||
* @param {Function} callback function({Error}, {PSQL})
|
||||
*/
|
||||
|
||||
PgConnection.prototype.getConnection = function(username, callback) {
|
||||
var self = this;
|
||||
|
||||
var params = {};
|
||||
|
||||
require('debug')('cachechan')("getConn1");
|
||||
step(
|
||||
function setAuth() {
|
||||
self.setDBAuth(username, params, this);
|
||||
},
|
||||
function setConn(err) {
|
||||
assert.ifError(err);
|
||||
self.setDBConn(username, params, this);
|
||||
},
|
||||
function openConnection(err) {
|
||||
assert.ifError(err);
|
||||
return callback(err, new PSQL({
|
||||
user: params.dbuser,
|
||||
pass: params.dbpass,
|
||||
host: params.dbhost,
|
||||
port: params.dbport,
|
||||
dbname: params.dbname
|
||||
}));
|
||||
}
|
||||
);
|
||||
};
|
||||
|
||||
24
lib/cartodb/cache/model/database_tables_entry.js
vendored
24
lib/cartodb/cache/model/database_tables_entry.js
vendored
@@ -1,24 +0,0 @@
|
||||
var crypto = require('crypto');
|
||||
|
||||
function DatabaseTables(dbName, tableNames) {
|
||||
this.namespace = 't';
|
||||
this.dbName = dbName;
|
||||
this.tableNames = tableNames;
|
||||
}
|
||||
|
||||
module.exports = DatabaseTables;
|
||||
|
||||
|
||||
DatabaseTables.prototype.key = function() {
|
||||
return this.tableNames.map(function(tableName) {
|
||||
return this.namespace + ':' + shortHashKey(this.dbName + ':' + tableName);
|
||||
}.bind(this));
|
||||
};
|
||||
|
||||
DatabaseTables.prototype.getCacheChannel = function() {
|
||||
return this.dbName + ':' + this.tableNames.join(',');
|
||||
};
|
||||
|
||||
function shortHashKey(target) {
|
||||
return crypto.createHash('sha256').update(target).digest('base64').substring(0,6);
|
||||
}
|
||||
@@ -7,11 +7,10 @@ var queue = require('queue-async');
|
||||
|
||||
var LruCache = require("lru-cache");
|
||||
|
||||
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi) {
|
||||
function NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi) {
|
||||
this.templateMaps = templateMaps;
|
||||
this.pgConnection = pgConnection;
|
||||
this.userLimitsApi = userLimitsApi;
|
||||
this.queryTablesApi = queryTablesApi;
|
||||
|
||||
this.namedLayersAdapter = new MapConfigNamedLayersAdapter(templateMaps);
|
||||
|
||||
@@ -30,7 +29,6 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
|
||||
this.templateMaps,
|
||||
this.pgConnection,
|
||||
this.userLimitsApi,
|
||||
this.queryTablesApi,
|
||||
this.namedLayersAdapter,
|
||||
user,
|
||||
templateId,
|
||||
|
||||
@@ -8,7 +8,8 @@ var cors = require('../middleware/cors');
|
||||
var userMiddleware = require('../middleware/user');
|
||||
|
||||
var MapStoreMapConfigProvider = require('../models/mapconfig/map_store_provider');
|
||||
var TablesCacheEntry = require('../cache/model/database_tables_entry');
|
||||
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
|
||||
/**
|
||||
* @param {AuthApi} authApi
|
||||
@@ -20,14 +21,14 @@ var TablesCacheEntry = require('../cache/model/database_tables_entry');
|
||||
* @param {WidgetBackend} widgetBackend
|
||||
* @param {SurrogateKeysCache} surrogateKeysCache
|
||||
* @param {UserLimitsApi} userLimitsApi
|
||||
* @param {QueryTablesApi} queryTablesApi
|
||||
* @param {LayergroupAffectedTables} layergroupAffectedTables
|
||||
* @constructor
|
||||
*/
|
||||
function LayergroupController(authApi, pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
|
||||
widgetBackend, surrogateKeysCache, userLimitsApi, queryTablesApi, layergroupAffectedTables) {
|
||||
widgetBackend, surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
|
||||
BaseController.call(this, authApi, pgConnection);
|
||||
|
||||
this.pgConnection = pgConnection;
|
||||
this.mapStore = mapStore;
|
||||
this.tileBackend = tileBackend;
|
||||
this.previewBackend = previewBackend;
|
||||
@@ -35,7 +36,6 @@ function LayergroupController(authApi, pgConnection, mapStore, tileBackend, prev
|
||||
this.widgetBackend = widgetBackend;
|
||||
this.surrogateKeysCache = surrogateKeysCache;
|
||||
this.userLimitsApi = userLimitsApi;
|
||||
this.queryTablesApi = queryTablesApi;
|
||||
this.layergroupAffectedTables = layergroupAffectedTables;
|
||||
}
|
||||
|
||||
@@ -320,9 +320,8 @@ LayergroupController.prototype.sendResponse = function(req, res, body, status, h
|
||||
global.logger.warn('ERROR generating cache channel: ' + err);
|
||||
}
|
||||
if (!!affectedTables) {
|
||||
var tablesCacheEntry = new TablesCacheEntry(dbName, affectedTables);
|
||||
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
|
||||
self.surrogateKeysCache.tag(res, tablesCacheEntry);
|
||||
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
|
||||
self.surrogateKeysCache.tag(res, affectedTables);
|
||||
}
|
||||
self.send(req, res, body, status, headers);
|
||||
}
|
||||
@@ -366,17 +365,24 @@ LayergroupController.prototype.getAffectedTables = function(user, dbName, layerg
|
||||
throw new Error("this request doesn't need an X-Cache-Channel generated");
|
||||
}
|
||||
|
||||
self.queryTablesApi.getAffectedTablesInQuery(user, sql, this); // in addCacheChannel
|
||||
step(
|
||||
function getConnection() {
|
||||
self.pgConnection.getConnection(user, this);
|
||||
},
|
||||
function getAffectedTables(err, connection) {
|
||||
assert.ifError(err);
|
||||
|
||||
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
|
||||
},
|
||||
this
|
||||
);
|
||||
},
|
||||
function buildCacheChannel(err, tableNames) {
|
||||
function buildCacheChannel(err, tables) {
|
||||
assert.ifError(err);
|
||||
self.layergroupAffectedTables.set(dbName, layergroupId, tables);
|
||||
|
||||
self.layergroupAffectedTables.set(dbName, layergroupId, tableNames);
|
||||
|
||||
return tableNames;
|
||||
return tables;
|
||||
},
|
||||
function finish(err, affectedTables) {
|
||||
callback(err, affectedTables);
|
||||
}
|
||||
callback
|
||||
);
|
||||
};
|
||||
|
||||
@@ -2,6 +2,7 @@ var _ = require('underscore');
|
||||
var assert = require('assert');
|
||||
var step = require('step');
|
||||
var windshaft = require('windshaft');
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
|
||||
var util = require('util');
|
||||
var BaseController = require('./base');
|
||||
@@ -13,7 +14,6 @@ var MapConfig = windshaft.model.MapConfig;
|
||||
var Datasource = windshaft.model.Datasource;
|
||||
|
||||
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
|
||||
var TablesCacheEntry = require('../cache/model/database_tables_entry');
|
||||
|
||||
var MapConfigNamedLayersAdapter = require('../models/mapconfig_named_layers_adapter');
|
||||
var NamedMapMapConfigProvider = require('../models/mapconfig/named_map_provider');
|
||||
@@ -26,7 +26,6 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
|
||||
* @param {TemplateMaps} templateMaps
|
||||
* @param {MapBackend} mapBackend
|
||||
* @param metadataBackend
|
||||
* @param {QueryTablesApi} queryTablesApi
|
||||
* @param {OverviewsMetadataApi} overviewsMetadataApi
|
||||
* @param {SurrogateKeysCache} surrogateKeysCache
|
||||
* @param {UserLimitsApi} userLimitsApi
|
||||
@@ -34,7 +33,7 @@ var MapConfigOverviewsAdapter = require('../models/mapconfig_overviews_adapter')
|
||||
* @constructor
|
||||
*/
|
||||
function MapController(authApi, pgConnection, templateMaps, mapBackend, metadataBackend,
|
||||
queryTablesApi, overviewsMetadataApi,
|
||||
overviewsMetadataApi,
|
||||
surrogateKeysCache, userLimitsApi, layergroupAffectedTables) {
|
||||
|
||||
BaseController.call(this, authApi, pgConnection);
|
||||
@@ -43,7 +42,6 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata
|
||||
this.templateMaps = templateMaps;
|
||||
this.mapBackend = mapBackend;
|
||||
this.metadataBackend = metadataBackend;
|
||||
this.queryTablesApi = queryTablesApi;
|
||||
this.overviewsMetadataApi = overviewsMetadataApi;
|
||||
this.surrogateKeysCache = surrogateKeysCache;
|
||||
this.userLimitsApi = userLimitsApi;
|
||||
@@ -216,7 +214,6 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn
|
||||
self.templateMaps,
|
||||
self.pgConnection,
|
||||
self.userLimitsApi,
|
||||
self.queryTablesApi,
|
||||
self.namedLayersAdapter,
|
||||
cdbuser,
|
||||
req.params.template_id,
|
||||
@@ -318,43 +315,34 @@ MapController.prototype.afterLayergroupCreate = function(req, res, mapconfig, la
|
||||
var layergroupId = layergroup.layergroupid;
|
||||
|
||||
step(
|
||||
function checkCachedAffectedTables() {
|
||||
return self.layergroupAffectedTables.hasAffectedTables(dbName, layergroupId);
|
||||
function getPgConnection() {
|
||||
self.pgConnection.getConnection(username, this);
|
||||
},
|
||||
function getAffectedTablesAndLastUpdatedTime(err, hasCache) {
|
||||
function getAffectedTablesAndLastUpdatedTime(err, connection) {
|
||||
assert.ifError(err);
|
||||
if (hasCache) {
|
||||
var next = this;
|
||||
var affectedTables = self.layergroupAffectedTables.get(dbName, layergroupId);
|
||||
self.queryTablesApi.getLastUpdatedTime(username, affectedTables, function(err, lastUpdatedTime) {
|
||||
if (err) {
|
||||
return next(err);
|
||||
}
|
||||
return next(null, { affectedTables: affectedTables, lastUpdatedTime: lastUpdatedTime });
|
||||
});
|
||||
} else {
|
||||
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(username, sql, this);
|
||||
}
|
||||
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
|
||||
},
|
||||
function handleAffectedTablesAndLastUpdatedTime(err, result) {
|
||||
if (req.profiler) {
|
||||
req.profiler.done('queryTablesAndLastUpdated');
|
||||
}
|
||||
assert.ifError(err);
|
||||
self.layergroupAffectedTables.set(dbName, layergroupId, result.affectedTables);
|
||||
// feed affected tables cache so it can be reused from, for instance, layergroup controller
|
||||
self.layergroupAffectedTables.set(dbName, layergroupId, result);
|
||||
|
||||
// last update for layergroup cache buster
|
||||
layergroup.layergroupid = layergroup.layergroupid + ':' + result.lastUpdatedTime;
|
||||
layergroup.last_updated = new Date(result.lastUpdatedTime).toISOString();
|
||||
layergroup.layergroupid = layergroup.layergroupid + ':' + result.getLastUpdatedAt();
|
||||
layergroup.last_updated = new Date(result.getLastUpdatedAt()).toISOString();
|
||||
|
||||
// TODO this should take into account several URL patterns
|
||||
addWidgetsUrl(username, layergroup);
|
||||
if (req.method === 'GET') {
|
||||
var tableCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
|
||||
var ttl = global.environment.varnish.layergroupTtl || 86400;
|
||||
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
|
||||
res.set('Last-Modified', (new Date()).toUTCString());
|
||||
res.set('X-Cache-Channel', tableCacheEntry.getCacheChannel());
|
||||
if (result.affectedTables && result.affectedTables.length > 0) {
|
||||
self.surrogateKeysCache.tag(res, tableCacheEntry);
|
||||
res.set('X-Cache-Channel', result.getCacheChannel());
|
||||
if (result.tables && result.tables.length > 0) {
|
||||
self.surrogateKeysCache.tag(res, result);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -9,8 +9,6 @@ var BaseController = require('./base');
|
||||
var cors = require('../middleware/cors');
|
||||
var userMiddleware = require('../middleware/user');
|
||||
|
||||
var TablesCacheEntry = require('../cache/model/database_tables_entry');
|
||||
|
||||
function NamedMapsController(authApi, pgConnection, namedMapProviderCache, tileBackend, previewBackend,
|
||||
surrogateKeysCache, tablesExtentApi, metadataBackend) {
|
||||
BaseController.call(this, authApi, pgConnection);
|
||||
@@ -44,7 +42,6 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
|
||||
|
||||
var self = this;
|
||||
|
||||
var dbName = req.params.dbname;
|
||||
step(
|
||||
function getAffectedTablesAndLastUpdatedTime() {
|
||||
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
|
||||
@@ -54,22 +51,21 @@ NamedMapsController.prototype.sendResponse = function(req, res, resource, header
|
||||
if (err) {
|
||||
global.logger.log('ERROR generating cache channel: ' + err);
|
||||
}
|
||||
if (!result || !!result.affectedTables) {
|
||||
if (!result || !!result.tables) {
|
||||
// we increase cache control as we can invalidate it
|
||||
res.set('Cache-Control', 'public,max-age=31536000');
|
||||
|
||||
var lastModifiedDate;
|
||||
if (Number.isFinite(result.lastUpdatedTime)) {
|
||||
lastModifiedDate = new Date(result.lastUpdatedTime);
|
||||
lastModifiedDate = new Date(result.getLastUpdatedAt());
|
||||
} else {
|
||||
lastModifiedDate = new Date();
|
||||
}
|
||||
res.set('Last-Modified', lastModifiedDate.toUTCString());
|
||||
|
||||
var tablesCacheEntry = new TablesCacheEntry(dbName, result.affectedTables);
|
||||
res.set('X-Cache-Channel', tablesCacheEntry.getCacheChannel());
|
||||
if (result.affectedTables.length > 0) {
|
||||
self.surrogateKeysCache.tag(res, tablesCacheEntry);
|
||||
res.set('X-Cache-Channel', result.getCacheChannel());
|
||||
if (result.tables.length > 0) {
|
||||
self.surrogateKeysCache.tag(res, result);
|
||||
}
|
||||
}
|
||||
self.send(req, res, resource, 200);
|
||||
@@ -231,7 +227,7 @@ NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, namedMap
|
||||
return next(null);
|
||||
}
|
||||
|
||||
var affectedTables = affectedTablesAndLastUpdate.affectedTables || [];
|
||||
var affectedTables = affectedTablesAndLastUpdate.tables || [];
|
||||
|
||||
if (affectedTables.length === 0) {
|
||||
return next(null);
|
||||
|
||||
@@ -5,17 +5,17 @@ var dot = require('dot');
|
||||
var step = require('step');
|
||||
var MapConfig = require('windshaft').model.MapConfig;
|
||||
var templateName = require('../../backends/template_maps').templateName;
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
|
||||
/**
|
||||
* @constructor
|
||||
* @type {NamedMapMapConfigProvider}
|
||||
*/
|
||||
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, queryTablesApi, namedLayersAdapter,
|
||||
function NamedMapMapConfigProvider(templateMaps, pgConnection, userLimitsApi, namedLayersAdapter,
|
||||
owner, templateId, config, authToken, params) {
|
||||
this.templateMaps = templateMaps;
|
||||
this.pgConnection = pgConnection;
|
||||
this.userLimitsApi = userLimitsApi;
|
||||
this.queryTablesApi = queryTablesApi;
|
||||
this.namedLayersAdapter = namedLayersAdapter;
|
||||
|
||||
this.owner = owner;
|
||||
@@ -256,7 +256,16 @@ NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = functi
|
||||
},
|
||||
function getAffectedTables(err, sql) {
|
||||
assert.ifError(err);
|
||||
self.queryTablesApi.getAffectedTablesAndLastUpdatedTime(self.owner, sql, this);
|
||||
step(
|
||||
function getConnection() {
|
||||
self.pgConnection.getConnection(self.owner, this);
|
||||
},
|
||||
function getAffectedTables(err, connection) {
|
||||
assert.ifError(err);
|
||||
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
|
||||
},
|
||||
this
|
||||
);
|
||||
},
|
||||
function finish(err, result) {
|
||||
self.affectedTablesAndLastUpdate = result;
|
||||
|
||||
@@ -19,7 +19,6 @@ var windshaft = require('windshaft');
|
||||
var mapnik = windshaft.mapnik;
|
||||
|
||||
var TemplateMaps = require('./backends/template_maps.js');
|
||||
var QueryTablesApi = require('./api/query_tables_api');
|
||||
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
|
||||
var UserLimitsApi = require('./api/user_limits_api');
|
||||
var AuthApi = require('./api/auth_api');
|
||||
@@ -52,7 +51,6 @@ module.exports = function(serverOptions) {
|
||||
var metadataBackend = cartodbRedis({pool: redisPool});
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
var pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
var queryTablesApi = new QueryTablesApi(pgQueryRunner);
|
||||
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
|
||||
var userLimitsApi = new UserLimitsApi(metadataBackend, {
|
||||
limits: {
|
||||
@@ -142,7 +140,7 @@ module.exports = function(serverOptions) {
|
||||
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
|
||||
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
|
||||
|
||||
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi, queryTablesApi);
|
||||
var namedMapProviderCache = new NamedMapProviderCache(templateMaps, pgConnection, userLimitsApi);
|
||||
['update', 'delete'].forEach(function(eventType) {
|
||||
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
|
||||
});
|
||||
@@ -166,7 +164,6 @@ module.exports = function(serverOptions) {
|
||||
new windshaft.backend.Widget(),
|
||||
surrogateKeysCache,
|
||||
userLimitsApi,
|
||||
queryTablesApi,
|
||||
layergroupAffectedTablesCache
|
||||
).register(app);
|
||||
|
||||
@@ -176,7 +173,6 @@ module.exports = function(serverOptions) {
|
||||
templateMaps,
|
||||
mapBackend,
|
||||
metadataBackend,
|
||||
queryTablesApi,
|
||||
overviewsMetadataApi,
|
||||
surrogateKeysCache,
|
||||
userLimitsApi,
|
||||
|
||||
@@ -74,14 +74,9 @@ function overviews_view_name(table) {
|
||||
}
|
||||
|
||||
// replace a table name in a query by anoter name
|
||||
function replace_table_in_query(sql, old_table_name, new_table_name) {
|
||||
function replace_table_in_query(sql, old_table_name, replacement) {
|
||||
var old_table = TableNameParser.parse(old_table_name);
|
||||
var new_table = TableNameParser.parse(new_table_name);
|
||||
var old_table_ident = TableNameParser.table_identifier(old_table);
|
||||
var new_table_ident = TableNameParser.table_identifier(new_table);
|
||||
|
||||
// text that will be substituted by the table name pattern
|
||||
var replacement = new_table_ident;
|
||||
|
||||
// regular expression prefix (beginning) to match a table name
|
||||
function pattern_prefix(schema, identifier) {
|
||||
@@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
|
||||
function overviews_query(query, overviews, zoom_level_expression) {
|
||||
var replaced_query = query;
|
||||
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
|
||||
var replacement;
|
||||
for ( var table in overviews ) {
|
||||
if (overviews.hasOwnProperty(table)) {
|
||||
var table_overviews = overviews[table];
|
||||
var table_view = overviews_view_name(table);
|
||||
replaced_query = replace_table_in_query(replaced_query, table, table_view);
|
||||
sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )";
|
||||
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
|
||||
replaced_query = replace_table_in_query(replaced_query, table, replacement);
|
||||
}
|
||||
}
|
||||
if ( replaced_query !== query ) {
|
||||
@@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
|
||||
|
||||
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
|
||||
return !!sql.match(
|
||||
/^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
|
||||
/^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
|
||||
);
|
||||
};
|
||||
|
||||
|
||||
96
npm-shrinkwrap.json
generated
96
npm-shrinkwrap.json
generated
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "2.26.0",
|
||||
"version": "2.27.0",
|
||||
"dependencies": {
|
||||
"body-parser": {
|
||||
"version": "1.14.2",
|
||||
@@ -85,7 +85,7 @@
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.10",
|
||||
"from": "mime-types@>=2.1.2 <2.2.0",
|
||||
"from": "mime-types@>=2.1.9 <2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
|
||||
"dependencies": {
|
||||
"mime-db": {
|
||||
@@ -123,6 +123,11 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"cartodb-query-tables": {
|
||||
"version": "0.1.0",
|
||||
"from": "cartodb-query-tables@>=0.1.0 <0.2.0",
|
||||
"resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.1.0.tgz"
|
||||
},
|
||||
"cartodb-redis": {
|
||||
"version": "0.13.0",
|
||||
"from": "cartodb-redis@>=0.13.0 <0.14.0",
|
||||
@@ -193,7 +198,7 @@
|
||||
"dependencies": {
|
||||
"mime-types": {
|
||||
"version": "2.1.10",
|
||||
"from": "mime-types@>=2.1.9 <2.2.0",
|
||||
"from": "mime-types@>=2.1.6 <2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
|
||||
"dependencies": {
|
||||
"mime-db": {
|
||||
@@ -382,7 +387,7 @@
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.10",
|
||||
"from": "mime-types@>=2.1.9 <2.2.0",
|
||||
"from": "mime-types@>=2.1.6 <2.2.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.10.tgz",
|
||||
"dependencies": {
|
||||
"mime-db": {
|
||||
@@ -784,9 +789,9 @@
|
||||
}
|
||||
},
|
||||
"is-my-json-valid": {
|
||||
"version": "2.13.1",
|
||||
"version": "2.13.0",
|
||||
"from": "is-my-json-valid@>=2.12.0 <3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/is-my-json-valid/-/is-my-json-valid-2.13.0.tgz",
|
||||
"dependencies": {
|
||||
"generate-function": {
|
||||
"version": "2.0.0",
|
||||
@@ -837,9 +842,9 @@
|
||||
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
|
||||
},
|
||||
"windshaft": {
|
||||
"version": "1.13.0",
|
||||
"from": "windshaft@1.13.0",
|
||||
"resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.13.0.tgz",
|
||||
"version": "1.13.2",
|
||||
"from": "windshaft@1.13.2",
|
||||
"resolved": "https://registry.npmjs.org/windshaft/-/windshaft-1.13.2.tgz",
|
||||
"dependencies": {
|
||||
"abaculus": {
|
||||
"version": "1.1.0-cdb4",
|
||||
@@ -3732,6 +3737,7 @@
|
||||
"turbo-cartocss": {
|
||||
"version": "0.4.0",
|
||||
"from": "turbo-cartocss@>=0.4.0 <0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/turbo-cartocss/-/turbo-cartocss-0.4.0.tgz",
|
||||
"dependencies": {
|
||||
"browser-request": {
|
||||
"version": "0.3.3",
|
||||
@@ -3749,9 +3755,9 @@
|
||||
"resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-3.1.2.tgz"
|
||||
},
|
||||
"less": {
|
||||
"version": "2.6.0",
|
||||
"version": "2.6.1",
|
||||
"from": "less@>=2.5.3 <3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/less/-/less-2.6.0.tgz",
|
||||
"resolved": "https://registry.npmjs.org/less/-/less-2.6.1.tgz",
|
||||
"dependencies": {
|
||||
"errno": {
|
||||
"version": "0.1.4",
|
||||
@@ -3766,14 +3772,14 @@
|
||||
}
|
||||
},
|
||||
"graceful-fs": {
|
||||
"version": "3.0.8",
|
||||
"from": "graceful-fs@>=3.0.5 <4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-3.0.8.tgz"
|
||||
"version": "4.1.3",
|
||||
"from": "graceful-fs@>=4.1.2 <5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.3.tgz"
|
||||
},
|
||||
"image-size": {
|
||||
"version": "0.3.5",
|
||||
"from": "image-size@>=0.3.5 <0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/image-size/-/image-size-0.3.5.tgz"
|
||||
"version": "0.4.0",
|
||||
"from": "image-size@>=0.4.0 <0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/image-size/-/image-size-0.4.0.tgz"
|
||||
},
|
||||
"mime": {
|
||||
"version": "1.3.4",
|
||||
@@ -3793,35 +3799,28 @@
|
||||
}
|
||||
},
|
||||
"promise": {
|
||||
"version": "6.1.0",
|
||||
"from": "promise@>=6.0.1 <7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/promise/-/promise-6.1.0.tgz",
|
||||
"version": "7.1.1",
|
||||
"from": "promise@>=7.1.1 <8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/promise/-/promise-7.1.1.tgz",
|
||||
"dependencies": {
|
||||
"asap": {
|
||||
"version": "1.0.0",
|
||||
"from": "asap@>=1.0.0 <1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/asap/-/asap-1.0.0.tgz"
|
||||
"version": "2.0.3",
|
||||
"from": "asap@>=2.0.3 <2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/asap/-/asap-2.0.3.tgz"
|
||||
}
|
||||
}
|
||||
},
|
||||
"source-map": {
|
||||
"version": "0.4.4",
|
||||
"from": "source-map@>=0.4.2 <0.5.0",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
|
||||
"dependencies": {
|
||||
"amdefine": {
|
||||
"version": "1.0.0",
|
||||
"from": "amdefine@>=0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.0.tgz"
|
||||
}
|
||||
}
|
||||
"version": "0.5.3",
|
||||
"from": "source-map@>=0.5.3 <0.6.0",
|
||||
"resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.3.tgz"
|
||||
}
|
||||
}
|
||||
},
|
||||
"postcss": {
|
||||
"version": "5.0.16",
|
||||
"version": "5.0.19",
|
||||
"from": "postcss@>=5.0.13 <6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-5.0.16.tgz",
|
||||
"resolved": "https://registry.npmjs.org/postcss/-/postcss-5.0.19.tgz",
|
||||
"dependencies": {
|
||||
"supports-color": {
|
||||
"version": "3.1.2",
|
||||
@@ -3868,9 +3867,28 @@
|
||||
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.6.0.tgz"
|
||||
},
|
||||
"aws4": {
|
||||
"version": "1.2.1",
|
||||
"version": "1.3.2",
|
||||
"from": "aws4@>=1.2.1 <2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.2.1.tgz"
|
||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.3.2.tgz",
|
||||
"dependencies": {
|
||||
"lru-cache": {
|
||||
"version": "4.0.0",
|
||||
"from": "lru-cache@>=4.0.0 <5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.0.0.tgz",
|
||||
"dependencies": {
|
||||
"pseudomap": {
|
||||
"version": "1.0.2",
|
||||
"from": "pseudomap@>=1.0.1 <2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz"
|
||||
},
|
||||
"yallist": {
|
||||
"version": "2.0.0",
|
||||
"from": "yallist@>=2.0.0 <3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-2.0.0.tgz"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"bl": {
|
||||
"version": "1.0.3",
|
||||
@@ -4160,9 +4178,9 @@
|
||||
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.0.tgz"
|
||||
},
|
||||
"tweetnacl": {
|
||||
"version": "0.14.0",
|
||||
"version": "0.14.1",
|
||||
"from": "tweetnacl@>=0.13.0 <1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.0.tgz"
|
||||
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.1.tgz"
|
||||
},
|
||||
"jodid25519": {
|
||||
"version": "1.0.2",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"private": true,
|
||||
"name": "windshaft-cartodb",
|
||||
"version": "2.26.0",
|
||||
"version": "2.27.0",
|
||||
"description": "A map tile server for CartoDB",
|
||||
"keywords": [
|
||||
"cartodb"
|
||||
@@ -26,7 +26,7 @@
|
||||
"node-statsd": "~0.0.7",
|
||||
"underscore" : "~1.6.0",
|
||||
"dot": "~1.0.2",
|
||||
"windshaft": "1.13.0",
|
||||
"windshaft": "1.13.2",
|
||||
"step": "~0.0.6",
|
||||
"queue-async": "~1.0.7",
|
||||
"request": "~2.62.0",
|
||||
@@ -36,7 +36,8 @@
|
||||
"redis-mpool": "~0.4.0",
|
||||
"lru-cache": "2.6.5",
|
||||
"lzma": "~1.3.7",
|
||||
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb"
|
||||
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
|
||||
"cartodb-query-tables": "~0.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"istanbul": "~0.3.6",
|
||||
|
||||
@@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
|
||||
var server = new CartodbWindshaft(serverOptions);
|
||||
server.setMaxListeners(0);
|
||||
|
||||
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
|
||||
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
|
||||
|
||||
@@ -274,9 +274,9 @@ describe(suiteName, function() {
|
||||
var parsedBody = JSON.parse(res.body);
|
||||
expected_token = parsedBody.layergroupid.split(':')[0];
|
||||
helper.checkCache(res);
|
||||
helper.checkSurrogateKey(res, new TablesCacheEntry('test_windshaft_cartodb_user_1_db', [
|
||||
'public.test_table',
|
||||
'public.test_table_2'
|
||||
helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
|
||||
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
|
||||
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
|
||||
]).key().join(' '));
|
||||
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ var _ = require('underscore');
|
||||
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
|
||||
|
||||
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
var CartodbWindshaft = require('../../lib/cartodb/server');
|
||||
var serverOptions = require('../../lib/cartodb/server_options');
|
||||
var server = new CartodbWindshaft(serverOptions);
|
||||
@@ -70,7 +71,8 @@ describe('tests from old api translated to multilayer', function() {
|
||||
},
|
||||
function(res) {
|
||||
var parsed = JSON.parse(res.body);
|
||||
assert.ok(parsed.errors[0].match(/<css input>:1:1: Unclosed block/));
|
||||
assert.ok(parsed.errors[0].match(/^style0/));
|
||||
assert.ok(parsed.errors[0].match(/missing closing/));
|
||||
done();
|
||||
}
|
||||
);
|
||||
@@ -333,7 +335,7 @@ describe('tests from old api translated to multilayer', function() {
|
||||
|
||||
var parsed = JSON.parse(res.body);
|
||||
assert.deepEqual(parsed, {
|
||||
errors: ["could not get overviews metadata: fake error message"]
|
||||
errors: ["fake error message"]
|
||||
});
|
||||
|
||||
done();
|
||||
@@ -359,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
|
||||
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
|
||||
keysToDelete['user:localhost:mapviews:global'] = 5;
|
||||
|
||||
var runQueryFn = PgQueryRunner.prototype.run;
|
||||
PgQueryRunner.prototype.run = function(username, query, callback) {
|
||||
return callback(new Error('failed to query database for affected tables'), []);
|
||||
var affectedFn = QueryTables.getAffectedTablesFromQuery;
|
||||
QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
|
||||
affectedFn({query: function(query, callback) {
|
||||
return callback(new Error('fake error message'), []);
|
||||
}}, username, query, callback);
|
||||
};
|
||||
|
||||
// reset internal cacheChannel cache
|
||||
@@ -386,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
|
||||
},
|
||||
function(res) {
|
||||
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
|
||||
PgQueryRunner.prototype.run = runQueryFn;
|
||||
QueryTables.getAffectedTablesFromQuery = affectedFn;
|
||||
done();
|
||||
}
|
||||
);
|
||||
|
||||
@@ -3,8 +3,8 @@ var _ = require('underscore');
|
||||
var redis = require('redis');
|
||||
var step = require('step');
|
||||
var strftime = require('strftime');
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
|
||||
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
|
||||
var redis_stats_db = 5;
|
||||
|
||||
// Pollute the PG environment to make sure
|
||||
@@ -1405,7 +1405,8 @@ describe('template_api', function() {
|
||||
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
|
||||
helper.checkCache(res);
|
||||
var expectedSurrogateKey = [
|
||||
new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
|
||||
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
|
||||
table_name: 'test_table_private_1'}]).key(),
|
||||
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
|
||||
].join(' ');
|
||||
helper.checkSurrogateKey(res, expectedSurrogateKey);
|
||||
@@ -1488,7 +1489,8 @@ describe('template_api', function() {
|
||||
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
|
||||
helper.checkCache(res);
|
||||
var expectedSurrogateKey = [
|
||||
new TablesCacheEntry('test_windshaft_cartodb_user_1_db', ['public.test_table_private_1']).key(),
|
||||
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
|
||||
table_name: 'test_table_private_1'}]).key(),
|
||||
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
|
||||
].join(' ');
|
||||
helper.checkSurrogateKey(res, expectedSurrogateKey);
|
||||
|
||||
@@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis');
|
||||
|
||||
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
|
||||
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
|
||||
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
|
||||
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
|
||||
|
||||
|
||||
describe('OverviewsMetadataApi', function() {
|
||||
|
||||
var queryTablesApi, overviewsMetadataApi;
|
||||
var overviewsMetadataApi;
|
||||
|
||||
before(function() {
|
||||
var redisPool = new RedisPool(global.environment.redis);
|
||||
var metadataBackend = cartodbRedis({pool: redisPool});
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
var pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
queryTablesApi = new QueryTablesApi(pgQueryRunner);
|
||||
overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
require('../support/test_helper');
|
||||
|
||||
var assert = require('assert');
|
||||
|
||||
var RedisPool = require('redis-mpool');
|
||||
var cartodbRedis = require('cartodb-redis');
|
||||
|
||||
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
|
||||
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
|
||||
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
|
||||
|
||||
|
||||
describe('QueryTablesApi', function() {
|
||||
|
||||
var queryTablesApi;
|
||||
|
||||
before(function() {
|
||||
var redisPool = new RedisPool(global.environment.redis);
|
||||
var metadataBackend = cartodbRedis({pool: redisPool});
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
var pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
queryTablesApi = new QueryTablesApi(pgQueryRunner);
|
||||
});
|
||||
|
||||
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
|
||||
|
||||
it('should return an object with affected tables array and last updated time', function(done) {
|
||||
var query = 'select * from test_table';
|
||||
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
|
||||
assert.ok(!err, err);
|
||||
|
||||
assert.deepEqual(result, {
|
||||
affectedTables: [ 'public.test_table' ],
|
||||
lastUpdatedTime: 1234567890123
|
||||
});
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should work with private tables', function(done) {
|
||||
var query = 'select * from test_table_private_1';
|
||||
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
|
||||
assert.ok(!err, err);
|
||||
|
||||
assert.deepEqual(result, {
|
||||
affectedTables: [ 'public.test_table_private_1' ],
|
||||
lastUpdatedTime: 1234567890123
|
||||
});
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
71
test/integration/query-tables.js
Normal file
71
test/integration/query-tables.js
Normal file
@@ -0,0 +1,71 @@
|
||||
require('../support/test_helper');
|
||||
|
||||
var assert = require('assert');
|
||||
|
||||
var RedisPool = require('redis-mpool');
|
||||
var cartodbRedis = require('cartodb-redis');
|
||||
|
||||
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
|
||||
|
||||
var QueryTables = require('cartodb-query-tables');
|
||||
|
||||
|
||||
describe('QueryTables', function() {
|
||||
|
||||
var connection;
|
||||
|
||||
before(function(done) {
|
||||
var redisPool = new RedisPool(global.environment.redis);
|
||||
var metadataBackend = cartodbRedis({pool: redisPool});
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
pgConnection.getConnection('localhost', function(err, pgConnection) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
connection = pgConnection;
|
||||
|
||||
return done();
|
||||
});
|
||||
});
|
||||
|
||||
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
|
||||
|
||||
it('should return an object with affected tables array and last updated time', function(done) {
|
||||
var query = 'select * from test_table';
|
||||
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
|
||||
assert.ok(!err, err);
|
||||
|
||||
assert.equal(result.getLastUpdatedAt(), 1234567890123);
|
||||
|
||||
assert.equal(result.tables.length, 1);
|
||||
assert.deepEqual(result.tables[0], {
|
||||
dbname: 'test_windshaft_cartodb_user_1_db',
|
||||
schema_name: 'public',
|
||||
table_name: 'test_table',
|
||||
updated_at: new Date(1234567890123)
|
||||
});
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
it('should work with private tables', function(done) {
|
||||
var query = 'select * from test_table_private_1';
|
||||
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
|
||||
assert.ok(!err, err);
|
||||
|
||||
assert.equal(result.getLastUpdatedAt(), 1234567890123);
|
||||
|
||||
assert.equal(result.tables.length, 1);
|
||||
assert.deepEqual(result.tables[0], {
|
||||
dbname: 'test_windshaft_cartodb_user_1_db',
|
||||
schema_name: 'public',
|
||||
table_name: 'test_table_private_1',
|
||||
updated_at: new Date(1234567890123)
|
||||
});
|
||||
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
});
|
||||
@@ -78,11 +78,15 @@ if test x"$PREPARE_PGSQL" = xyes; then
|
||||
sed "s/:TESTPASS/${TESTPASS}/" |
|
||||
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
|
||||
|
||||
psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
|
||||
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryStatements.sql -o sql/CDB_QueryStatements.sql
|
||||
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/cdb/scripts-available/CDB_QueryTables.sql -o sql/CDB_QueryTables.sql
|
||||
cat sql/CDB_QueryStatements.sql sql/CDB_QueryTables.sql sql/CDB_Overviews.sql |
|
||||
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
|
||||
cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
|
||||
|
||||
SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews'
|
||||
for i in ${SQL_SCRIPTS}
|
||||
do
|
||||
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
|
||||
cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
|
||||
| psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
|
||||
done
|
||||
|
||||
fi
|
||||
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
-- Mockup for CDB_Overviews
|
||||
CREATE OR REPLACE FUNCTION CDB_Overviews(table_names regclass[])
|
||||
RETURNS TABLE(base_table regclass, z integer, overview_table regclass)
|
||||
AS $$
|
||||
BEGIN
|
||||
IF (SELECT 'test_table_overviews'::regclass = ANY (table_names)) THEN
|
||||
RETURN QUERY
|
||||
SELECT 'test_table_overviews'::regclass AS base_table, 1 AS z, '_vovw_1_test_table_overviews'::regclass AS overview_table
|
||||
UNION ALL
|
||||
SELECT 'test_table_overviews'::regclass AS base_table, 2 AS z, '_vovw_2_test_table_overviews'::regclass AS overview_table;
|
||||
ELSE
|
||||
RETURN;
|
||||
END IF;
|
||||
END
|
||||
$$ LANGUAGE PLPGSQL;
|
||||
|
||||
CREATE OR REPLACE FUNCTION CDB_ZoomFromScale(scaleDenominator numeric) RETURNS int AS $$
|
||||
BEGIN
|
||||
CASE
|
||||
WHEN scaleDenominator > 500000000 THEN RETURN 0;
|
||||
WHEN scaleDenominator <= 500000000 AND scaleDenominator > 200000000 THEN RETURN 1;
|
||||
WHEN scaleDenominator <= 200000000 AND scaleDenominator > 100000000 THEN RETURN 2;
|
||||
WHEN scaleDenominator <= 100000000 AND scaleDenominator > 50000000 THEN RETURN 3;
|
||||
WHEN scaleDenominator <= 50000000 AND scaleDenominator > 25000000 THEN RETURN 4;
|
||||
WHEN scaleDenominator <= 25000000 AND scaleDenominator > 12500000 THEN RETURN 5;
|
||||
WHEN scaleDenominator <= 12500000 AND scaleDenominator > 6500000 THEN RETURN 6;
|
||||
WHEN scaleDenominator <= 6500000 AND scaleDenominator > 3000000 THEN RETURN 7;
|
||||
WHEN scaleDenominator <= 3000000 AND scaleDenominator > 1500000 THEN RETURN 8;
|
||||
WHEN scaleDenominator <= 1500000 AND scaleDenominator > 750000 THEN RETURN 9;
|
||||
WHEN scaleDenominator <= 750000 AND scaleDenominator > 400000 THEN RETURN 10;
|
||||
WHEN scaleDenominator <= 400000 AND scaleDenominator > 200000 THEN RETURN 11;
|
||||
WHEN scaleDenominator <= 200000 AND scaleDenominator > 100000 THEN RETURN 12;
|
||||
WHEN scaleDenominator <= 100000 AND scaleDenominator > 50000 THEN RETURN 13;
|
||||
WHEN scaleDenominator <= 50000 AND scaleDenominator > 25000 THEN RETURN 14;
|
||||
WHEN scaleDenominator <= 25000 AND scaleDenominator > 12500 THEN RETURN 15;
|
||||
WHEN scaleDenominator <= 12500 AND scaleDenominator > 5000 THEN RETURN 16;
|
||||
WHEN scaleDenominator <= 5000 AND scaleDenominator > 2500 THEN RETURN 17;
|
||||
WHEN scaleDenominator <= 2500 AND scaleDenominator > 1500 THEN RETURN 18;
|
||||
WHEN scaleDenominator <= 1500 AND scaleDenominator > 750 THEN RETURN 19;
|
||||
WHEN scaleDenominator <= 750 AND scaleDenominator > 500 THEN RETURN 20;
|
||||
WHEN scaleDenominator <= 500 AND scaleDenominator > 250 THEN RETURN 21;
|
||||
WHEN scaleDenominator <= 250 AND scaleDenominator > 100 THEN RETURN 22;
|
||||
WHEN scaleDenominator <= 100 THEN RETURN 23;
|
||||
END CASE;
|
||||
END
|
||||
$$ LANGUAGE plpgsql IMMUTABLE;
|
||||
@@ -1,14 +0,0 @@
|
||||
-- Return an array of statements found in the given query text
|
||||
--
|
||||
-- Regexp curtesy of Hubert Lubaczewski (depesz)
|
||||
-- Implemented in plpython for performance reasons
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
|
||||
RETURNS SETOF TEXT AS $$
|
||||
import re
|
||||
pat = re.compile( r'''((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)''', re.DOTALL )
|
||||
for match in pat.findall(query):
|
||||
cleaned = match[0].strip()
|
||||
if ( cleaned ):
|
||||
yield cleaned
|
||||
$$ language 'plpythonu' IMMUTABLE STRICT;
|
||||
@@ -1,78 +0,0 @@
|
||||
-- Return an array of table names scanned by a given query
|
||||
--
|
||||
-- Requires PostgreSQL 9.x+
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryTablesText(query text)
|
||||
RETURNS text[]
|
||||
AS $$
|
||||
DECLARE
|
||||
exp XML;
|
||||
tables text[];
|
||||
rec RECORD;
|
||||
rec2 RECORD;
|
||||
BEGIN
|
||||
|
||||
tables := '{}';
|
||||
|
||||
FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
|
||||
|
||||
IF NOT ( rec.q ilike 'select%' or rec.q ilike 'with%' ) THEN
|
||||
--RAISE WARNING 'Skipping %', rec.q;
|
||||
CONTINUE;
|
||||
END IF;
|
||||
|
||||
BEGIN
|
||||
EXECUTE 'EXPLAIN (FORMAT XML, VERBOSE) ' || rec.q INTO STRICT exp;
|
||||
EXCEPTION WHEN others THEN
|
||||
-- TODO: if error is 'relation "xxxxxx" does not exist', take xxxxxx as
|
||||
-- the affected table ?
|
||||
RAISE WARNING 'CDB_QueryTables cannot explain query: % (%: %)', rec.q, SQLSTATE, SQLERRM;
|
||||
RAISE EXCEPTION '%', SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
|
||||
-- Now need to extract all values of <Relation-Name>
|
||||
|
||||
-- RAISE DEBUG 'Explain: %', exp;
|
||||
|
||||
FOR rec2 IN WITH
|
||||
inp AS (
|
||||
SELECT
|
||||
xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x,
|
||||
xpath('//x:Relation-Name/../x:Schema/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as s
|
||||
)
|
||||
SELECT unnest(x)::text as p, unnest(s)::text as sc from inp
|
||||
LOOP
|
||||
-- RAISE DEBUG 'tab: %', rec2.p;
|
||||
-- RAISE DEBUG 'sc: %', rec2.sc;
|
||||
tables := array_append(tables, format('%s.%s', quote_ident(rec2.sc), quote_ident(rec2.p)));
|
||||
END LOOP;
|
||||
|
||||
-- RAISE DEBUG 'Tables: %', tables;
|
||||
|
||||
END LOOP;
|
||||
|
||||
-- RAISE DEBUG 'Tables: %', tables;
|
||||
|
||||
-- Remove duplicates and sort by name
|
||||
IF array_upper(tables, 1) > 0 THEN
|
||||
WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
|
||||
SELECT array_agg(p) from dist into tables;
|
||||
END IF;
|
||||
|
||||
--RAISE DEBUG 'Tables: %', tables;
|
||||
|
||||
return tables;
|
||||
END
|
||||
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
|
||||
|
||||
|
||||
-- Keep CDB_QueryTables with same signature for backwards compatibility.
|
||||
-- It should probably be removed in the future.
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
|
||||
RETURNS name[]
|
||||
AS $$
|
||||
BEGIN
|
||||
RETURN CDB_QueryTablesText(query)::name[];
|
||||
END
|
||||
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
|
||||
16
test/support/sql/_CDB_QueryStatements.sql
Normal file
16
test/support/sql/_CDB_QueryStatements.sql
Normal file
@@ -0,0 +1,16 @@
|
||||
-- DUMMY IMPLEMENTATION
|
||||
-- Ref: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_QueryStatements.sql
|
||||
-- Originally implemented in plpython for performance reasons
|
||||
|
||||
-- Return an array of statements found in the given query text
|
||||
--
|
||||
-- Regexp curtesy of Hubert Lubaczewski (depesz)
|
||||
--
|
||||
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
|
||||
RETURNS SETOF TEXT AS $$
|
||||
with matches as (
|
||||
select regexp_matches($1, $regexp$((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)$regexp$, 'g') as m
|
||||
)
|
||||
select btrim(m[1]) from matches
|
||||
$$
|
||||
LANGUAGE SQL IMMUTABLE STRICT;
|
||||
@@ -63,7 +63,15 @@ function checkCache(res) {
|
||||
|
||||
function checkSurrogateKey(res, expectedKey) {
|
||||
assert.ok(res.headers.hasOwnProperty('surrogate-key'));
|
||||
assert.equal(res.headers['surrogate-key'], expectedKey);
|
||||
|
||||
function createSet(keys, key) {
|
||||
keys[key] = true;
|
||||
return keys;
|
||||
}
|
||||
var keys = res.headers['surrogate-key'].split(' ').reduce(createSet, {});
|
||||
var expectedKeys = expectedKey.split(' ').reduce(createSet, {});
|
||||
|
||||
assert.deepEqual(keys, expectedKeys);
|
||||
}
|
||||
|
||||
var redisClient;
|
||||
|
||||
@@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
|
||||
}
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
|
||||
@@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
|
||||
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
|
||||
@@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
|
||||
SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
@@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
\"_vovw_table 1\" AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT * FROM \"_vovw_table 1\"\
|
||||
) AS \"_vovw_table 1\"\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT * FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
\"_vovw_table 1\" AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT * FROM (\
|
||||
SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT * FROM \"_vovw_table 1\"\
|
||||
) AS \"_vovw_table 1\"\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT column1, column2, column3 FROM (\
|
||||
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT column1, column2, column3 FROM _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
});
|
||||
|
||||
it('generates query using overviews for queries with selected columns and all columns', function(done){
|
||||
var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
2: { table: 'table1_ov2' }
|
||||
}
|
||||
}
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
});
|
||||
|
||||
it('generates query using overviews for queries with a semicolon', function(done){
|
||||
var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
|
||||
var sql = "SELECT column1, column2, column3 FROM table1;";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
@@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
|
||||
}
|
||||
};
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT column1, column2, column3 FROM (\
|
||||
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
|
||||
) AS _vovw_table1;\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
});
|
||||
|
||||
it('generates query using overviews for queries with extra whitespace', function(done){
|
||||
var sql = " SELECT table1.* , column1,column2, column3 FROM table1 ";
|
||||
var sql = " SELECT column1,column2, column3 FROM table1 ";
|
||||
var data = {
|
||||
overviews: {
|
||||
table1: {
|
||||
@@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
|
||||
var overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
var expected_sql = "\
|
||||
WITH\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
|
||||
_vovw_table1 AS (\
|
||||
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
|
||||
SELECT column1,column2, column3 FROM (\
|
||||
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
|
||||
UNION ALL\
|
||||
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
|
||||
)\
|
||||
SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
|
||||
) AS _vovw_table1\
|
||||
";
|
||||
assertSameSql(overviews_sql, expected_sql);
|
||||
done();
|
||||
@@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() {
|
||||
overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
assert.equal(overviews_sql, sql);
|
||||
|
||||
sql = "SELECT table1.*, column1, column2, column3 FROM table1";
|
||||
overviews_sql = overviewsQueryRewriter.query(sql, data);
|
||||
assert.equal(overviews_sql, sql);
|
||||
|
||||
done();
|
||||
});
|
||||
|
||||
|
||||
@@ -1,8 +0,0 @@
|
||||
Deprecated tools
|
||||
================
|
||||
|
||||
All tools and scripts found in this directory are deprecated and no longer maintained.
|
||||
|
||||
Use at your own peril.
|
||||
|
||||
In future releases they might get removed.
|
||||
@@ -1,81 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
var path = require('path');
|
||||
|
||||
var grainstore = require('../node_modules/windshaft/node_modules/grainstore');
|
||||
var mapnik = require('mapnik');
|
||||
var redis = require('redis');
|
||||
|
||||
function usage(me, exitcode) {
|
||||
console.log("Usage: " + me + " <database_name> <table_name> [<target_mapnik_version>]");
|
||||
process.exit(exitcode);
|
||||
}
|
||||
|
||||
var node_path = process.argv.shift();
|
||||
var script_path = process.argv.shift();
|
||||
var me = path.basename(script_path);
|
||||
|
||||
var database_name = process.argv.shift()
|
||||
var table_name = process.argv.shift()
|
||||
var MAPNIK_VERSION = process.argv.shift()
|
||||
|
||||
|
||||
if ( ! MAPNIK_VERSION ) {
|
||||
MAPNIK_VERSION = mapnik.versions.mapnik;
|
||||
}
|
||||
|
||||
if ( ! database_name || ! table_name) {
|
||||
usage(me, 1);
|
||||
}
|
||||
|
||||
var REDIS_PORT = 6379; // TODO: make a command line parameter
|
||||
|
||||
var dbnum = 0;
|
||||
|
||||
var mml_store = new grainstore.MMLStore({port:REDIS_PORT}, {mapnik_version:MAPNIK_VERSION});
|
||||
|
||||
var failures = [];
|
||||
|
||||
var client = redis.createClient(REDIS_PORT, 'localhost');
|
||||
client.on('connect', function() {
|
||||
client.select(dbnum);
|
||||
client.keys('map_style|' + database_name + '|' + table_name, function(err, matches) {
|
||||
|
||||
processNext = function() {
|
||||
if ( ! matches.length ) process.exit(failures.length);
|
||||
var k = matches.shift();
|
||||
|
||||
if ( /map_style\|.*\|.*\|/.test(k) ) {
|
||||
//console.warn("Key " + k + " is EXTENDED, skipping");
|
||||
processNext();
|
||||
}
|
||||
|
||||
var out = 'map_style|' + database_name + '|' + table_name + ': ';
|
||||
|
||||
var mml_builder = mml_store.mml_builder({
|
||||
dbname:database_name,
|
||||
table:table_name},
|
||||
function(err, payload) {
|
||||
if ( err ) {
|
||||
console.warn(out + err.message);
|
||||
failures.push(k); processNext();
|
||||
}
|
||||
else {
|
||||
mml_builder.resetStyle(function(err, data) {
|
||||
if ( err ) {
|
||||
console.warn(out + err.message);
|
||||
failures.push(k);
|
||||
}
|
||||
else console.log(out + 'OK');
|
||||
processNext();
|
||||
}, true);
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
processNext();
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
@@ -1,40 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/layergroup
|
||||
|
||||
# This is for direct windshaft connection
|
||||
#tiler_url=http://dev.localhost.lan:8083/database/cartodb_dev_user_1_db/layergroup
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test -z "$cfg"; then
|
||||
cfg="$1"
|
||||
else
|
||||
tiler_url="$1"
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$cfg"; then
|
||||
echo "Usage: $0 [-v] <config_file> [<tiler_url>]" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
|
||||
echo $tok
|
||||
@@ -1,49 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/template
|
||||
apikey=${CDB_APIKEY}
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test "$1" = "-k"; then
|
||||
shift
|
||||
apikey="$1"
|
||||
elif test "$1" = "-u"; then
|
||||
shift
|
||||
tiler_url="$1"
|
||||
elif test -z "$cfg"; then
|
||||
cfg="$1"
|
||||
else
|
||||
echo "Unused parameter $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$cfg"; then
|
||||
echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cmd="curl -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
# Successful response contains no space
|
||||
echo "$res" | grep " " && { echo $res && exit 1; }
|
||||
|
||||
tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
|
||||
echo $tok
|
||||
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/template
|
||||
apikey=${CDB_APIKEY}
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test "$1" = "-k"; then
|
||||
shift
|
||||
apikey="$1"
|
||||
elif test "$1" = "-u"; then
|
||||
shift
|
||||
tiler_url="$1"
|
||||
elif test -z "$tpl"; then
|
||||
tpl="$1"
|
||||
else
|
||||
echo "Unused parameter $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$tpl"; then
|
||||
echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id>" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cmd="curl -X DELETE -skH Content-Type:application/json ${tiler_url}/${tpl}?api_key=${apikey}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
|
||||
echo $tok
|
||||
@@ -1,11 +0,0 @@
|
||||
{"version":"1.0.1",
|
||||
"layers":[{
|
||||
"type":"cartodb",
|
||||
"options":{
|
||||
"sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
|
||||
"cartocss":"#style{ marker-width: 12;}",
|
||||
"cartocss_version":"2.1.1",
|
||||
"Interactivity":"id"
|
||||
}
|
||||
}]
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
{"version":"1.0.1",
|
||||
"layers":[{
|
||||
"type":"cartodb",
|
||||
"options":{
|
||||
"sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
|
||||
"cartocss":"#style{ marker-width: 12;}",
|
||||
"cartocss_version":"2.1.1",
|
||||
"Interactivity":"id"
|
||||
}
|
||||
}]
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
{"version":"1.0.1",
|
||||
"layers":[{
|
||||
"type":"torque",
|
||||
"options":{
|
||||
"sql":"select 1 as id, ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
|
||||
"cartocss":"Map{ -torque-time-attribute:'id'; -torque-aggregation-function:'count(id)'; -torque-frame-count:2; -torque-resolution:2}",
|
||||
"cartocss_version": "2.1.1"
|
||||
}
|
||||
}]
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"version":"0.0.1",
|
||||
"name":"simple",
|
||||
"placeholders":{},
|
||||
"auth":{ "method":"open" },
|
||||
"layergroup":{
|
||||
"version":"1.0.1",
|
||||
"layers":[{
|
||||
"type":"cartodb",
|
||||
"options":{
|
||||
"sql":"select ST_SetSRID(ST_MakePoint(0,0),3857) as the_geom_webmercator",
|
||||
"cartocss":"#s{ marker-width: 12;}",
|
||||
"cartocss_version":"2.1.1"
|
||||
}
|
||||
}]
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/template
|
||||
apikey=${CDB_APIKEY}
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test "$1" = "-k"; then
|
||||
shift
|
||||
apikey="$1"
|
||||
elif test "$1" = "-u"; then
|
||||
shift
|
||||
tiler_url="$1"
|
||||
elif test -z "$tpl"; then
|
||||
tpl="$1"
|
||||
elif test -z "$cfg"; then
|
||||
cfg="$1"
|
||||
else
|
||||
echo "Unused parameter $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$tpl"; then
|
||||
echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> [<template_params>]" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if test -z "$cfg"; then
|
||||
cfg="/dev/null"
|
||||
fi
|
||||
|
||||
tiler_url="${tiler_url}/${tpl}"
|
||||
|
||||
cmd="curl -X POST -skH Content-Type:application/json --data-binary @- ${tiler_url}?api_key=${apikey}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
tok=`echo "$res" | sed 's/.*"layergroupid":"\([^"]*\)".*/\1/'`
|
||||
echo $tok
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/template
|
||||
apikey=${CDB_APIKEY}
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test "$1" = "-k"; then
|
||||
shift
|
||||
apikey="$1"
|
||||
elif test "$1" = "-u"; then
|
||||
shift
|
||||
tiler_url="$1"
|
||||
elif test "$1" = "-h" -o "$1" = "-?"; then
|
||||
echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>]" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
|
||||
exit 0
|
||||
else
|
||||
echo "Unused parameter $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
|
||||
cmd="curl -X GET -sk ${tiler_url}?api_key=${apikey}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
node <<EOF
|
||||
var parsed = JSON.parse('$res');
|
||||
console.dir(parsed);
|
||||
EOF
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
MUNIN_PLUGINS_DIR=/etc/munin/plugins
|
||||
MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
|
||||
PWD=$(shell pwd)
|
||||
|
||||
all: windshaft.conf
|
||||
|
||||
windshaft.conf: windshaft.conf.in
|
||||
sed 's#@PWD@#$(PWD)#' < $< > $@
|
||||
|
||||
install-munin-plugin-conf: windshaft.conf
|
||||
install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/windshaft.conf
|
||||
|
||||
install-munin-plugin: windshaft
|
||||
install -m 755 $< $(MUNIN_PLUGINS_DIR)/windshaft
|
||||
|
||||
install: install-munin-plugin install-munin-plugin-conf
|
||||
|
||||
clean:
|
||||
rm -f windshaft.conf
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
envnik=$(basename "${TILER_ENVIRONMENT}" .js)
|
||||
|
||||
if test "$1" = "config"; then
|
||||
echo "graph_title fd usage (${envnik})"
|
||||
cat <<"EOM"
|
||||
graph_vlabel number of file descriptors
|
||||
graph_category windshaft
|
||||
graph_scale no
|
||||
procs.label Number of tiler processes
|
||||
pgsql.label PostgreSQL connections (max)
|
||||
redis.label Redis connections (max)
|
||||
http.label Incoming http requests (max)
|
||||
caches.label Number of renderer caches (max)
|
||||
nfd.label Total file descriptors (max)
|
||||
EOM
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if test x"$1" != x; then
|
||||
TILER_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
|
||||
fi
|
||||
|
||||
if test -z "$TILER_ENVIRONMENT"; then
|
||||
echo "Usage: $0 [<environment>]" >&2
|
||||
echo " or: [TILER_ENVIRONMENT=<environment>] $0" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
http_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').port)" | node) || exit 1
|
||||
pgsql_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').postgres.port)" | node) || exit 1
|
||||
redis_port=$(echo "console.log(require('${TILER_ENVIRONMENT}').redis.port)" | node) || exit 1
|
||||
|
||||
pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
|
||||
nworkers=$(echo "${pids}" | wc -l)
|
||||
pids=$(echo "${pids}" | paste -sd ' ')
|
||||
|
||||
if test -z "${pids}"; then
|
||||
echo "No processes found listening on tcp port '${http_port}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmpreport="/tmp/checkfd.$$.txt"
|
||||
|
||||
lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
|
||||
|
||||
maxdb=0
|
||||
maxredis=0
|
||||
maxhttp=0
|
||||
maxtot=0
|
||||
maxcache=0
|
||||
|
||||
for pid in ${pids}; do
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
|
||||
if test $cnt -gt $maxdb; then maxdb=$cnt; fi
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
|
||||
if test $cnt -gt $maxredis; then maxredis=$cnt; fi
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
|
||||
if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
|
||||
if test $cnt -gt $maxtot; then maxtot=$cnt; fi
|
||||
|
||||
log=$(grep "${pid}" "${tmpreport}" | grep -w 1w | awk '{print $9}')
|
||||
if test -e "${log}"; then
|
||||
kill -USR2 "${pid}"
|
||||
cnt=$(tac ${log} | sed -n -e '/ItemKey/p;/ RenderCache /q' | wc -l)
|
||||
if test $cnt -gt $maxcache; then maxcache=$cnt; fi
|
||||
else
|
||||
# report the error...
|
||||
maxcache=-1
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
echo "procs.value ${nworkers}"
|
||||
echo "pgsql.value ${maxdb}"
|
||||
echo "redis.value ${maxredis}"
|
||||
echo "http.value ${maxhttp}"
|
||||
echo "caches.value ${maxcache}"
|
||||
echo "nfd.value ${maxtot}"
|
||||
|
||||
rm -f "${tmpreport}"
|
||||
@@ -1,5 +0,0 @@
|
||||
# Configuration file for munin plugin
|
||||
|
||||
[windshaft]
|
||||
user root
|
||||
env.TILER_ENVIRONMENT @PWD@/../../config/environments/production.js
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/template
|
||||
apikey=${CDB_APIKEY}
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-k"; then
|
||||
shift
|
||||
apikey="$1"
|
||||
elif test "$1" = "-u"; then
|
||||
shift
|
||||
tiler_url="$1"
|
||||
elif test -z "$tpl"; then
|
||||
tpl="$1"
|
||||
else
|
||||
echo "Unused parameter $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$tpl"; then
|
||||
echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_config>" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
basedir=$(cd $(dirname $0); cd ..; pwd)
|
||||
export CDB_APIKEY=${apikey}
|
||||
max=3000000
|
||||
i=0
|
||||
while test "$i" -le "$max"; do
|
||||
tpln=`cat ${tpl} | sed "s/\"name\":\"\(.*\)\"/\"name\":\"\1${i}\"/"`
|
||||
tpl_id=`echo ${tpln} | ${basedir}/create_template -u ${tiler_url} /dev/stdin`
|
||||
if test $? -ne 0; then
|
||||
echo $tpl_id >&2
|
||||
break
|
||||
fi
|
||||
tpl_id=`echo ${tpln} | ${basedir}/update_template -u ${tiler_url} ${tpl_id} /dev/stdin`
|
||||
if test $? -ne 0; then
|
||||
echo $tpl_id >&2
|
||||
break
|
||||
fi
|
||||
out=`${basedir}/delete_template -u ${tiler_url} ${tpl_id}`
|
||||
if test $? -ne 0; then
|
||||
echo $out >&2
|
||||
break
|
||||
fi
|
||||
i=$((i+1))
|
||||
if test `expr $i % 100` -eq 0; then
|
||||
echo -n "."
|
||||
fi
|
||||
done
|
||||
@@ -1,105 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/*
|
||||
|
||||
This scripts drops all extended map_style keys in redis and regenerates
|
||||
the XML caches in all the base ones to target the configured mapnik_version.
|
||||
|
||||
Optionally (with --convert) it also re-writes the CartoCSS if needed
|
||||
to target the configured mapnik_version.
|
||||
|
||||
It is recommended to make a backup of the redis database before using
|
||||
this script.
|
||||
|
||||
*/
|
||||
|
||||
var path = require('path');
|
||||
|
||||
// Reset all styles in the store
|
||||
var grainstore = require('../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
|
||||
var mapnik = require('mapnik');
|
||||
var redis = require('redis');
|
||||
|
||||
function usage(me, exitcode) {
|
||||
console.log("Usage: " + me + " [--convert] <environment>");
|
||||
process.exit(exitcode);
|
||||
}
|
||||
|
||||
var doConvert = false;
|
||||
|
||||
var node_path = process.argv.shift();
|
||||
var script_path = process.argv.shift();
|
||||
var me = path.basename(script_path);
|
||||
var ENV;
|
||||
var arg;
|
||||
while ( arg = process.argv.shift() ) {
|
||||
if ( arg == '--convert' ) {
|
||||
doConvert = true;
|
||||
}
|
||||
else if ( ! ENV ) {
|
||||
ENV = arg;
|
||||
}
|
||||
else {
|
||||
usage(me, 1);
|
||||
}
|
||||
}
|
||||
|
||||
if ( ! ENV ) usage(me, 1);
|
||||
|
||||
global.environment = require('../config/environments/' + ENV);
|
||||
var serverOptions = require('../lib/cartodb/server_options'); // _after_ setting global.environment
|
||||
|
||||
var MAPNIK_VERSION = global.environment.mapnik_version || mapnik.versions.mapnik;
|
||||
|
||||
console.log( (doConvert ? "Converting" : "Resetting" ) + ' all styles to target ' + MAPNIK_VERSION);
|
||||
|
||||
var dbnum = 0;
|
||||
|
||||
var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
|
||||
|
||||
var failures = [];
|
||||
|
||||
var client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
|
||||
client.on('connect', function() {
|
||||
client.select(dbnum);
|
||||
client.keys('map_style|*', function(err, matches) {
|
||||
|
||||
processNext = function() {
|
||||
if ( ! matches.length ) process.exit(failures.length);
|
||||
var k = matches.shift();
|
||||
|
||||
if ( /map_style\|.*\|.*\|/.test(k) ) {
|
||||
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/58
|
||||
//console.warn("Key " + k + " is EXTENDED, dropping");
|
||||
client.del(k, function(err) {
|
||||
if ( err ) console.warn("Error dropping key " + k);
|
||||
processNext();
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
var params = RegExp(/map_style\|(.*)\|(.*)/).exec(k);
|
||||
var db = params[1];
|
||||
var tab = params[2];
|
||||
var out = 'map_style|' + db + '|' + tab + ': ';
|
||||
|
||||
var mml_builder = mml_store.mml_builder({dbname:db, table:tab},
|
||||
function(err, payload) {
|
||||
|
||||
if ( err ) { console.warn(out + err.message); failures.push(k); processNext(); }
|
||||
else {
|
||||
mml_builder.resetStyle(function(err, data) {
|
||||
if ( err ) { console.warn(out + err.message); failures.push(k); }
|
||||
else console.log(out + 'OK' + ( doConvert ? ' (converted)' : '' ));
|
||||
processNext();
|
||||
}, doConvert);
|
||||
}
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
processNext();
|
||||
|
||||
});
|
||||
|
||||
});
|
||||
148
tools/show_style
148
tools/show_style
@@ -1,148 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
var path = require('path');
|
||||
var redis = require('redis');
|
||||
var Step = require('step');
|
||||
|
||||
|
||||
function usage(me, exitcode) {
|
||||
console.log("Usage: " + me + " [--env <environment>] <username> [<tablename>|~<token>]");
|
||||
process.exit(exitcode);
|
||||
}
|
||||
|
||||
var node_path = process.argv.shift();
|
||||
var script_path = process.argv.shift();
|
||||
var basedir = path.dirname(script_path);
|
||||
var me = path.basename(script_path);
|
||||
|
||||
var ENV = 'development.js';
|
||||
var username, token;
|
||||
var arg;
|
||||
while ( arg = process.argv.shift() ) {
|
||||
if ( arg == '--env' ) {
|
||||
ENV = process.argv.shift();
|
||||
}
|
||||
else if ( ! username ) {
|
||||
username = arg;
|
||||
}
|
||||
else if ( ! token ) {
|
||||
token = arg;
|
||||
}
|
||||
else {
|
||||
console.warn("Unused parameter " + arg);
|
||||
}
|
||||
}
|
||||
|
||||
if ( ! username ) usage(me, 1);
|
||||
|
||||
console.log("Using environment " + ENV);
|
||||
|
||||
global.environment = require('../config/environments/' + ENV);
|
||||
// _after_ setting global.environment
|
||||
var serverOptions = require('../lib/cartodb/server_options')();
|
||||
|
||||
var client;
|
||||
var dbname;
|
||||
Step(
|
||||
function getClient() {
|
||||
client = redis.createClient(serverOptions.redis.port, serverOptions.redis.host);
|
||||
client.on('connect', this);
|
||||
},
|
||||
function getUserMeta(err) {
|
||||
if ( err ) throw err;
|
||||
client.select(5);
|
||||
client.hgetall('rails:users:' + username, this);
|
||||
},
|
||||
function readDB(err, data) {
|
||||
if ( err ) throw err;
|
||||
if ( ! data )
|
||||
throw new Error('Username ' + username + ' unknown by redis on port '
|
||||
+ serverOptions.redis.port + ' (try CARTODB/script/restore_redis?)');
|
||||
//console.log("Data:"); console.dir(data);
|
||||
dbname = data['database_name'];
|
||||
console.log("Database name for user " + username + ": " + dbname);
|
||||
client.select(0);
|
||||
return null;
|
||||
},
|
||||
function showTokens(err) {
|
||||
if ( err ) throw err;
|
||||
if ( token ) return null;
|
||||
var next = this;
|
||||
Step(
|
||||
function getTokens() {
|
||||
client.keys('map_style|' + dbname + '|*', this);
|
||||
},
|
||||
function showTokens(err, data) {
|
||||
if (err) throw err;
|
||||
if ( data ) console.log(data.join('\n'));
|
||||
return null;
|
||||
},
|
||||
function showTokensFinish(err) {
|
||||
next(err);
|
||||
}
|
||||
);
|
||||
},
|
||||
function showStyle(err) {
|
||||
if ( err ) throw err;
|
||||
if ( ! token ) return null;
|
||||
var next = this;
|
||||
Step(
|
||||
function getStyle() {
|
||||
client.get('map_style|' + dbname + '|' + token, this);
|
||||
},
|
||||
function showStyle(err, data) {
|
||||
if ( err ) throw err;
|
||||
if ( ! data ) {
|
||||
throw new Error(token + ': no such map style known by redis on port '
|
||||
+ serverOptions.redis.port);
|
||||
}
|
||||
//console.log("data: " + data);
|
||||
var x=JSON.parse(data);
|
||||
printMapnikStyle(x, this);
|
||||
},
|
||||
function showStyleFinish(err) {
|
||||
next(err);
|
||||
}
|
||||
);
|
||||
},
|
||||
function finish(err) {
|
||||
if ( err ) {
|
||||
console.error(err.message)
|
||||
process.exit(1);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
);
|
||||
|
||||
function printMapnikStyle(x, callback) {
|
||||
console.log('style: ' + x.style);
|
||||
console.log('version: ' + x.version);
|
||||
var grainstore = require(basedir + '/../node_modules/windshaft/node_modules/grainstore/lib/grainstore');
|
||||
var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
|
||||
var builderconfig = {dbname:dbname};
|
||||
if ( token.match(/^~/) ) {
|
||||
builderconfig.token = token.substring(1);
|
||||
} else {
|
||||
builderconfig.table = token;
|
||||
}
|
||||
var mml_builder;
|
||||
Step(
|
||||
function getBuilder() {
|
||||
mml_builder = mml_store.mml_builder(builderconfig, this);
|
||||
},
|
||||
function getXML(err, builder) {
|
||||
if ( err ) throw err;
|
||||
mml_builder.toXML(this);
|
||||
},
|
||||
function showXML(err, xml) {
|
||||
if ( err ) throw err;
|
||||
console.log('- XML - ');
|
||||
console.log(xml);
|
||||
return null;
|
||||
},
|
||||
function finish(err) {
|
||||
callback(err);
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
verbose=no
|
||||
tiler_url=http://dev.localhost.lan:8181/tiles/template
|
||||
apikey=${CDB_APIKEY}
|
||||
|
||||
while test -n "$1"; do
|
||||
if test "$1" = "-v"; then
|
||||
verbose=yes
|
||||
elif test "$1" = "-k"; then
|
||||
shift
|
||||
apikey="$1"
|
||||
elif test "$1" = "-u"; then
|
||||
shift
|
||||
tiler_url="$1"
|
||||
elif test -z "$tpl"; then
|
||||
tpl="$1"
|
||||
elif test -z "$cfg"; then
|
||||
cfg="$1"
|
||||
else
|
||||
echo "Unused parameter $1" >&2
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
if test -z "$cfg"; then
|
||||
echo "Usage: $0 [-v] [-k <api_key>] [-u <tiler_url>] <template_id> <template_config>" >&2
|
||||
echo "Default <tiler_url> is ${tiler_url}" >&2
|
||||
echo "Default <api_key> is read from CDB_APIKEY env variable" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cmd="curl -X PUT -skH Content-Type:application/json --data-binary @- ${tiler_url}/${tpl}?api_key=${apikey}"
|
||||
if test x${verbose} = xyes; then
|
||||
cmd="${cmd} -v"
|
||||
fi
|
||||
res=`cat ${cfg} | tr '\n' ' ' | ${cmd}`
|
||||
if test $? -gt 0; then
|
||||
echo "curl command failed: ${cmd}"
|
||||
fi
|
||||
|
||||
if test x${verbose} = xyes; then
|
||||
echo "${res}"
|
||||
fi
|
||||
|
||||
tok=`echo "$res" | sed 's/.*"template_id":"\([^"]*\)".*/\1/'`
|
||||
echo $tok
|
||||
Reference in New Issue
Block a user