Compare commits

...

88 Commits

Author SHA1 Message Date
Raul Ochoa
86fb58155a Release 1.15.0 2014-08-13 15:26:25 +02:00
Raul Ochoa
d3c656893c Merge pull request #202 from CartoDB/CDB-3135
Upgrades dependencies to get redis-mpool improvements
2014-08-13 15:24:59 +02:00
Raul Ochoa
713e394e7b Slow pool configuration in example configurations 2014-08-13 15:15:30 +02:00
Raul Ochoa
40acf533ae Specifies name in the redis pool 2014-08-13 15:12:46 +02:00
Raul Ochoa
13fdfc602e Upgrades dependencies 2014-08-13 15:10:58 +02:00
Raul Ochoa
3eab0d6349 Stubs next version 2014-08-07 12:35:38 +02:00
Raul Ochoa
58047fac17 Release 1.14.0 2014-08-07 12:34:49 +02:00
Raul Ochoa
6e4144d015 Prepares next release 2014-08-07 12:33:53 +02:00
Raul Ochoa
8b0fda8d89 Merge pull request #201 from CartoDB/CDB-3664
Upgrades windshaft (and grainstore) to be able to specify the tile format
2014-08-07 12:20:01 +02:00
Raul Ochoa
2ed656ca0d Upgrades windshaft (and grainstore) to be able to specify the tile
format, see: https://github.com/mapnik/mapnik/wiki/OutputFormats
2014-08-07 01:57:21 +02:00
Raul Ochoa
fa72f52ad4 Merge pull request #200 from CartoDB/CDB-3686
Affected tables and last updated time for a query into a single SQL API request
2014-08-04 14:14:00 +02:00
Raul Ochoa
528815a564 Updates news 2014-08-04 13:24:44 +02:00
Raul Ochoa
dabcba9f5f Merge branch 'master' into CDB-3686 2014-08-04 13:11:40 +02:00
Raul Ochoa
995dabc9b7 Stubs next version 2014-08-04 13:05:08 +02:00
Raul Ochoa
36145542af Release 1.13.1 2014-08-04 13:04:13 +02:00
Raul Ochoa
06eca6525a Merge pull request #199 from CartoDB/CDB-3657
CDB-3657 Adds profiler as JSON to the header
2014-08-04 13:02:27 +02:00
Raul Ochoa
414673b347 CDB-3657 Adds profiler as JSON to the header 2014-08-04 12:53:15 +02:00
Raul Ochoa
a9767c049f CDB-3686 Adds support for per mil tolerance when comparing images as in Mac OS X some results from ImageMagick are a bit odd 2014-08-04 12:35:54 +02:00
Raul Ochoa
507a6a8979 CDB-3686 Style changes 2014-08-04 01:32:49 +02:00
Raul Ochoa
73d1db3bd2 CDB-3686 Adds support for per mil tolerance when comparing images as in Mac OS X some results from ImageMagick are a bit odd 2014-08-04 01:30:24 +02:00
Raul Ochoa
9b5921e8e1 CDB-3686 Fixes expected queries based on changes to request table names and last updated time in one request 2014-08-04 01:29:23 +02:00
Raul Ochoa
799a999148 CDB-3686 Makes SQL API emulator to handle new query with both names and updated time for affected tables. 2014-08-04 01:28:30 +02:00
Raul Ochoa
eafe3af13e Fixes reference to redis-mpool 2014-08-01 18:27:55 +02:00
Raul Ochoa
4e420c2f33 Merge tag '1.13.0' into CDB-3686 2014-07-30 18:25:10 +02:00
Raul Ochoa
654b3ad6d3 Fixes reference to redis-mpool 2014-07-30 18:23:45 +02:00
Raul Ochoa
9f8d73a1df Removes duplicated file 2014-07-30 18:17:14 +02:00
Raul Ochoa
f6e0b4ca9f Merge branch 'master' of https://github.com/cartodb/windshaft-cartodb into CDB-3686 2014-07-30 18:13:47 +02:00
Raul Ochoa
1dbad1f0b8 Stubs next version 2014-07-30 18:11:32 +02:00
Raul Ochoa
8f9e19e3e2 Fix date in NEWS 2014-07-30 18:10:28 +02:00
Raul Ochoa
b1a0b5e235 Release 1.13.0 2014-07-30 18:08:07 +02:00
Raul Ochoa
bce13944c3 Merge pull request #198 from CartoDB/multiuser
Support for multiple schemas, multiple auth tokens and public user from redis
2014-07-30 17:56:05 +02:00
Raul Ochoa
c8fc3d1e7a Updates to correct version of step profiler 2014-07-30 16:23:03 +02:00
Raul Ochoa
e6f7b9c1f9 Adds news about changes in multiuser branch 2014-07-30 16:08:45 +02:00
Raul Ochoa
552ebaaaac Upgrades Windshaft to version 0.22.0 2014-07-30 15:25:15 +02:00
Raul Ochoa
6019fb2ca3 Merge pull request #197 from CartoDB/CDB-3678
[CDB-3678] Creates api_hostname global variable
2014-07-30 15:09:58 +02:00
Raul Ochoa
3af45e1a32 Moves calls to SQL API to its own entity.
Groups affected tables and last updated time for affected tables into one request.
2014-07-30 13:46:46 +02:00
Raul Ochoa
75088c89d3 Style fixes 2014-07-30 13:45:53 +02:00
Luis Bosque
2c1d46f159 [CDB-3678] Creates api_hostname global variable 2014-07-29 14:54:35 +02:00
javi santana
15b9a1f34b fixed documentation 2014-07-24 13:01:35 +02:00
Carlos Matallín
5c70dd0557 run tests 2014-07-21 11:11:21 +02:00
Carlos Matallín
dc0acdbee1 Update Map-API.md 2014-07-21 10:18:15 +02:00
Carlos Matallín
ae01047e8c Merge pull request #195 from matallo/master
move maps api doc
2014-07-09 14:24:27 +02:00
Carlos Matallín
1b7c2a0208 move maps api doc 2014-07-09 14:23:53 +02:00
Carlos Matallín
a8b01f523a Merge pull request #194 from matallo/master
update doc
2014-07-09 14:20:43 +02:00
Carlos Matallín
23cbad8ba6 update doc 2014-07-09 14:19:25 +02:00
Carlos Matallín
984e0f6e83 Rename Map-API.md to Map-API-internal.md 2014-07-09 13:13:21 +02:00
Raul Ochoa
67df6a4d73 Adds support for several auth tokens 2014-07-08 10:35:45 +02:00
Raul Ochoa
f756b9d77f Removes search_path param 2014-07-04 12:18:35 +02:00
Raul Ochoa
0dfd51f81a Adds host to redis setup as it does not make sense to continue if there is no host in redis. 2014-07-04 11:47:44 +02:00
Raul Ochoa
bfdcee3772 Retrieving db public user from redis. It uses a new multiget method from cartodb-redis 2014-07-03 21:39:47 +02:00
Raul Ochoa
470aea22d9 Sets full search_path 2014-07-03 10:24:37 +02:00
Raul Ochoa
32e4c26c95 Sets origin for grainstore in shrinkwrap 2014-07-02 19:34:28 +02:00
Raul Ochoa
6a34568935 Forcing grainstore version in shrinkwrap 2014-07-02 19:16:06 +02:00
javi
3548106a6c changed branch for windshaft 2014-06-27 23:42:03 +02:00
javi
3806ad8843 Merge remote-tracking branch 'origin/CDB-2891-search_path' into multiuser 2014-06-27 09:10:39 +02:00
Raul Ochoa
037ce2dc12 CDB-2891 Exposes username as search_path in params 2014-06-27 00:48:48 +01:00
javi
338c0bcdbe use regclass instead table name to look for last_updated in CDB_tablemetadata 2014-06-26 15:00:55 +02:00
Raul Ochoa
bc3baf3094 CDB-3256 Prepares 1.12.1 release 2014-06-24 16:26:57 +02:00
Raul Ochoa
8a91b5cfb5 CDB-3256 Fixes test related to cache in templated layergroup creation 2014-06-24 16:05:54 +02:00
Raul Ochoa
4cf1ddd6fc CDB-3256 Adds response and method references to fake request object 2014-06-24 15:52:47 +02:00
Raul Ochoa
cb781aeb00 CDB-3256 Prepares 1.12.0 Release 2014-06-24 14:24:14 +02:00
Raul Ochoa
2dd03e21e1 CDB-3256 fix test and adds a couple more of tests for testing the no-cache scenarios 2014-06-24 13:13:00 +02:00
Raul Ochoa
055bacbad7 Sets PGUSER environment variable 2014-06-24 12:39:57 +02:00
Raul Ochoa
46ae6d1fe4 Changes travis configuration to be similar to windshaft one 2014-06-24 12:39:46 +02:00
Raul Ochoa
5e73b12cf5 CDB-3256 adds headers based on affected tables when creating a layergroup via HTTP GET 2014-06-24 12:16:30 +02:00
Sandro Santilli
86c6f3eeac Wrap all json strings and string values in double-quotes 2014-06-09 12:19:16 +02:00
Raul Ochoa
8922ae3a45 adds document about metrics being tracked 2014-05-29 13:10:46 +02:00
Raul Ochoa
318e22e9fa Merge commit '4738b880a6c29a6d10dda3ad178f35a54bd576d3'
Conflicts:
	NEWS.md
	package.json
2014-05-07 19:07:20 +02:00
Raul Ochoa
4738b880a6 Prepares release 1.10.3 2014-05-07 18:28:10 +02:00
Sandro Santilli
49829f8935 Set default PostgreSQL application name to "cartodb_tiler" 2014-05-07 16:19:22 +02:00
Sandro Santilli
8e9d72982a Refuse to start if log_filename points to a non-existing directory
Closes #189
2014-05-07 11:03:25 +02:00
Raul Ochoa
d2f0180475 Merge remote-tracking branch 'rochoa/master' 2014-04-22 11:40:48 +02:00
Raul Ochoa
4da0b1e07c CDB-2096 Configures the CWD for log4js logger. 2014-04-22 10:52:59 +02:00
Sandro Santilli
5a4a35b665 Fix documentation for redis.max setting
Closes #192
2014-04-16 17:53:42 +02:00
Raul Ochoa
248cb4bd76 Removes unused dependency. 2014-04-11 15:14:59 +02:00
Sandro Santilli
140001f036 Update release document 2014-04-09 09:14:20 +02:00
Sandro Santilli
3917cac800 Add 1.10.2 section 2014-04-08 10:00:41 +02:00
Sandro Santilli
ee37da5b35 Prepare for 1.10.3 2014-04-08 10:00:10 +02:00
Sandro Santilli
882ec65ba0 Use signer's map_key when contacting sql-api
Includes testcase.
Fixes #188
2014-04-08 09:44:49 +02:00
Sandro Santilli
bbd4db6ddb Fix show_style tool broken since 1.8.1 2014-03-31 12:53:48 +02:00
Sandro Santilli
312194228a Stop duplicating global.environment as global.settings 2014-03-28 18:47:59 +01:00
Sandro Santilli
5c1125900b Add support for log_filename directive, reopen logfile on SIGHUP 2014-03-28 18:05:18 +01:00
Sandro Santilli
08b8741282 Reload log files on SIGUSR2
This is an attempt to play more nicely with logrotate
2014-03-28 17:06:44 +01:00
Sandro Santilli
e8367b765a Add persist_connection setting in .example configs 2014-03-24 17:40:43 +01:00
Sandro Santilli
91cd0df7b3 Typo in comment 2014-03-24 17:03:32 +01:00
Sandro Santilli
dff0a2aa1f Merge branch 'b1.10'
Fixes bogus caching of failing jsonp responses
2014-03-21 15:17:43 +01:00
Sandro Santilli
5f30b9e798 Add an example of a slow mapconfig (using lots of data) 2014-03-20 18:19:30 +01:00
Sandro Santilli
7c892de7b1 Prepare for 1.11.0 2014-03-20 17:11:06 +01:00
29 changed files with 2020 additions and 464 deletions

View File

@@ -1,15 +1,27 @@
before_install:
- sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
- sudo apt-get -qq purge postgis* postgresql*
- sudo apt-add-repository --yes ppa:cartodb/postgresql-9.3
- sudo apt-add-repository --yes ppa:cartodb/gis
- sudo rm -Rf /var/lib/postgresql /etc/postgresql
- sudo apt-add-repository --yes ppa:mapnik/v2.1.0
- sudo apt-get update -q
- sudo apt-get install -q libmapnik-dev
- sudo apt-get update
- sudo apt-get install -y postgresql-9.3-postgis-2.1
- sudo apt-get install -y postgresql-contrib-9.3
- sudo apt-get install -y libmapnik-dev
- sudo apt-get install -y gdal-bin
- echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.3/main/pg_hba.conf
- sudo service postgresql restart
- createdb template_postgis
- psql -c "CREATE EXTENSION postgis" template_postgis
before_script:
# Tell npm to use known registrars:
# see http://blog.npmjs.org/post/78085451721/npms-self-signed-certificate-is-no-more
- npm config set ca ""
env:
- NPROCS=1 JOBS=1
- NPROCS=1 JOBS=1 PGUSER=postgres
language: node_js
node_js:

View File

@@ -1,13 +1,20 @@
1. Ensure proper version in package.json
2. Ensure NEWS section exists for the new version, review it, add release date
3. Drop npm-shrinkwrap.json
4. Run npm install
5. Test (make check or npm test), fix if broken before proceeding
6. Run npm shrinkwrap
7. Set "from" in npm-shrinkwrap.json for known packages
(windshaft, node-varnish, grainstore...)
8. Commit package.json, npm-shrinwrap.json, NEWS
9. git tag -a Major.Minor.Patch # use NEWS section as content
10. Announce
11. Stub NEWS/package for next version
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. Drop npm-shrinkwrap.json
5. Run npm shrinkwrap to recreate npm-shrinkwrap.json
6. Commit package.json, npm-shrinwrap.json, NEWS
7. git tag -a Major.Minor.Patch # use NEWS section as content
8. Announce on cartodb@googlegroups.com
9. Stub NEWS/package for next version
Versions:
Bugfix releases increment Patch component of version.
Feature releases increment Minor and set Patch to zero.
If backward compatibility is broken, increment Major and
set to zero Minor and Patch.
Branches named 'b<Major>.<Minor>' are kept for any critical
fix that might need to be shipped before next feature release
is ready.

72
NEWS.md
View File

@@ -1,8 +1,76 @@
1.15.0 -- 2014-08-13
--------------------
Enhancements:
- Upgrades dependencies:
- redis-mpool
- cartodb-redis
- windshaft
- Specifies name in the redis pool
- Slow pool configuration in example configurations
1.14.0 -- 2014-08-07
--------------------
Enhancements:
- SQL API requests moved to its own entity
New features:
- Affected tables and last updated time for a query are performed in a single request to the SQL API
- Allow specifying the tile format, upgrades windshaft and grainstore dependencies for this matter
1.13.1 -- 2014-08-04
--------------------
Enhancements:
- Profiler header sent as JSON string
1.13.0 -- 2014-07-30
--------------------
New features:
- Support for postgresql schemas
- Use public user from redis
- Support for several auth tokens
1.12.1 -- 2014-06-24
--------------------
Enhancements:
- Caches layergroup and sets X-Cache-Channel in GET requests also in named maps
1.12.0 -- 2014-06-24
--------------------
New features:
- Caches layergroup and sets X-Cache-Channel in GET requests
1.11.1 -- 2014-05-07
--------------------
Enhancements:
- Upgrade Windshaft to 0.21.0, see
http://github.com/CartoDB/Windshaft/blob/0.21.0/NEWS
1.11.0 -- 2014-04-28
--------------------
New features:
- Add support for log_filename directive
- Reopen log file on SIGHUP, for better logrotate integration
Enhancements:
- Set default PostgreSQL application name to "cartodb_tiler"
1.10.2 -- 2014-04-08
--------------------
Bug fixes:
- Fix show_style tool broken since 1.8.1
- Fix X-Cache-Channel of tiles accessed via signed token (#188)
@@ -14,7 +82,7 @@ Bug fixes:
- Do not cache non-success jsonp responses (#186)
1.10.0 -- 2014-03-20
-------------------
--------------------
New features:

41
app.js
View File

@@ -7,6 +7,10 @@
* environments: [development, production]
*/
var path = require('path'),
fs = require('fs')
;
if ( process.argv[2] ) ENV = process.argv[2];
else if ( process.env['NODE_ENV'] ) ENV = process.env['NODE_ENV'];
@@ -21,22 +25,36 @@ if (ENV != 'development' && ENV != 'production' && ENV != 'staging' ){
process.exit(1);
}
var _ = require('underscore')
, Step = require('step')
;
var _ = require('underscore');
// set environment specific variables
global.settings = require(__dirname + '/config/settings');
global.environment = require(__dirname + '/config/environments/' + ENV);
_.extend(global.settings, global.environment);
global.environment.api_hostname = require('os').hostname().split('.')[0];
global.log4js = require('log4js')
log4js_config = {
appenders: [
{ type: "console", layout: { type:'basic' } }
],
appenders: [],
replaceConsole:true
};
if ( global.environment.log_filename ) {
var logdir = path.dirname(global.environment.log_filename);
// See cwd inlog4js.configure call below
logdir = path.resolve(__dirname, logdir);
if ( ! fs.existsSync(logdir) ) {
console.error("Log filename directory does not exist: " + logdir);
process.exit(1);
}
console.log("Logs will be written to " + global.environment.log_filename);
log4js_config.appenders.push(
{ type: "file", filename: global.environment.log_filename }
);
} else {
log4js_config.appenders.push(
{ type: "console", layout: { type:'basic' } }
);
}
if ( global.environment.rollbar ) {
log4js_config.appenders.push({
type: __dirname + "/lib/cartodb/log4js_rollbar.js",
@@ -44,7 +62,7 @@ if ( global.environment.rollbar ) {
});
}
log4js.configure(log4js_config);
log4js.configure(log4js_config, { cwd: __dirname });
global.logger = log4js.getLogger();
// Include cartodb_windshaft only _after_ the "global" variable is set
@@ -80,6 +98,11 @@ process.on('SIGUSR2', function() {
ws.dumpCacheStats();
});
process.on('SIGHUP', function() {
log4js.configure(log4js_config);
console.log('Log files reloaded');
});
process.on('uncaughtException', function(err) {
logger.error('Uncaught exception: ' + err.stack);
});

View File

@@ -31,11 +31,15 @@ var config = {
// to be able to navigate the map without a reload ?
// Defaults to 7200 (2 hours)
,mapConfigTTL: 7200
// idle socket timeout, in miliseconds
// idle socket timeout, in milliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: false
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: 'logs/node-windshaft.log'
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@@ -58,9 +62,18 @@ var config = {
*/
row_limit: 65535,
simplify_geometries: true,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
,statsd: {
host: 'localhost',
port: 8125,
@@ -84,13 +97,17 @@ var config = {
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// There are currently 2 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// by 2 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 1, // idle time before dropping connection
reapIntervalMillis: 1 // time between cleanups
reapIntervalMillis: 1, // time between cleanups
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
}
}
,sqlapi: {
protocol: 'http',

View File

@@ -31,11 +31,15 @@ var config = {
// to be able to navigate the map without a reload ?
// Defaults to 7200 (2 hours)
,mapConfigTTL: 7200
// idle socket timeout, in miliseconds
// idle socket timeout, in milliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: 'logs/node-windshaft.log'
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'cartodb_user_<%= user_id %>'
@@ -51,10 +55,19 @@ var config = {
port: 6432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
max_size: 500
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
,statsd: {
host: 'localhost',
port: 8125,
@@ -78,13 +91,17 @@ var config = {
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// There are currently 2 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// by 2 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 30000, // idle time before dropping connection
reapIntervalMillis: 1000 // time between cleanups
reapIntervalMillis: 1000, // time between cleanups
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
}
}
,sqlapi: {
protocol: 'https',

View File

@@ -31,11 +31,15 @@ var config = {
// to be able to navigate the map without a reload ?
// Defaults to 7200 (2 hours)
,mapConfigTTL: 7200
// idle socket timeout, in miliseconds
// idle socket timeout, in milliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type]'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: 'logs/node-windshaft.log'
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'cartodb_staging_user_<%= user_id %>'
@@ -52,9 +56,18 @@ var config = {
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
row_limit: 65535,
simplify_geometries: true,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
,statsd: {
host: 'localhost',
port: 8125,
@@ -78,13 +91,17 @@ var config = {
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// There are currently 2 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// by 2 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 30000, // idle time before dropping connection
reapIntervalMillis: 1000 // time between cleanups
reapIntervalMillis: 1000, // time between cleanups
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
}
}
,sqlapi: {
protocol: 'https',

View File

@@ -31,11 +31,15 @@ var config = {
// to be able to navigate the map without a reload ?
// Defaults to 7200 (2 hours)
,mapConfigTTL: 7200
// idle socket timeout, in miliseconds
// idle socket timeout, in milliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: false
,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
//,log_filename: 'logs/node-windshaft.log'
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@@ -52,9 +56,18 @@ var config = {
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
row_limit: 65535,
simplify_geometries: true,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
}
,mapnik_version: ''
,mapnik_tile_format: 'png8:m=h'
,statsd: {
host: 'localhost',
port: 8125,
@@ -78,13 +91,17 @@ var config = {
// Max number of connections in each pool.
// Users will be put on a queue when the limit is hit.
// Set to maxConnection to have no possible queues.
// There are currently 3 pools involved in serving
// There are currently 2 pools involved in serving
// windshaft-cartodb requests so multiply this number
// by 3 to know how many possible connections will be
// by 2 to know how many possible connections will be
// kept open by the server. The default is 50.
max: 50,
idleTimeoutMillis: 1, // idle time before dropping connection
reapIntervalMillis: 1 // time between cleanups
reapIntervalMillis: 1, // time between cleanups
slowPool: {
log: true, // whether a slow acquire must be logged or not
elapsedThreshold: 25 // the threshold to determine an slow acquire must be reported or not
}
}
,sqlapi: {
protocol: 'http',

View File

@@ -1 +0,0 @@
module.exports.oneDay = 86400000;

111
docs/Map-API-internal.md Normal file
View File

@@ -0,0 +1,111 @@
# Kind of maps
Windshaft-CartoDB supports these kind of maps:
- [Temporary maps](#temporary-maps) (created by anyone)
- [Detached maps](#detached-maps)
- [Inline maps](#inline-maps) (legacy)
- [Persistent maps](#peristent-maps) (created by CartDB user)
- [Template maps](#template-maps)
- [Table maps](#table-maps) (legacy, deprecated)
## Temporary maps
Temporary maps have no owners and are anonymous in nature.
There are two kind of temporary maps:
- Detached maps (aka MultiLayer-API)
- Inline maps
### Detached maps
Detached maps are maps which are configured with a request
obtaining a temporary token and then used by referencing
the obtained token. The token expires automatically when unused.
Anyone can create detached maps, but users will need read access
to the data source of the map layers.
The configuration format is a [MapConfig]
(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
The HTTP endpoints for creating the map and using it are described [here]
(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
*TODO* cleanup the referenced document
### Inline maps
Inline maps are maps that only exist for a single request,
being the request for a specific map resource (tile).
Inline maps are always bound to a table, and can only be
obtained by those having read access to the that table.
Additionally, users need to have access to any datasource
specified as part of the configuration.
Inline maps only support PNG and UTF8GRID tiles.
The configuration consist in a set of parameters, to be
specified in the query string of the tile request:
* sql - the query to run as datasource, can be an array
* style - the CartoCSS style for the datasource, can be an array
* style_version - version of the CartoCSS style, can be an array
* interactivity - only for fetching UTF8GRID,
If the style is not provided, style of the associated table is
used; if the sql is not provided, all records of the associated
table are used as the datasource; the two possibilities result
in a mix between _inline_ maps and [Table maps][].
*TODO* specify (or link) api endpoints
## Persistent maps
Persistent maps can only be created by a CartoDB user who has full
responsibility over editing and deleting them. There are two
kind of persistent maps:
- Template maps
- Table maps (legacy, deprecated)
### Templated maps
Templated maps are templated [MapConfig]
(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
associated with an authorization certificate.
The authorization certificate determines who can instanciate the
template and use the resulting map. Authorized users of the instanciated
maps will have the same database access privilege of the template owner.
The HTTP endpoints for creating and using templated maps are described [here]
(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
*TODO* cleanup the referenced document
### Table maps
Table maps are maps associated with a table.
Configuration of such maps is limited to the CartoCSS style.
* style - the CartoCSS style for the datasource, can be an array
* style_version - version of the CartoCSS style, can be an array
You can only fetch PNG or UTF8GRID tiles from these maps.
Access method is the same as the one for [Inline maps](#inline-maps)
# Endpoints description
- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
NOTE: in case Multilayer-API does not contain this info yet, the
endpoint for fetching attributes is this:
- **/api/maps/:map_id/:layer_index/attributes/:feature_id**
- would return { c: 1, d: 2 }

View File

@@ -1,111 +1,638 @@
# Kind of maps
## Maps API
Windshaft-CartoDB supports these kind of maps:
The CartoDB Maps API allows you to generate maps based on data hosted in your CartoDB account and style them using CartoCSS. The API generates a XYZ based URL to fetch Web Mercator projected tiles using web clients like Leaflet, Google Maps, OpenLayers.
- [Temporary maps](#temporary-maps) (created by anyone)
- [Detached maps](#detached-maps)
- [Inline maps](#inline-maps) (legacy)
- [Persistent maps](#peristent-maps) (created by CartDB user)
- [Template maps](#template-maps)
- [Table maps](#table-maps) (legacy, deprecated)
You can create two types of maps with the Maps API:
## Temporary maps
- **Anonymous maps**
Maps that can be created using your CartoDB public data. Any client can change the read-only SQL and CartoCSS parameters that generate the map tiles. These maps can be created from a JavaScript application alone and no authenticated calls are needed. See [this CartoDB.js example]({{ '/cartodb-platform/cartodb-js.html' | prepend: site.baseurl }}).
Temporary maps have no owners and are anonymous in nature.
There are two kind of temporary maps:
- **Named maps**
Maps that access to your private data. These maps require an owner to setup and modify any SQL and CartoCSS parameters and are not modifiable without new setup calls.
- Detached maps (aka MultiLayer-API)
- Inline maps
## Quickstart
### Detached maps
### Anonymous maps
Detached maps are maps which are configured with a request
obtaining a temporary token and then used by referencing
the obtained token. The token expires automatically when unused.
Here is an example of how to create an anonymous map with JavaScript:
Anyone can create detached maps, but users will need read access
to the data source of the map layers.
{% highlight javascript %}
var mapconfig = {
"version": "1.0.1",
"layers": [{
"type": "cartodb",
"options": {
"cartocss_version": "2.1.1",
"cartocss": "#layer { polygon-fill: #FFF; }",
"sql": "select * from european_countries_e"
}
}]
}
The configuration format is a [MapConfig]
(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) document.
$.ajax({
crossOrigin: true,
type: 'POST',
dataType: 'json',
contentType: 'application/json',
url: 'http://documentation.cartodb.com/api/v1/map',
data: JSON.stringify(mapconfig),
success: function(data) {
var templateUrl = 'http://documentation.cartodb.com/api/v1/map/' + data.layergroupid + '{z}/{x}/{y}.png'
console.log(templateUrl);
}
})
{% endhighlight %}
The HTTP endpoints for creating the map and using it are described [here]
(http://github.com/CartoDB/Windshaft-cartodb/wiki/MultiLayer-API)
### Named maps
*TODO* cleanup the referenced document
Let's create a named map using some private tables in a CartoDB account.
The following API call creates a map of European countries that have a white fill color:
### Inline maps
{% highlight javascript %}
// mapconfig.json
{
"version": "0.0.1"
"name": "test",
"auth": {
"method": "open"
},
"layergroup": {
"layers": [{
"type": "cartodb",
"options": {
"cartocss_version": "2.1.1",
"cartocss": "#layer { polygon-fill: #FFF; }",
"sql": "select * from european_countries_e"
}
}]
}
}
{% endhighlight %}
Inline maps are maps that only exist for a single request,
being the request for a specific map resource (tile).
The map config needs to be sent to CartoDB's Map API using an authenticated call. Here we use a command line tool called `curl`. For more info about this tool see [this blog post](http://quickleft.com/blog/command-line-tutorials-curl) or type ``man curl`` in bash. Using `curl` the call would look like:
Inline maps are always bound to a table, and can only be
obtained by those having read access to the that table.
Additionally, users need to have access to any datasource
specified as part of the configuration.
<div class="code-title notitle code-request"></div>
{% highlight bash %}
curl 'https://{account}.cartodb.com/api/v1/map/named?api_key=APIKEY' -H 'Content-Type: application/json' -d @mapconfig.json
{% endhighlight %}
Inline maps only support PNG and UTF8GRID tiles.
To get the `URL` to fetch the tiles you need to instantiate the map.
The configuration consist in a set of parameters, to be
specified in the query string of the tile request:
<div class="code-title notitle code-request"></div>
{% highlight bash %}
curl 'http://{account}.cartodb.com/api/v1/map/named/test' -H 'Content-Type: application/json'
{% endhighlight %}
* sql - the query to run as datasource, can be an array
* style - the CartoCSS style for the datasource, can be an array
* style_version - version of the CartoCSS style, can be an array
* interactivity - only for fetching UTF8GRID,
The response will return JSON with properties for the `layergroupid` and the timestamp (`last_updated`) of the last data modification.
If the style is not provided, style of the associated table is
used; if the sql is not provided, all records of the associated
table are used as the datasource; the two possibilities result
in a mix between _inline_ maps and [Table maps][].
Here is an example response:
*TODO* specify (or link) api endpoints
{% highlight javascript %}
{
"layergroupid": "c01a54877c62831bb51720263f91fb33:0",
"last_updated": "1970-01-01T00:00:00.000Z"
}
{% endhighlight %}
## Persistent maps
You can use the `layergroupid` to instantiate a URL template for accessing tiles on the client. Here we use the `layergroupid` from the example response above in this URL template:
Persistent maps can only be created by a CartoDB user who has full
responsibility over editing and deleting them. There are two
kind of persistent maps:
{% highlight bash %}
http://documentation.cartodb.com/tiles/layergroup/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
{% endhighlight %}
- Template maps
- Table maps (legacy, deprecated)
## General Concepts
### Templated maps
The following concepts are the same for every endpoint in the API except when it's noted explicitly.
Templated maps are templated [MapConfig]
(http://github.com/CartoDB/Windshaft/wiki/MapConfig-specification) documents
associated with an authorization certificate.
### Auth
The authorization certificate determines who can instanciate the
template and use the resulting map. Authorized users of the instanciated
maps will have the same database access privilege of the template owner.
By default, users do not have access to private tables in CartoDB. In order to instantiate a map from private table data an API Key is required. Additionally, to include some endpoints an API Key must be included (e.g. creating a named map).
The HTTP endpoints for creating and using templated maps are described [here]
(http://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps).
To execute an authorized request, api_key=YOURAPIKEY should be added to the request URL. The param can be also passed as POST param. We **strongly advise** using HTTPS when you are performing requests that include your `api_key`.
*TODO* cleanup the referenced document
### Errors
### Table maps
Errors are reported using standard HTTP codes and extended information encoded in JSON with this format:
Table maps are maps associated with a table.
Configuration of such maps is limited to the CartoCSS style.
{% highlight javascript %}
{
"errors": [
"access forbidden to table TABLE"
]
}
{% endhighlight %}
* style - the CartoCSS style for the datasource, can be an array
* style_version - version of the CartoCSS style, can be an array
If you use JSONP, the 200 HTTP code is always returned so the JavaScript client can receive errors from the JSON object.
You can only fetch PNG or UTF8GRID tiles from these maps.
### CORS support
Access method is the same as the one for [Inline maps](#inline-maps)
All the endpoints which might be accessed using a web browser add CORS headers and allow OPTIONS method.
# Endpoints description
## Anonymous Maps
- **/api/maps/** (same interface than https://github.com/CartoDB/Windshaft/wiki/Multilayer-API)
- **/api/maps/named** (same interface than https://github.com/CartoDB/Windshaft-cartodb/wiki/Template-maps)
Anonymous maps allows you to instantiate a map given SQL and CartoCSS. It also allows you to add interaction capabilities using [UTF Grid.](https://github.com/mapbox/utfgrid-spec)
### Instantiate
NOTE: in case Multilayer-API does not contain this info yet, the
endpoint for fetching attributes is this:
#### Definition
- **/api/maps/:map_id/:layer_index/attributes/:feature_id**
- would return { c: 1, d: 2 }
<div class="code-title notitle code-request"></div>
{% highlight html %}
POST /api/v1/map
{% endhighlight %}
#### Params
{% highlight javascript %}
{
"version": "1.0.1",
"layers": [{
"type": "cartodb",
"options": {
"cartocss_version": "2.1.1",
"cartocss": "#layer { polygon-fill: #FFF; }",
"sql": "select * from european_countries_e",
"interactivity": ["cartodb_id", "iso3"]
}
}]
}
{% endhighlight %}
Should be a [Mapconfig](https://github.com/CartoDB/Windshaft/blob/0.19.1/doc/MapConfig-1.1.0.md).
#### Response
The response includes:
- **layergroupid**
The ID for that map, used to compose the URL for the tiles. The final URL is:
{% highlight html %}
http://{account}.cartodb.com/api/v1/map/:layergroupid/{z}/{x}/{y}.png
{% endhighlight %}
- **updated_at**
The ISO date of the last time the data involved in the query was updated.
- **metadata** *(optional)*
Includes information about the layers. Some layers may not have metadata.
- **cdn_url**
URLs to fetch the data using the best CDN for your zone.
#### Example
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl 'http://documentation.cartodb.com/api/v1/map' -H 'Content-Type: application/json' -d @mapconfig.json
{% endhighlight %}
<div class="code-title">RESPONSE</div>
{% highlight javascript %}
{
"layergroupid":"c01a54877c62831bb51720263f91fb33:0",
"last_updated":"1970-01-01T00:00:00.000Z"
"cdn_url": {
"http": "http://cdb.com",
"https": "https://cdb.com"
}
}
{% endhighlight %}
The tiles can be accessed using:
{% highlight bash %}
http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/{z}/{x}/{y}.png
{% endhighlight %}
For UTF grid tiles:
{% highlight bash %}
http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/{z}/{x}/{y}.grid.json
{% endhighlight %}
For attributes defined in `attributes` section:
{% highlight bash %}
http://documentation.cartodb.com/api/v1/map/c01a54877c62831bb51720263f91fb33:0/:layer/attributes/:feature_id
{% endhighlight %}
Which returns JSON with the attributes defined, like:
{% highlight javascript %}
{ c: 1, d: 2 }
{% endhighlight %}
Notice UTF Grid and attributes endpoints need an intenger parameter, ``layer``. That number is the 0-based index of the layer inside the mapconfig. So in this case 0 returns the UTF grid tiles/attributes for layer 0, the only layer in the example mapconfig. If a second layer was available it could be returned with 1, a third layer with 2, etc.
### Create JSONP
The JSONP endpoint is provided in order to allow web browsers access which don't support CORS.
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight bash %}
GET /api/v1/map?callback=method
{% endhighlight %}
#### Params
- **auth_token** *(optional)*
If the named map needs authorization.
- **config**
Encoded JSON with the params for creating named maps (the variables defined in the template).
- **lmza**
This attribute contains the same as config but LZMA compressed. It cannot be used at the same time as `config`.
- **callback**
JSON callback name.
#### Example
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl http://...
{% endhighlight %}
<div class="code-title">RESPONSE</div>
{% highlight javascript %}
{
}
{% endhighlight %}
### Remove
Anonymous maps cannot be removed by an API call. They will expire after about five minutes but sometimes longer. If an anonymous map expires and tiles are requested from it, an error will be raised. This could happen if a user leaves a map open and after time returns to the map an attempts to interact with it in a way that requires new tiles (e.g. zoom). The client will need to go through the steps of creating the map again to fix the problem.
## Named Maps
Named maps are essentially the same as anonymous maps but the mapconfig is stored in the server and given a unique name. Two other big differences are that you can created named maps from private data and that users without an API Key can see them even though they are from that private data.
The main two differences compared to anonymous maps are:
- **auth layer**
This allows you to control who is able to see the map based on a token auth
- **templates**
Since the mapconfig is static it can contain some variables so the client con modify the map appearance using those variables.
Template maps are persistent with no preset expiration. They can only be created or deleted by a CartoDB user with a valid API_KEY (see auth section).
### Create
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight html %}
POST /api/v1/map/named
{% endhighlight %}
#### Params
{% highlight javascript %}
// template.json
{
version: '0.0.1',
// there can be at most 1 template with the same name for any user
// valid names start with a letter and only contains letter, numbers
// or underscores
name: 'template_name',
auth: {
method: 'token', // or "open" (the default if no "method" is given)
valid_tokens: ['auth_token1','auth_token2'] // only (required and non empty) for 'token' method
},
// Variables not listed here are not substituted
// Variable not provided at instantiation time trigger an error
// A default is required for optional variables
// Type specification is used for quoting, to avoid injections
// see template format section below
placeholders: {
color: {
type:'css_color',
default:'red'
},
cartodb_id: {
type:'number',
default: 1
}
},
// the layer list definition
layergroup: {
// this is the MapConfig explained in anonymous maps
// see https://github.com/CartoDB/Windshaft/blob/0.19.1/doc/MapConfig-1.1.0.md)
"version": "1.0.1",
"layers": [{
"type": "cartodb",
"options": {
"cartocss_version": "2.1.1",
"cartocss": "#layer { polygon-fill: <%= color %>; }",
"sql": "select * from european_countries_e WHERE cartodb_id = <%= cartodb_id %>"
}
}]
}
}
{% endhighlight %}
#### Template Format
A templated `layergroup` allows using placeholders in the "cartocss" and "sql" elements of the "option" object in any "layer" of a layergroup configuration
Valid placeholder names start with a letter and can only contain letters, numbers or underscores. They have to be written between `<%=` and `%>` strings in order to be replaced.
##### Example
{% highlight javascript %}
<%= my_color %>
{% endhighlight %}
The set of supported placeholders for a template will need to be explicitly defined with a specific type and default value for each.
#### Placeholder Types
The placeholder type will determine the kind of escaping for the associated value. Supported types are:
- **sql_literal** internal single-quotes will be sql-escaped
- **sql_ident** internal double-quotes will be sql-escaped
- **number** can only contain numerical representation
- **css_color** can only contain color names or hex-values
Placeholder default values will be used whenever new values are not provided as options at the time of creation on the client. They can also be used to test the template by creating a default version with now options provided.
When using templates, be very careful about your selections as they can give broad access to your data if they are defined losely.
<div class="code-title code-request with-result">REQUEST</div>
{% highlight html %}
curl -X POST \
-H 'Content-Type: application/json' \
-d @template.json \
'https://docs.cartodb.com/api/v1/map/named?api_key=APIKEY'
{% endhighlight %}
<div class="code-title">RESPONSE</div>
{% highlight javascript %}
{
"templateid":"name",
}
{% endhighlight %}
### Instantiate
Instantiating a map allows you to get the information needed to fetch tiles. That temporal map is an anonymous map.
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight html %}
POST /api/v1/map/named/:template_name
{% endhighlight %}
#### Param
{% highlight javascript %}
// params.json
{
color: "#ff0000",
cartodb_id: 3
}
{% endhighlight %}
The fields you pass as `params.json` depend on the variables allowed by the named map. If there are variables missing it will raise an error (HTTP 400)
- **auth_token** *optional* if the named map needs auth
#### Example
You can initialize a template map by passing all of the required parameters in a POST to `/api/v1/map/named/:template_name`.
Valid credentials will be needed if required by the template.
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl -X POST \
-H 'Content-Type: application/json' \
-d @params.json \
'https://docs.cartodb.com/api/v1/template/@template_name?auth_token=AUTH_TOKEN'
{% endhighlight %}
<div class="code-title">Response</div>
{% highlight javascript %}
{
"layergroupid": "docs@fd2861af@c01a54877c62831bb51720263f91fb33:123456788",
"last_updated": "2013-11-14T11:20:15.000Z"
}
{% endhighlight %}
<div class="code-title">Error</div>
{% highlight javascript %}
{
"error": "Some error string here"
}
{% endhighlight %}
You can then use the `layergroupid` for fetching tiles and grids as you would normally (see anonymous map section). However, you'll need to show the `auth_token`, if required by the template.
### Using JSONP
There is also a special endpoint to be able to initialize a map using JSONP (for old browsers).
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight bash %}
GET /api/v1/map/named/:template_name/jsonp
{% endhighlight %}
#### Params
- **auth_token** *(optional)* If the named map needs auth
- **config** Encoded JSON with the params for creating named maps (the variables defined in the template)
- **lmza** This attribute contains the same as config but LZMA compressed. It cannot be used at the same time than `config`.
- **callback:** JSON callback name
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl 'https://docs.cartodb.com/api/v1/map/named/:template_name/jsonp?auth_token=AUTH_TOKEN&callback=function_name&config=template_params_json'
{% endhighlight %}
<div class="code-title">RESPONSE</div>
{% highlight javascript %}
callback(
"layergroupid":"c01a54877c62831bb51720263f91fb33:0",
"last_updated":"1970-01-01T00:00:00.000Z"
"cdn_url": {
"http": "http://cdb.com",
"https": "https://cdb.com"
}
)
{% endhighlight %}
This takes the `callback` function (required), `auth_token` if the template needs auth, and `config` which is the variable for the template (in cases where it has variables).
{% highlight javascript %}
url += "config=" + encodeURIComponent(
JSON.stringify({ color: 'red' });
{% endhighlight %}
The response is in this format:
{% highlight javascript %}
jQuery17205720721024554223_1390996319118({
layergroupid: "dev@744bd0ed9b047f953fae673d56a47b4d:1390844463021.1401",
last_updated: "2014-01-27T17:41:03.021Z"
})
{% endhighlight %}
### Update
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight bash %}
PUT /api/v1/map/:map_name
{% endhighlight %}
#### Params
Same params used to create a map.
#### Response
Same as updating a map.
#### Other Info
Updating a named map removes all the named map instances so they need to be initialized again.
#### Example
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl -X PUT \
-H 'Content-Type: application/json' \
-d @template.json \
'https://docs.cartodb.com/tiles/template/:template_name?api_key=APIKEY'
{% endhighlight %}
<div class="code-title">RESPONSE</div>
{% highlight javascript %}
{
"template_id": "@template_name"
}
{% endhighlight %}
If any template has the same name, it will be updated.
If a template with the same name does NOT exist, a 400 HTTP response is generated with an error in this format:
{% highlight javascript %}
{
"error": "error string here"
}
{% endhighlight %}
Updating a template map will also remove all signatures from previously initialized maps.
### Delete
Delete the specified template map from the server and disables any previously initialized versions of the map.
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight bash %}
DELETE /template/:template_name
{% endhighlight %}
#### Example
<div class="code-title code-request">REQUEST</div>
{% highlight bash %}
curl -X DELETE 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
{% endhighlight %}
<div class="code-title">RESPONSE</div>
{% highlight javascript %}
{
"error": "Some error string here"
}
{% endhighlight %}
On success, a 204 (No Content) response would be issued. Otherwise a 4xx response with with an error will be returned:
### Listing Available Templates
This allows you to get a list of all available templates.
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight bash %}
GET /api/v1/map/named/
{% endhighlight %}
#### Params
- **api_key** is required
#### Example
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl -X GET 'https://docs.cartodb.com/tiles/template?api_key=APIKEY'
{% endhighlight %}
<div class="code-title with-result">RESPONSE</div>
{% highlight javascript %}
{
"template_ids": ["@template_name1","@template_name2"]
}
{% endhighlight %}
<div class="code-title">ERROR</div>
{% highlight javascript %}
{
"error": "Some error string here"
}
{% endhighlight %}
### Getting a Specific Template
This gets the definition of a template
#### Definition
<div class="code-title notitle code-request"></div>
{% highlight bash %}
GET /api/v1/map/named/:template_name
{% endhighlight %}
#### Params
- **api_key** is required
#### Example
<div class="code-title code-request with-result">REQUEST</div>
{% highlight bash %}
curl -X GET 'https://docs.cartodb.com/tiles/template/@template_name?auth_token=AUTH_TOKEN'
{% endhighlight %}
<div class="code-title with-result">RESPONSE</div>
{% highlight javascript %}
{
"template": {...} // see template.json above
}
{% endhighlight %}
<div class="code-title">ERROR</div>
{% highlight javascript %}
{
"error": "Some error string here"
}
{% endhighlight %}

View File

@@ -49,16 +49,17 @@ certificate that would be used to sign any instance of the template.
```js
// template.json
{
version: '0.0.1',
version: "0.0.1",
// there can be at most 1 template with the same name for any user
// valid names start with a letter and only contains letter, numbers
// or underscores
name: 'template_name',
name: "template_name",
// embedded authorization certificate
auth: {
// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps
method: 'token', // or "open" (the default if no "method" is given)
valid_tokens: ['auth_token1','auth_token2'] // only (required and non empty) for 'token' method
method: "token", // or "open" (the default if no "method" is given)
// only (required and non empty) for "token" method
valid_tokens: ["auth_token1","auth_token2"]
},
// Variables not listed here are not substituted
// Variable not provided at instantiation time trigger an error
@@ -66,11 +67,11 @@ certificate that would be used to sign any instance of the template.
// Type specification is used for quoting, to avoid injections
placeholders: {
color: {
type:'css_color',
default:'red'
type:"css_color",
default:"red"
},
cartodb_id: {
type:'number',
type:"number",
default: 1
}
},

50
docs/metrics.md Normal file
View File

@@ -0,0 +1,50 @@
Windshaft-cartodb metrics
=========================
See [Windshaft metrics documentation](https://github.com/CartoDB/Windshaft/blob/master/doc/metrics.md) to understand the full picture.
The next list includes the API endpoints, each endpoint may have several inner timers, some of them are displayed within this list as subitems. Find the description for them in the Inner timers section.
## Timers
- **windshaft-cartodb.get_infowindow**: time to retrieve an infowindow popup
- **windshaft-cartodb.get_map_metadata**: time to retrieve metadata for embedded maps
- **windshaft-cartodb.flush_cache**: time to flush the tile and sql cache
- **windshaft-cartodb.get_template**: time to retrieve an specific template
- **windshaft-cartodb.delete_template**: time to delete an specific template
- **windshaft-cartodb.get_template_list**: time to retrieve the list of owned templates
- **windshaft-cartodb.instance_template_post**: time to create a template via HTTP POST
- **windshaft-cartodb.instance_template_get**: time to create a template via HTTP GET
+ TemplateMaps_instance
+ createLayergroup
There are some endpoints that are not being tracked:
- Adding a template
- Updating a template
### Inner timers
Again, each inner timer may have several inner timers.
- **addCacheChannel**: time to add X-Cache-Channel header based on table last modifications
- **LZMA decompress**: time to decompress request params with LZMA
- **TemplateMaps_instance**: time to retrieve a map template instance, see *getTemplate* and *authorizedByCert*
- **affectedTables**: time to check what are the affected tables for adding the cache channel, see *addCacheChannel*
- **authorize**: time to authorize a request, see *authorizedByAPIKey*, *authorizedByCert*, *authorizedBySigner*
- **authorizedByAPIKey**: time to authorize using an API KEY
- **authorizedByCert**: time to authorize a request by a cert, see [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
- **authorizedBySigner**: time to authorize a request for a [signed map](https://github.com/CartoDB/Windshaft-cartodb/wiki/Signed-maps)
- **cartoData.getTableGeometryType**: time to retrieve from redis the geom type for a given table
- **cors**: time to set the CORS headers
- **findLastUpdated**: time to retrieve the last update time for a list of tables, see *affectedTables*
- **fingerPrint**: time to create a fingerprint for a signed map
- **generateCacheChannel**: time to generate the headers for the cache channel based on the request, see *addCacheChannel*
- **getSignerMapKey**: time to retrieve from redis the authorized key for a signed map
- **getTablePrivacy**: time to retrieve from redis the privacy of a table
- **getTemplate**: time to retrieve from redis the template for a map
- **getUserMapKey**: time to retrieve from redis the user key for a map
- **incMapviewCount**: time to incremenent in redis the map views
- **mapStore_load**: time to retrieve from redis a map configuration
- **req2params.setup**: time to prepare the params from a request, see *req2params* in Windshaft documentation
- **setDBAuth**: time to retrieve from redis and set db user and db password from a user
- **setDBConn**: time to retrieve from redis and set db host and db name from a user
- **setDBParams**: time to prepare all db params to be able to connect/query a database, see *setDBAuth* and *setDBConn*
- **signMap**: time to sign in redis layergroup for a map, see signed maps
- **tablePrivacy_getUserDBName**: time to retrieve from redis the database for a user

View File

@@ -0,0 +1,95 @@
var sqlApi = require('../sql/sql_api');
function QueryTablesApi() {
}
var affectedTableRegexCache = {
bbox: /!bbox!/g,
pixel_width: /!pixel_width!/g,
pixel_height: /!pixel_height!/g
};
module.exports = QueryTablesApi;
QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, tableNames, callback) {
var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname = any (ARRAY['+
tableNames.map(function(t) { return "'" + t + "'::regclass"; }).join(',') +
'])';
// call sql api
sqlApi.query(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not find last updated timestamp: ' + msg));
return;
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var last_updated = 0;
if(rows.length !== 0) {
last_updated = rows[0].max || 0;
}
callback(null, last_updated*1000);
});
};
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql, callback) {
// Replace mapnik tokens
sql = sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1')
;
// Pass to CDB_QueryTables
sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
sqlApi.query(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
callback(null, tableNames);
});
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, api_key, sql, callback) {
sql = sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.pixel_width, '1')
.replace(affectedTableRegexCache.pixel_height, '1')
;
var query = [
'SELECT',
'CDB_QueryTables($windshaft$' + sql + '$windshaft$) as tablenames,',
'EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
'WHERE m.tabname = any (CDB_QueryTables($windshaft$' + sql + '$windshaft$)::regclass[])'
].join(' ');
sqlApi.query(username, api_key, query, function(err, rows){
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg));
return;
}
var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
var lastUpdatedTime = result.max || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
});
});
};

View File

@@ -2,7 +2,7 @@
var _ = require('underscore')
, Step = require('step')
, Windshaft = require('windshaft')
, redisPool = new require('redis-mpool')(global.environment.redis)
, redisPool = require('redis-mpool')(_.extend(global.environment.redis, {name: 'windshaft:cartodb'}))
// TODO: instanciate cartoData with redisPool
, cartoData = require('cartodb-redis')(global.environment.redis)
, SignedMaps = require('./signed_maps.js')
@@ -11,6 +11,9 @@ var _ = require('underscore')
, os = require('os')
;
if ( ! process.env['PGAPPNAME'] )
process.env['PGAPPNAME']='cartodb_tiler';
var CartodbWindshaft = function(serverOptions) {
var debug = global.environment.debug;
@@ -237,8 +240,7 @@ var CartodbWindshaft = function(serverOptions) {
},
function finish(err, response){
if ( req.profiler ) {
var report = req.profiler.toString();
res.header('X-Tiler-Profiler', report);
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
if (err){
response = { error: ''+err };
@@ -294,8 +296,7 @@ var CartodbWindshaft = function(serverOptions) {
},
function finish(err, response){
if ( req.profiler ) {
var report = req.profiler.toString();
res.header('X-Tiler-Profiler', report);
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
if (err){
var statusCode = 400;
@@ -554,6 +555,8 @@ var CartodbWindshaft = function(serverOptions) {
if ( err ) throw err;
layergroup = instance;
fakereq = { query: {}, params: {}, headers: _.clone(req.headers),
method: req.method,
res: res,
profiler: req.profiler
};
ws.setDBParams(cdbuser, fakereq.params, this);
@@ -604,8 +607,7 @@ var CartodbWindshaft = function(serverOptions) {
function finish_instanciation(err, response, res, req) {
if ( req.profiler ) {
var report = req.profiler.toString();
res.header('X-Tiler-Profiler', report);
res.header('X-Tiler-Profiler', req.profiler.toJSONString());
}
if (err) {
var statusCode = 400;

View File

@@ -1,10 +1,10 @@
var _ = require('underscore')
, Step = require('step')
, cartoData = require('cartodb-redis')(global.environment.redis)
, Cache = require('./cache_validator')
, Cache = require('./cache_validator')
, QueryTablesApi = require('./api/query_tables_api')
, mapnik = require('mapnik')
, crypto = require('crypto')
, request = require('request')
, LZMA = require('lzma/lzma_worker.js').LZMA
;
@@ -19,6 +19,8 @@ if ( _.isUndefined(global.environment.sqlapi.domain) ) {
module.exports = function(){
var queryTablesApi = new QueryTablesApi();
var rendererConfig = _.defaults(global.environment.renderer || {}, {
cache_ttl: 60000, // milliseconds
metatile: 4,
@@ -47,6 +49,7 @@ module.exports = function(){
datasource: global.environment.postgres,
cachedir: global.environment.millstone.cache_basedir,
mapnik_version: global.environment.mapnik_version || mapnik.versions.mapnik,
mapnik_tile_format: global.environment.mapnik_tile_format || 'png',
default_layergroup_ttl: global.environment.mapConfigTTL || 7200,
gc_prob: 0.01 // @deprecated since Windshaft-1.8.0
},
@@ -88,120 +91,6 @@ module.exports = function(){
// we have no SQL after layer creation.
me.channelCache = {};
// Run a query through the SQL api
me.sqlQuery = function (username, api_key, sql, callback) {
var api = global.environment.sqlapi;
// build up api string
var sqlapihostname = username;
if ( api.domain ) sqlapihostname += '.' + api.domain;
var sqlapi = api.protocol + '://';
if ( api.host && api.host != api.domain ) sqlapi += api.host;
else sqlapi += sqlapihostname;
sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
var qs = { q: sql }
// add api_key if given
if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
// call sql api
//
// NOTE: using POST to avoid size limits:
// See http://github.com/CartoDB/Windshaft-cartodb/issues/111
//
// NOTE: uses "host" header to allow IP based specification
// of sqlapi address (and avoid a DNS lookup)
//
// NOTE: allows for keeping up to "maxConnections" concurrent
// sockets opened per SQL-API host.
// See http://nodejs.org/api/http.html#http_agent_maxsockets
//
var maxSockets = global.environment.maxConnections || 128;
var maxGetLen = api.max_get_sql_length || 2048;
var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
var reqSpec = {
url:sqlapi,
json:true,
headers:{host: sqlapihostname}
// http://nodejs.org/api/http.html#http_agent_maxsockets
,pool:{maxSockets:maxSockets}
// timeout in milliseconds
,timeout:maxSQLTime
}
if ( sql.length > maxGetLen ) {
reqSpec.method = 'POST';
reqSpec.body = qs;
} else {
reqSpec.method = 'GET';
reqSpec.qs = qs;
}
request(reqSpec, function(err, res, body) {
if (err){
console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
callback(err);
return;
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error(msg));
console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
return;
}
callback(null, body.rows);
});
};
//
// Invoke callback with number of milliseconds since
// last update in any of the given tables
//
me.findLastUpdated = function (username, api_key, tableNames, callback) {
var sql = 'SELECT EXTRACT(EPOCH FROM max(updated_at)) as max FROM CDB_TableMetadata m WHERE m.tabname::name = any (\'{'
+ tableNames.join(',') + '}\')';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not find last updated timestamp: ' + msg));
return;
}
// when the table has not updated_at means it hasn't been changed so a default last_updated is set
var last_updated = 0;
if(rows.length !== 0) {
last_updated = rows[0].max || 0;
}
callback(null, last_updated*1000);
});
};
me.affectedTables = function (username, api_key, sql, callback) {
// Replace mapnik tokens
sql = sql.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
.replace(RegExp('!pixel_width!', 'g'), '1')
.replace(RegExp('!pixel_height!', 'g'), '1')
;
// Pass to CDB_QueryTables
sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
me.sqlQuery(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
callback(null, tableNames);
});
};
me.buildCacheChannel = function (dbName, tableNames){
return dbName + ':' + tableNames.join(',');
};
@@ -210,7 +99,7 @@ module.exports = function(){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
};
me.generateCacheChannel = function(app, req, callback){
@@ -240,7 +129,6 @@ module.exports = function(){
// See http://github.com/CartoDB/Windshaft-cartodb/issues/152
if ( ! app.mapStore ) {
throw new Error('missing channel cache for token ' + req.params.token);
return;
}
var next = this;
var mapStore = app.mapStore;
@@ -304,7 +192,7 @@ module.exports = function(){
if ( req.profiler ) req.profiler.done('getSignerMapKey');
key = data;
}
me.affectedTables(user, key, sql, this); // in addCacheChannel
queryTablesApi.getAffectedTablesInQuery(user, key, sql, this); // in addCacheChannel
},
function finish(err, data) {
next(err,data);
@@ -396,7 +284,7 @@ module.exports = function(){
err = errors.length ? new Error(errors.join('\n')) : null;
callback(err);
}
}
};
// include in layergroup response the variables in serverMedata
// those variables are useful to send to the client information
@@ -426,33 +314,37 @@ module.exports = function(){
var key = req.params.map_key || req.params.api_key;
var cacheKey = dbName + ':' + token;
var tabNames;
Step(
function getTables() {
me.affectedTables(usr, key, sql, this); // in afterLayergroupCreate
},
function getLastupdated(err, tableNames) {
if (req.profiler) req.profiler.done('affectedTables');
if ( err ) throw err;
var cacheChannel = me.buildCacheChannel(dbName,tableNames);
// store for caching from me.afterLayergroupCreate
me.channelCache[cacheKey] = cacheChannel;
// find last updated
if ( ! tableNames.length ) return 0; // skip for no affected tables
tabNames = tableNames;
me.findLastUpdated(usr, key, tableNames, this);
},
function(err, lastUpdated) {
if ( err ) throw err;
if (req.profiler && tabNames) req.profiler.done('findLastUpdated');
response.layergroupid = response.layergroupid + ':' + lastUpdated; // use epoch
response.last_updated = new Date(lastUpdated).toISOString();
return null;
},
function finish(err) {
done(err);
}
function getAffectedTablesAndLastUpdatedTime() {
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, key, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');
if ( err ) throw err;
var cacheChannel = me.buildCacheChannel(dbName, result.affectedTables);
me.channelCache[cacheKey] = cacheChannel;
if (req.res && req.method == 'GET') {
var res = req.res;
if ( req.query && req.query.cache_policy == 'persist' ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
var ttl = global.environment.varnish.ttl || 86400;
res.header('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
}
res.header('Last-Modified', (new Date()).toUTCString());
res.header('X-Cache-Channel', cacheChannel);
}
// last update for layergroup cache buster
response.layergroupid = response.layergroupid + ':' + result.lastUpdatedTime;
response.last_updated = new Date(result.lastUpdatedTime).toISOString();
return null;
},
function finish(err) {
done(err);
}
);
};
@@ -478,7 +370,7 @@ module.exports = function(){
return;
}
return mat[1];
}
};
// Set db authentication parameters to those of the given username
//
@@ -547,21 +439,20 @@ module.exports = function(){
dbport: global.environment.postgres.port
});
Step(
function getDatabaseHost(){
cartoData.getUserDBHost(dbowner, this);
function getConnectionParams() {
cartoData.getUserDBConnectionParams(dbowner, this);
},
function getDatabase(err, data){
if(err) throw err;
if ( data ) _.extend(params, {dbhost:data});
cartoData.getUserDBName(dbowner, this);
},
function extendParams(err, data){
function extendParams(err, dbParams){
if (err) throw err;
if ( data ) _.extend(params, {dbname:data});
// we don't want null values or overwrite a non public user
if (params.dbuser != 'publicuser' || !dbParams.dbuser) {
delete dbParams.dbuser;
}
if ( dbParams ) _.extend(params, dbParams);
return null;
},
function finish(err) {
callback(err);
callback(err);
}
);
};
@@ -731,7 +622,7 @@ module.exports = function(){
//console.log("type of req.query.lzma is " + typeof(req.query.lzma));
// Decode (from base64)
var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
var lzma = (new Buffer(req.query.lzma, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 });
// Decompress
LZMA.decompress(
@@ -739,8 +630,8 @@ module.exports = function(){
function(result) {
if (req.profiler) req.profiler.done('LZMA decompress');
try {
delete req.query.lzma
_.extend(req.query, JSON.parse(result))
delete req.query.lzma;
_.extend(req.query, JSON.parse(result));
me.req2params(req, callback);
} catch (err) {
callback(new Error('Error parsing lzma as JSON: ' + err));
@@ -772,7 +663,7 @@ module.exports = function(){
req.params.signer = tksplit.shift();
if ( ! req.params.signer ) req.params.signer = user;
else if ( req.params.signer != user ) {
var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"')
var err = new Error('Cannot use map signature of user "' + req.params.signer + '" on database of user "' + user + '"');
err.http_status = 403;
callback(err);
return;

View File

@@ -128,6 +128,7 @@ console.log("Cert is : "); console.dir(cert);
// Check if the given certificate authorizes waiver of "auth"
o.authorizedByCert = function(cert, auth) {
auth = _.isArray(auth) ? auth : [auth];
var err = this.checkInvalidCertificate(cert);
if ( err ) throw err;
@@ -139,12 +140,7 @@ o.authorizedByCert = function(cert, auth) {
// Token based authentication requires valid token
if ( method === 'token' ) {
var found = cert.auth.valid_tokens.indexOf(auth);
//if ( found !== -1 ) {
//console.log("Token " + auth + " is found at position " + found + " in valid tokens " + cert.auth.valid_tokens);
// return true;
//} else return false;
return cert.auth.valid_tokens.indexOf(auth) !== -1;
return _.intersection(cert.auth.valid_tokens, auth).length > 0;
}
throw new Error("Unsupported authentication method: " + cert.auth.method);

View File

@@ -0,0 +1,66 @@
var _ = require('underscore'),
request = require('request');
module.exports.query = function (username, api_key, sql, callback) {
var api = global.environment.sqlapi;
// build up api string
var sqlapihostname = username;
if ( api.domain ) sqlapihostname += '.' + api.domain;
var sqlapi = api.protocol + '://';
if ( api.host && api.host != api.domain ) sqlapi += api.host;
else sqlapi += sqlapihostname;
sqlapi += ':' + api.port + '/api/' + api.version + '/sql';
var qs = { q: sql };
// add api_key if given
if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
// call sql api
//
// NOTE: using POST to avoid size limits:
// See http://github.com/CartoDB/Windshaft-cartodb/issues/111
//
// NOTE: uses "host" header to allow IP based specification
// of sqlapi address (and avoid a DNS lookup)
//
// NOTE: allows for keeping up to "maxConnections" concurrent
// sockets opened per SQL-API host.
// See http://nodejs.org/api/http.html#http_agent_maxsockets
//
var maxSockets = global.environment.maxConnections || 128;
var maxGetLen = api.max_get_sql_length || 2048;
var maxSQLTime = api.timeout || 100; // 1/10 of a second by default
var reqSpec = {
url:sqlapi,
json:true,
headers:{host: sqlapihostname}
// http://nodejs.org/api/http.html#http_agent_maxsockets
,pool:{maxSockets:maxSockets}
// timeout in milliseconds
,timeout:maxSQLTime
};
if ( sql.length > maxGetLen ) {
reqSpec.method = 'POST';
reqSpec.body = qs;
} else {
reqSpec.method = 'GET';
reqSpec.qs = qs;
}
request(reqSpec, function(err, res, body) {
if (err){
console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
callback(err);
return;
}
if (res.statusCode != 200) {
var msg = res.body.error ? res.body.error : res.body;
callback(new Error(msg));
console.log('unexpected response status (' + res.statusCode + ') for sql query: ' + sql + ': ' + msg);
return;
}
callback(null, body.rows);
});
};

697
npm-shrinkwrap.json generated
View File

@@ -1,6 +1,6 @@
{
"name": "windshaft-cartodb",
"version": "1.10.2",
"version": "1.15.0",
"dependencies": {
"node-varnish": {
"version": "0.3.0",
@@ -10,11 +10,12 @@
"version": "1.3.3"
},
"windshaft": {
"version": "0.20.0",
"from": "http://github.com/CartoDB/Windshaft/tarball/0.20.0",
"version": "0.24.0",
"from": "https://github.com/CartoDB/Windshaft/tarball/0.24.0",
"dependencies": {
"grainstore": {
"version": "0.18.1",
"version": "0.20.0",
"from": "git://github.com/CartoDB/grainstore.git#0.20.0",
"dependencies": {
"carto": {
"version": "0.9.5-cdb2",
@@ -38,23 +39,23 @@
"version": "0.0.2"
},
"minimist": {
"version": "0.0.8"
"version": "0.0.10"
}
}
}
}
},
"mapnik-reference": {
"version": "5.0.7"
"version": "5.0.9"
},
"millstone": {
"version": "0.6.11",
"version": "0.6.14",
"dependencies": {
"underscore": {
"version": "1.5.2"
"version": "1.6.0"
},
"request": {
"version": "2.26.0",
"version": "2.34.0",
"dependencies": {
"qs": {
"version": "0.6.6"
@@ -65,6 +66,33 @@
"forever-agent": {
"version": "0.5.2"
},
"node-uuid": {
"version": "1.4.1"
},
"tough-cookie": {
"version": "0.12.1",
"dependencies": {
"punycode": {
"version": "1.2.4"
}
}
},
"form-data": {
"version": "0.1.4",
"dependencies": {
"combined-stream": {
"version": "0.0.5",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"async": {
"version": "0.9.0"
}
}
},
"tunnel-agent": {
"version": "0.3.0"
},
@@ -82,6 +110,9 @@
}
}
},
"oauth-sign": {
"version": "0.3.0"
},
"hawk": {
"version": "1.0.0",
"dependencies": {
@@ -99,50 +130,252 @@
}
}
},
"aws-sign": {
"version": "0.3.0"
"aws-sign2": {
"version": "0.5.0"
}
}
},
"srs": {
"version": "0.4.3",
"dependencies": {
"nan": {
"version": "1.2.0"
},
"oauth-sign": {
"version": "0.3.0"
},
"cookie-jar": {
"version": "0.3.0"
},
"node-uuid": {
"version": "1.4.1"
},
"form-data": {
"version": "0.1.2",
"node-pre-gyp": {
"version": "0.5.17",
"dependencies": {
"combined-stream": {
"version": "0.0.4",
"nopt": {
"version": "2.2.1",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
"abbrev": {
"version": "1.0.5"
}
}
},
"async": {
"version": "0.2.10"
"npmlog": {
"version": "0.0.6",
"dependencies": {
"ansi": {
"version": "0.2.1"
}
}
},
"request": {
"version": "2.36.0",
"dependencies": {
"qs": {
"version": "0.6.6"
},
"json-stringify-safe": {
"version": "5.0.0"
},
"mime": {
"version": "1.2.11"
},
"forever-agent": {
"version": "0.5.2"
},
"node-uuid": {
"version": "1.4.1"
},
"tough-cookie": {
"version": "0.12.1",
"dependencies": {
"punycode": {
"version": "1.2.4"
}
}
},
"form-data": {
"version": "0.1.2",
"dependencies": {
"combined-stream": {
"version": "0.0.4",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"async": {
"version": "0.2.10"
}
}
},
"tunnel-agent": {
"version": "0.4.0"
},
"http-signature": {
"version": "0.10.0",
"dependencies": {
"assert-plus": {
"version": "0.1.2"
},
"asn1": {
"version": "0.1.11"
},
"ctype": {
"version": "0.5.2"
}
}
},
"oauth-sign": {
"version": "0.3.0"
},
"hawk": {
"version": "1.0.0",
"dependencies": {
"hoek": {
"version": "0.9.1"
},
"boom": {
"version": "0.4.2"
},
"cryptiles": {
"version": "0.2.2"
},
"sntp": {
"version": "0.2.4"
}
}
},
"aws-sign2": {
"version": "0.5.0"
}
}
},
"semver": {
"version": "2.3.0"
},
"tar": {
"version": "0.1.19",
"dependencies": {
"inherits": {
"version": "2.0.1"
},
"block-stream": {
"version": "0.0.7"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"mkdirp": {
"version": "0.3.5"
},
"graceful-fs": {
"version": "2.0.3"
}
}
}
}
},
"tar-pack": {
"version": "2.0.0",
"dependencies": {
"uid-number": {
"version": "0.0.3"
},
"once": {
"version": "1.1.1"
},
"debug": {
"version": "0.7.4"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"mkdirp": {
"version": "0.3.5"
},
"graceful-fs": {
"version": "2.0.3"
},
"inherits": {
"version": "2.0.1"
}
}
},
"fstream-ignore": {
"version": "0.0.7",
"dependencies": {
"minimatch": {
"version": "0.2.14",
"dependencies": {
"lru-cache": {
"version": "2.5.0"
},
"sigmund": {
"version": "1.0.0"
}
}
},
"inherits": {
"version": "2.0.1"
}
}
},
"readable-stream": {
"version": "1.0.27-1",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.25-1"
},
"inherits": {
"version": "2.0.1"
}
}
},
"graceful-fs": {
"version": "1.2.3"
}
}
},
"mkdirp": {
"version": "0.5.0",
"dependencies": {
"minimist": {
"version": "0.0.8"
}
}
},
"rc": {
"version": "0.4.0",
"dependencies": {
"minimist": {
"version": "0.0.10"
},
"deep-extend": {
"version": "0.2.10"
},
"strip-json-comments": {
"version": "0.1.3"
},
"ini": {
"version": "1.1.0"
}
}
},
"rimraf": {
"version": "2.2.8"
}
}
}
}
},
"srs": {
"version": "0.3.11"
},
"zipfile": {
"version": "0.4.3"
},
"sqlite3": {
"version": "2.2.0",
"version": "0.5.2",
"dependencies": {
"node-pre-gyp": {
"version": "0.2.6",
"version": "0.5.8",
"dependencies": {
"nopt": {
"version": "2.1.2",
"version": "2.2.0",
"dependencies": {
"abbrev": {
"version": "1.0.4"
@@ -157,8 +390,92 @@
}
}
},
"request": {
"version": "2.34.0",
"dependencies": {
"qs": {
"version": "0.6.6"
},
"json-stringify-safe": {
"version": "5.0.0"
},
"forever-agent": {
"version": "0.5.2"
},
"node-uuid": {
"version": "1.4.1"
},
"mime": {
"version": "1.2.11"
},
"tough-cookie": {
"version": "0.12.1",
"dependencies": {
"punycode": {
"version": "1.2.4"
}
}
},
"form-data": {
"version": "0.1.2",
"dependencies": {
"combined-stream": {
"version": "0.0.4",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"async": {
"version": "0.2.10"
}
}
},
"tunnel-agent": {
"version": "0.3.0"
},
"http-signature": {
"version": "0.10.0",
"dependencies": {
"assert-plus": {
"version": "0.1.2"
},
"asn1": {
"version": "0.1.11"
},
"ctype": {
"version": "0.5.2"
}
}
},
"oauth-sign": {
"version": "0.3.0"
},
"hawk": {
"version": "1.0.0",
"dependencies": {
"hoek": {
"version": "0.9.1"
},
"boom": {
"version": "0.4.2"
},
"cryptiles": {
"version": "0.2.2"
},
"sntp": {
"version": "0.2.4"
}
}
},
"aws-sign2": {
"version": "0.5.0"
}
}
},
"semver": {
"version": "2.1.0"
"version": "2.2.1"
},
"tar": {
"version": "0.1.19",
@@ -208,6 +525,9 @@
"minimatch": {
"version": "0.2.14",
"dependencies": {
"lru-cache": {
"version": "2.5.0"
},
"sigmund": {
"version": "1.0.0"
}
@@ -219,10 +539,19 @@
}
},
"readable-stream": {
"version": "1.0.26-2",
"version": "1.0.26-4",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.25-1"
},
"inherits": {
"version": "2.0.1"
}
}
},
@@ -231,22 +560,225 @@
}
}
},
"aws-sdk": {
"version": "2.0.0-rc9",
"mkdirp": {
"version": "0.3.5"
},
"rc": {
"version": "0.3.4",
"dependencies": {
"xml2js": {
"version": "0.2.4",
"minimist": {
"version": "0.0.8"
},
"deep-extend": {
"version": "0.2.8"
},
"ini": {
"version": "1.1.0"
}
}
},
"rimraf": {
"version": "2.2.6"
}
}
}
}
},
"sqlite3": {
"version": "2.2.3",
"dependencies": {
"node-pre-gyp": {
"version": "0.5.9",
"dependencies": {
"nopt": {
"version": "2.2.0",
"dependencies": {
"abbrev": {
"version": "1.0.4"
}
}
},
"npmlog": {
"version": "0.0.6",
"dependencies": {
"ansi": {
"version": "0.2.1"
}
}
},
"request": {
"version": "2.34.0",
"dependencies": {
"qs": {
"version": "0.6.6"
},
"json-stringify-safe": {
"version": "5.0.0"
},
"forever-agent": {
"version": "0.5.2"
},
"node-uuid": {
"version": "1.4.1"
},
"mime": {
"version": "1.2.11"
},
"tough-cookie": {
"version": "0.12.1",
"dependencies": {
"sax": {
"version": "0.6.0"
"punycode": {
"version": "1.2.4"
}
}
},
"xmlbuilder": {
"version": "0.4.2"
"form-data": {
"version": "0.1.2",
"dependencies": {
"combined-stream": {
"version": "0.0.4",
"dependencies": {
"delayed-stream": {
"version": "0.0.5"
}
}
},
"async": {
"version": "0.2.10"
}
}
},
"tunnel-agent": {
"version": "0.3.0"
},
"http-signature": {
"version": "0.10.0",
"dependencies": {
"assert-plus": {
"version": "0.1.2"
},
"asn1": {
"version": "0.1.11"
},
"ctype": {
"version": "0.5.2"
}
}
},
"oauth-sign": {
"version": "0.3.0"
},
"hawk": {
"version": "1.0.0",
"dependencies": {
"hoek": {
"version": "0.9.1"
},
"boom": {
"version": "0.4.2"
},
"cryptiles": {
"version": "0.2.2"
},
"sntp": {
"version": "0.2.4"
}
}
},
"aws-sign2": {
"version": "0.5.0"
}
}
},
"semver": {
"version": "2.2.1"
},
"tar": {
"version": "0.1.19",
"dependencies": {
"inherits": {
"version": "2.0.1"
},
"block-stream": {
"version": "0.0.7"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"graceful-fs": {
"version": "2.0.3"
}
}
}
}
},
"tar-pack": {
"version": "2.0.0",
"dependencies": {
"uid-number": {
"version": "0.0.3"
},
"once": {
"version": "1.1.1"
},
"debug": {
"version": "0.7.4"
},
"fstream": {
"version": "0.1.25",
"dependencies": {
"graceful-fs": {
"version": "2.0.3"
},
"inherits": {
"version": "2.0.1"
}
}
},
"fstream-ignore": {
"version": "0.0.7",
"dependencies": {
"minimatch": {
"version": "0.2.14",
"dependencies": {
"lru-cache": {
"version": "2.5.0"
},
"sigmund": {
"version": "1.0.0"
}
}
},
"inherits": {
"version": "2.0.1"
}
}
},
"readable-stream": {
"version": "1.0.26-4",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.25-1"
},
"inherits": {
"version": "2.0.1"
}
}
},
"graceful-fs": {
"version": "1.2.3"
}
}
},
"mkdirp": {
"version": "0.3.5"
},
"rc": {
"version": "0.3.4",
"dependencies": {
@@ -281,7 +813,7 @@
"version": "0.0.2"
},
"minimist": {
"version": "0.0.8"
"version": "0.0.10"
}
}
}
@@ -299,7 +831,7 @@
"version": "1.9.2",
"dependencies": {
"formidable": {
"version": "1.0.14"
"version": "1.0.15"
}
}
},
@@ -331,11 +863,8 @@
}
},
"tilelive-mapnik": {
"version": "0.6.8",
"version": "0.6.9",
"dependencies": {
"eio": {
"version": "0.2.2"
},
"mime": {
"version": "1.2.11"
},
@@ -344,9 +873,6 @@
}
}
},
"lru-cache": {
"version": "2.3.1"
},
"carto": {
"version": "0.9.5-cdb3",
"from": "http://github.com/CartoDB/carto/tarball/0.9.5-cdb3",
@@ -355,7 +881,7 @@
"version": "1.4.4"
},
"mapnik-reference": {
"version": "5.0.7"
"version": "5.0.9"
},
"xml2js": {
"version": "0.2.8",
@@ -372,23 +898,15 @@
"version": "0.0.2"
},
"minimist": {
"version": "0.0.8"
"version": "0.0.10"
}
}
}
}
},
"step-profiler": {
"version": "0.0.1",
"from": "git://github.com/CartoDB/node-step-profiler.git#0.0.1"
},
"underscore.string": {
"version": "1.1.6",
"dependencies": {
"underscore": {
"version": "1.1.7"
}
}
"version": "0.1.0",
"from": "git://github.com/CartoDB/node-step-profiler.git#0.1.0"
},
"pg": {
"version": "2.6.2",
@@ -402,7 +920,7 @@
}
},
"torque.js": {
"version": "2.2.00"
"version": "2.2.0"
},
"node-statsd": {
"version": "0.0.7"
@@ -416,20 +934,24 @@
"version": "2.9.202"
},
"cartodb-redis": {
"version": "0.3.0"
"version": "0.9.0",
"from": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0"
},
"redis-mpool": {
"version": "0.0.4",
"from": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
"dependencies": {
"generic-pool": {
"version": "2.0.4"
},
"hiredis": {
"version": "0.1.16",
"version": "0.1.17",
"dependencies": {
"bindings": {
"version": "1.1.1"
"version": "1.2.0"
},
"nan": {
"version": "1.1.2"
}
}
}
@@ -443,29 +965,41 @@
"version": "1.2.3"
},
"log4js": {
"version": "0.6.10",
"version": "0.6.14",
"dependencies": {
"async": {
"version": "0.1.15"
},
"readable-stream": {
"version": "1.0.26",
"version": "1.0.27-1",
"dependencies": {
"core-util-is": {
"version": "1.0.1"
},
"isarray": {
"version": "0.0.1"
},
"string_decoder": {
"version": "0.10.25-1"
},
"inherits": {
"version": "2.0.1"
}
}
}
}
},
"rollbar": {
"version": "0.3.1",
"version": "0.3.8",
"dependencies": {
"node-uuid": {
"version": "1.4.1"
},
"lru-cache": {
"version": "2.2.4"
},
"json-stringify-safe": {
"version": "5.0.0"
}
}
},
@@ -502,7 +1036,12 @@
"version": "1.0.7"
},
"debug": {
"version": "0.7.4"
"version": "1.0.2",
"dependencies": {
"ms": {
"version": "0.6.2"
}
}
},
"mkdirp": {
"version": "0.3.5"
@@ -522,7 +1061,7 @@
}
},
"graceful-fs": {
"version": "2.0.2"
"version": "2.0.3"
},
"inherits": {
"version": "2.0.1"

View File

@@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "1.10.2",
"version": "1.15.0",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@@ -24,11 +24,11 @@
"dependencies": {
"node-varnish": "http://github.com/Vizzuality/node-varnish/tarball/0.3.0",
"underscore" : "~1.3.3",
"windshaft" : "http://github.com/CartoDB/Windshaft/tarball/0.20.0",
"step": "0.0.x",
"windshaft": "https://github.com/CartoDB/Windshaft/tarball/0.24.0",
"step": "~0.0.5",
"request": "2.9.202",
"cartodb-redis": "~0.3.0",
"redis-mpool": "http://github.com/CartoDB/node-redis-mpool/tarball/0.0.4",
"cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0",
"redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
"mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1",
"lzma": "~1.2.3",
"log4js": "~0.6.10",

View File

@@ -5,6 +5,8 @@ OPT_CREATE_PGSQL=yes # create the PostgreSQL test environment
OPT_DROP_REDIS=yes # drop the redis test environment
OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment
export PGAPPNAME=cartodb_tiler_tester
cd $(dirname $0)
BASEDIR=$(pwd)
cd -

View File

@@ -14,6 +14,9 @@ var helper = require(__dirname + '/../support/test_helper');
var windshaft_fixtures = __dirname + '/../../node_modules/windshaft/test/fixtures';
var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20;
var IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL = 25;
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var ServerOptions = require(__dirname + '/../../lib/cartodb/server_options');
serverOptions = ServerOptions();
@@ -108,12 +111,16 @@ suite('multilayer', function() {
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql + ';'
+ layergroup.layers[1].options.sql
+ '$windshaft$)');
+ expectedQuery
+ '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$)::regclass[])');
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', 2,
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL,
function(err, similarity) {
next(err);
});
@@ -226,9 +233,7 @@ suite('multilayer', function() {
});
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
// NOTE: another test like this is in templates.js
test("get creation requests no cache", function(done) {
test("get creation requests has cache", function(done) {
var layergroup = {
version: '1.0.0',
@@ -257,7 +262,7 @@ suite('multilayer', function() {
assert.equal(res.statusCode, 200, res.body);
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkNoCache(res);
helper.checkCache(res);
return null;
},
function finish(err) {
@@ -279,6 +284,49 @@ suite('multilayer', function() {
);
});
test("get creation has no cache if sql is bogus", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select bogus(0,0) as the_geom_webmercator',
cartocss: '#layer { polygon-fill: red; }',
cartocss_version: '2.0.1'
} }
]
};
assert.response(server, {
url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
method: 'GET',
headers: {host: 'localhost'}
}, {}, function(res) {
assert.notEqual(res.statusCode, 200);
helper.checkNoCache(res);
done();
});
});
test("get creation has no cache if cartocss is not valid", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom_webmercator, 5e6, 0) as the_geom_webmercator from test_table limit 2',
cartocss: '#layer { invalid-rule:red; }',
cartocss_version: '2.0.1'
} }
]
};
assert.response(server, {
url: '/tiles/layergroup?config=' + encodeURIComponent(JSON.stringify(layergroup)),
method: 'GET',
headers: {host: 'localhost'}
}, {}, function(res) {
assert.notEqual(res.statusCode, 200);
helper.checkNoCache(res);
done();
});
});
test("layergroup can hold substitution tokens", function(done) {
@@ -343,14 +391,19 @@ suite('multilayer', function() {
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = layergroup.layers[0].options.sql
.replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
.replace(/!pixel_width!/g, '1')
.replace(/!pixel_height!/g, '1');
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
.replace(RegExp('!bbox!', 'g'), 'ST_MakeEnvelope(0,0,0,0)')
.replace(RegExp('!pixel_width!', 'g'), '1')
.replace(RegExp('!pixel_height!', 'g'), '1')
+ '$windshaft$)');
+ expectedQuery
+ '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$)::regclass[])');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
next(err);
});
@@ -376,14 +429,19 @@ suite('multilayer', function() {
assert.equal(cc.substring(0, dbname.length), dbname);
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = layergroup.layers[0].options.sql
.replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
.replace('!pixel_width!', '1')
.replace('!pixel_height!', '1');
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ layergroup.layers[0].options.sql
.replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
.replace('!pixel_width!', '1')
.replace('!pixel_height!', '1')
+ '$windshaft$)');
+ expectedQuery
+ '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$)::regclass[])');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', 2,
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
next(err);
});
@@ -1012,7 +1070,7 @@ suite('multilayer', function() {
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png', 2,
assert.imageEqualsFile(res.body, windshaft_fixtures + '/test_default_mapnik_point.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
next(err);
});

View File

@@ -11,6 +11,9 @@ var SQLAPIEmu = require(__dirname + '/../support/SQLAPIEmu.js');
var helper = require(__dirname + '/../support/test_helper');
var IMAGE_EQUALS_TOLERANCE_PER_MIL = 20,
IMAGE_EQUALS_ZERO_TOLERANCE_PER_MIL = 0;
var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var server = new CartodbWindshaft(serverOptions);
@@ -842,7 +845,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
if (err) throw err;
done();
@@ -873,7 +876,7 @@ suite('server', function() {
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body,
'./test/fixtures/test_table_15_16046_12354_styled_black.png',
2, this);
IMAGE_EQUALS_TOLERANCE_PER_MIL, this);
},
function checkImage(err, similarity) {
if (err) throw err;
@@ -889,8 +892,8 @@ suite('server', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/89
test("get'ing a tile with a user-specific database password", function(done){
var style = querystring.stringify({style: test_style_black_200, style_version: '2.0.0'});
var backupDBPass = global.settings.postgres_auth_pass;
global.settings.postgres_auth_pass = '<%= user_password %>';
var backupDBPass = global.environment.postgres_auth_pass;
global.environment.postgres_auth_pass = '<%= user_password %>';
Step (
function() {
var next = this;
@@ -910,14 +913,14 @@ suite('server', function() {
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body,
'./test/fixtures/test_table_15_16046_12354_styled_black.png',
2, this);
IMAGE_EQUALS_TOLERANCE_PER_MIL, this);
},
function checkImage(err, similarity) {
if (err) throw err;
return null
},
function finish(err) {
global.settings.postgres_auth_pass = backupDBPass;
global.environment.postgres_auth_pass = backupDBPass;
done(err);
}
);
@@ -934,7 +937,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
if (err) throw err;
done();
@@ -971,7 +974,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
next(err);
});
@@ -1011,7 +1014,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/blank.png', 0,
assert.imageEqualsFile(res.body, './test/fixtures/blank.png', IMAGE_EQUALS_ZERO_TOLERANCE_PER_MIL,
function(err, similarity) {
if (err) next(err);
else next();
@@ -1031,7 +1034,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/blank.png', 0,
assert.imageEqualsFile(res.body, './test/fixtures/blank.png', IMAGE_EQUALS_ZERO_TOLERANCE_PER_MIL,
function(err, similarity) {
if (err) next(err);
else next();
@@ -1068,7 +1071,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
// NOTE: we expect them to be EQUAL here
if (err) { next(err); return; }
@@ -1105,7 +1108,7 @@ suite('server', function() {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var ct = res.headers['content-type'];
assert.equal(ct, 'image/png');
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', 2,
assert.imageEqualsFile(res.body, './test/fixtures/test_table_15_16046_12354_styled_black.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
// NOTE: we expect them to be different here
if (err) next();

View File

@@ -1593,7 +1593,7 @@ suite('template_api', function() {
if ( err ) throw err;
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkNoCache(res);
helper.checkCache(res);
return null;
},
function finish(err) {

View File

@@ -2,7 +2,7 @@ var http = require('http');
var url = require('url');
var _ = require('underscore');
var o = function(port, cb) {
var SQLAPIEmulator = function(port, cb) {
this.queries = [];
var that = this;
@@ -37,47 +37,45 @@ var o = function(port, cb) {
}).listen(port, cb);
};
o.prototype.handleQuery = function(query, res) {
SQLAPIEmulator.prototype.handleQuery = function(query, res) {
this.queries.push(query);
if ( query.q.match('SQLAPIERROR') ) {
res.statusCode = 400;
res.write(JSON.stringify({'error':'Some error occurred'}));
} else if ( query.q.match('SQLAPINOANSWER') ) {
console.log("SQLAPIEmulator will never respond, on request");
return;
console.log("SQLAPIEmulator will never respond, on request");
return;
} else if (query.q.match('tablenames')) {
var tableNames = JSON.stringify(query);
res.write(queryResult({tablenames: '{' + tableNames + '}', max: 1234567890.123}));
} else if ( query.q.match('EPOCH.* as max') ) {
// This is the structure of the known query sent by tiler
var row = {
'max': 1234567890.123
};
res.write(JSON.stringify({rows: [ row ]}));
res.write(queryResult({max: 1234567890.123}));
} else {
if ( query.q.match('_private_') && query.api_key === undefined) {
res.statusCode = 403;
res.write(JSON.stringify({'error':'forbidden: ' + JSON.stringify(query)}));
} else {
var qs = JSON.stringify(query);
var row = {
// This is the structure of the known query sent by tiler
'cdb_querytables': '{' + qs + '}',
'max': qs
};
var out_obj = {rows: [ row ]};
var out = JSON.stringify(out_obj);
res.write(out);
res.write(queryResult({cdb_querytables: '{' + qs + '}', max: 1234567890.123}));
}
}
res.end();
};
o.prototype.close = function(cb) {
SQLAPIEmulator.prototype.close = function(cb) {
this.sqlapi_server.close(cb);
};
o.prototype.getLastRequest = function() {
SQLAPIEmulator.prototype.getLastRequest = function() {
return this.requests.pop();
};
module.exports = o;
function queryResult(row) {
return JSON.stringify({
rows: [row]
});
}
module.exports = SQLAPIEmulator;

View File

@@ -1,10 +1,11 @@
// Cribbed from the ever prolific Konstantin Kaefer
// https://github.com/mapbox/tilelive-mapnik/blob/master/test/support/assert.js
var fs = require('fs');
var http = require('http');
var path = require('path');
var exec = require('child_process').exec;
var exec = require('child_process').exec,
fs = require('fs'),
http = require('http'),
path = require('path'),
util = require('util');
var assert = module.exports = exports = require('assert');
@@ -66,35 +67,51 @@ assert.utfgridEqualsFile = function(buffer, file_b, tolerance, callback) {
callback(err);
};
//
// @param tol tolerated color distance as a percent over max channel value
// by default this is zero. For meaningful values, see
// http://www.imagemagick.org/script/command-line-options.php#metric
//
assert.imageEqualsFile = function(buffer, file_b, tol, callback) {
/**
* Takes an image data as an input and an image path and compare them using ImageMagick fuzz algorithm, if case the
* similarity is not within the tolerance limit it will callback with an error.
*
* @param buffer The image data to compare from
* @param {string} referenceImageRelativeFilePath The relative file to compare against
* @param {number} tolerance tolerated mean color distance, as a per mil (‰)
* @param {function} callback Will call to home with null in case there is no error, otherwise with the error itself
* @see FUZZY in http://www.imagemagick.org/script/command-line-options.php#metric
*/
assert.imageEqualsFile = function(buffer, referenceImageRelativeFilePath, tolerance, callback) {
if (!callback) callback = function(err) { if (err) throw err; };
file_b = path.resolve(file_b);
var file_a = '/tmp/windshaft-test-image-test.png'; // + (Math.random() * 1e16); // TODO: make predictable
var err = fs.writeFileSync(file_a, buffer, 'binary');
var referenceImageFilePath = path.resolve(referenceImageRelativeFilePath),
testImageFilePath = '/tmp/windshaft-test-image-' + (Math.random() * 1e16); // TODO: make predictable
var err = fs.writeFileSync(testImageFilePath, buffer, 'binary');
if (err) throw err;
var fuzz = tol + '%';
exec('compare -fuzz ' + fuzz + ' -metric AE "' + file_a + '" "' +
file_b + '" /dev/null', function(err, stdout, stderr) {
var imageMagickCmd = util.format(
'compare -metric fuzz "%s" "%s" /dev/null',
testImageFilePath, referenceImageFilePath
);
exec(imageMagickCmd, function(err, stdout, stderr) {
if (err) {
fs.unlinkSync(file_a);
fs.unlinkSync(testImageFilePath);
callback(err);
} else {
stderr = stderr.trim();
var similarity = parseFloat(stderr);
if ( similarity > 0 ) {
var err = new Error('Images not equal(' + similarity + '): ' +
file_a + ' ' + file_b);
err.similarity = similarity;
callback(err);
var metrics = stderr.match(/([0-9]*) \((.*)\)/);
if ( ! metrics ) {
callback(new Error("No match for " + stderr));
return;
}
var similarity = parseFloat(metrics[2]),
tolerancePerMil = (tolerance / 1000);
if (similarity > tolerancePerMil) {
err = new Error(util.format(
'Images %s and %s are not equal (got %d similarity, expected %d)',
testImageFilePath, referenceImageFilePath, similarity, tolerancePerMil)
);
err.similarity = similarity;
callback(err);
} else {
fs.unlinkSync(file_a);
callback(null);
fs.unlinkSync(testImageFilePath);
callback(null);
}
}
});

View File

@@ -87,6 +87,7 @@ if test x"$PREPARE_REDIS" = xyes; then
cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
HMSET rails:users:localhost id ${TESTUSERID} \
database_name '${TEST_DB}' \
database_host localhost \
map_key 1234
SADD rails:users:localhost:map_key 1235
EOF

View File

@@ -10,9 +10,7 @@ var assert = require('assert');
var LZMA = require('lzma/lzma_worker.js').LZMA;
// set environment specific variables
global.settings = require(__dirname + '/../../config/settings');
global.environment = require(__dirname + '/../../config/environments/test');
_.extend(global.settings, global.environment);
process.env.NODE_ENV = 'test';
@@ -39,8 +37,21 @@ function checkNoCache(res) {
}
module.exports = {
lzma_compress_to_base64: lzma_compress_to_base64,
checkNoCache: checkNoCache
/**
* Check that the response headers do not request caching
* @see checkNoCache
* @param res
*/
function checkCache(res) {
assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
assert.ok(res.headers.hasOwnProperty('cache-control'));
assert.ok(res.headers.hasOwnProperty('last-modified'));
}
module.exports = {
lzma_compress_to_base64: lzma_compress_to_base64,
checkNoCache: checkNoCache,
checkCache: checkCache
};

View File

@@ -0,0 +1,11 @@
{"version":"1.0.1",
"layers":[{
"type":"cartodb",
"options":{
"sql":"select 1 as id, ST_Transform(ST_SetSRID(ST_MakePoint(x/1000,x/2000),4326),3857) as the_geom_webmercator FROM generate_series(-170000,170000) x",
"cartocss":"#style{ marker-width: 12;}",
"cartocss_version":"2.1.1",
"Interactivity":"id"
}
}]
}