diff --git a/.gitignore b/.gitignore
new file mode 100644
index 00000000..3c3629e6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+node_modules
diff --git a/app.js b/app.js
index 5a2a7fd5..c8e10ad8 100755
--- a/app.js
+++ b/app.js
@@ -18,7 +18,8 @@ if (ENV != 'development' && ENV != 'production'){
var _ = require('underscore')
, Step = require('step')
- , cartoData = require('./lib/cartodb/carto_data');
+ , cartoData = require('./lib/cartodb/carto_data')
+ , CartodbWindshaft = require('./lib/cartodb/cartodb_windshaft');
@@ -29,47 +30,9 @@ _.extend(global.settings, global.environment);
var Windshaft = require('windshaft');
var serverOptions = require('./lib/cartodb/server_options');
+var Cache = require('./lib/cartodb/tile_cache');
-// boot
-var ws = new Windshaft.Server(serverOptions);
-
-/**
- * Helper to allow access to the layer to be used in the maps infowindow popup.
- */
-ws.get(serverOptions.base_url + '/infowindow', function(req, res){
- Step(
- function(){
- serverOptions.getInfowindow(req, this);
- },
- function(err, data){
- if (err){
- res.send(err.message, 400);
- } else {
- res.send({infowindow: data}, 200);
- }
- }
- );
-});
-
-/**
- * Helper to allow access to metadata to be used in embedded maps.
- */
-ws.get(serverOptions.base_url + '/map_metadata', function(req, res){
- Step(
- function(){
- serverOptions.getMapMetadata(req, this);
- },
- function(err, data){
- if (err){
- res.send(err.message, 400);
- } else {
- res.send({map_metadata: data}, 200);
- }
- }
- );
-});
-
-
+ws = CartodbWindshaft(serverOptions);
ws.listen(global.environment.windshaft_port);
console.log("Windshaft tileserver started on port " + global.environment.windshaft_port);
diff --git a/lib/cartodb/cache_validator.js b/lib/cartodb/cache_validator.js
new file mode 100644
index 00000000..29d53e78
--- /dev/null
+++ b/lib/cartodb/cache_validator.js
@@ -0,0 +1,76 @@
+//
+// this module allows to invalidate cache for a table or check if the cache for a table
+// is valid
+//
+// usage:
+//
+// Cache.getTimestamp('table_name', function(err, timestamp) {
+// });
+//
+// Cache.setTimestamp('table_name', timestamp, function(err, result) {
+// });
+//
+//
+
+var _ = require('underscore'),
+ RedisPool = require('./redis_pool'),
+ Step = require('step')
+
+
+function Cache(redis_opts) {
+
+ var redis_pool = new RedisPool(redis_opts);
+
+ var me = {}
+
+ var redis_options = {
+ db: 0
+ }
+
+ var redisCommand = function(fn, callback) {
+ var redis_client;
+ Step(
+ function getRedisClient(){
+ redis_pool.acquire(redis_options.db, this);
+ },
+ function getDataForTable(err, redis) {
+ if (err) throw err;
+ redis_client = redis;
+ fn(redis_client, this);
+ },
+ function exit(err, data) {
+ if (!_.isUndefined(redis_client))
+ redis_pool.release(redis_options.db, redis_client);
+ if (callback) {
+ callback(err, data);
+ }
+ }
+ );
+ }
+
+ // get the timestamp for the table
+ me.getTimestamp = function(database, table, callback) {
+ redisCommand(function(redis, step) {
+ redis.GET("cache:" + database + ":" + table + ":last_updated_at", step);
+ }, function(err, t) {
+ if(t != null)
+ callback(err, parseFloat(t));
+ else
+ callback(err, t);
+ });
+ }
+
+ me.setTimestamp = function(database, table, timestamp, callback) {
+ timestamp = timestamp || new Date().getTime();
+ redisCommand(function(redis, step) {
+ redis.SET("cache:" + database + ":" + table + ":last_updated_at", timestamp, step);
+ }, callback);
+ }
+
+ return me;
+};
+
+
+module.exports = function(redis_opts) {
+ return new Cache(redis_opts);
+}
diff --git a/lib/cartodb/carto_data.js b/lib/cartodb/carto_data.js
index ec437405..451f6bde 100644
--- a/lib/cartodb/carto_data.js
+++ b/lib/cartodb/carto_data.js
@@ -203,4 +203,4 @@ module.exports = function() {
};
return me;
-}();
\ No newline at end of file
+}();
diff --git a/lib/cartodb/cartodb_windshaft.js b/lib/cartodb/cartodb_windshaft.js
new file mode 100644
index 00000000..c033e717
--- /dev/null
+++ b/lib/cartodb/cartodb_windshaft.js
@@ -0,0 +1,73 @@
+
+var _ = require('underscore')
+ , Step = require('step')
+ , Windshaft = require('windshaft')
+ , Cache = require('./tile_cache');
+
+var CartodbWindshaft = function(serverOptions) {
+
+ // set cache if requested
+ if(serverOptions.lru_cache) {
+ var lru_cache = Cache.LRUcache(serverOptions.lru_cache_size || 10000, serverOptions.redis);
+ _.extend(serverOptions, {
+ beforeTileRender: lru_cache.beforeTileRender,
+ afterTileRender: lru_cache.afterTileRender,
+ cacheStats: lru_cache.getStats,
+ afterStateChange: lru_cache.afterStateChange
+ })
+ }
+
+
+ // boot
+ var ws = new Windshaft.Server(serverOptions);
+ /**
+ * Helper to allow access to the layer to be used in the maps infowindow popup.
+ */
+ ws.get(serverOptions.base_url + '/infowindow', function(req, res){
+ Step(
+ function(){
+ serverOptions.getInfowindow(req, this);
+ },
+ function(err, data){
+ if (err){
+ res.send(err.message, 400);
+ } else {
+ res.send({infowindow: data}, 200);
+ }
+ }
+ );
+ });
+
+ /**
+ * Helper to allow access to metadata to be used in embedded maps.
+ */
+ ws.get(serverOptions.base_url + '/map_metadata', function(req, res){
+ Step(
+ function(){
+ serverOptions.getMapMetadata(req, this);
+ },
+ function(err, data){
+ if (err){
+ res.send(err.message, 400);
+ } else {
+ res.send({map_metadata: data}, 200);
+ }
+ }
+ );
+ });
+
+ /**
+ * tile cache stats
+ */
+ ws.get('/cache', function(req, res){
+ if(serverOptions.cacheStats) {
+ res.send(serverOptions.cacheStats());
+ } else {
+ res.send("Cache no enabled")
+ }
+ });
+
+ return ws;
+}
+
+module.exports = CartodbWindshaft;
diff --git a/lib/cartodb/lru.js b/lib/cartodb/lru.js
new file mode 100644
index 00000000..5539adc6
--- /dev/null
+++ b/lib/cartodb/lru.js
@@ -0,0 +1,249 @@
+/**
+ * see original file: https://github.com/rsms/js-lru
+ *
+ * A doubly linked list-based Least Recently Used (LRU) cache. Will keep most
+ * recently used items while discarding least recently used items when its limit
+ * is reached.
+ *
+ * Licensed under MIT. Copyright (c) 2010 Rasmus Andersson
+ * See README.md for details.
+ *
+ * Illustration of the design:
+ *
+ * entry entry entry entry
+ * ______ ______ ______ ______
+ * | head |.newer => | |.newer => | |.newer => | tail |
+ * | A | | B | | C | | D |
+ * |______| <= older.|______| <= older.|______| <= older.|______|
+ *
+ * removed <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- added
+ */
+function LRUCache (limit) {
+ // Current size of the cache. (Read-only).
+ this.size = 0;
+ // Maximum number of items this cache can hold.
+ this.limit = limit;
+ this._keymap = {};
+}
+
+/**
+ * Put into the cache associated with . Returns the entry which was
+ * removed to make room for the new entry. Otherwise undefined is returned
+ * (i.e. if there was enough room already).
+ */
+LRUCache.prototype.put = function(key, value) {
+ var entry = {key:key, value:value};
+ // Note: No protection agains replacing, and thus orphan entries. By design.
+ this._keymap[key] = entry;
+ if (this.tail) {
+ // link previous tail to the new tail (entry)
+ this.tail.newer = entry;
+ entry.older = this.tail;
+ } else {
+ // we're first in -- yay
+ this.head = entry;
+ }
+ // add new entry to the end of the linked list -- it's now the freshest entry.
+ this.tail = entry;
+ if (this.size === this.limit) {
+ // we hit the limit -- remove the head
+ return this.shift();
+ } else {
+ // increase the size counter
+ this.size++;
+ }
+}
+
+/**
+ * Purge the least recently used (oldest) entry from the cache. Returns the
+ * removed entry or undefined if the cache was empty.
+ *
+ * If you need to perform any form of finalization of purged items, this is a
+ * good place to do it. Simply override/replace this function:
+ *
+ * var c = new LRUCache(123);
+ * c.shift = function() {
+ * var entry = LRUCache.prototype.shift.call(this);
+ * doSomethingWith(entry);
+ * return entry;
+ * }
+ */
+LRUCache.prototype.shift = function() {
+ // todo: handle special case when limit == 1
+ var entry = this.head;
+ if (entry) {
+ if (this.head.newer) {
+ this.head = this.head.newer;
+ this.head.older = undefined;
+ } else {
+ this.head = undefined;
+ }
+ // Remove last strong reference to and remove links from the purged
+ // entry being returned:
+ entry.newer = entry.older = undefined;
+ // delete is slow, but we need to do this to avoid uncontrollable growth:
+ delete this._keymap[entry.key];
+ }
+ return entry;
+}
+
+/**
+ * Get and register recent use of . Returns the value associated with
+ * or undefined if not in cache.
+ */
+LRUCache.prototype.get = function(key, returnEntry) {
+ // First, find our cache entry
+ var entry = this._keymap[key];
+ if (entry === undefined) return; // Not cached. Sorry.
+ // As was found in the cache, register it as being requested recently
+ if (entry === this.tail) {
+ // Already the most recenlty used entry, so no need to update the list
+ return entry.value;
+ }
+ // HEAD--------------TAIL
+ // <.older .newer>
+ // <--- add direction --
+ // A B C E
+ if (entry.newer) {
+ if (entry === this.head)
+ this.head = entry.newer;
+ entry.newer.older = entry.older; // C <-- E.
+ }
+ if (entry.older)
+ entry.older.newer = entry.newer; // C. --> E
+ entry.newer = undefined; // D --x
+ entry.older = this.tail; // D. --> E
+ if (this.tail)
+ this.tail.newer = entry; // E. <-- D
+ this.tail = entry;
+ return returnEntry ? entry : entry.value;
+}
+
+// ----------------------------------------------------------------------------
+// Following code is optional and can be removed without breaking the core
+// functionality.
+
+/**
+ * Check if is in the cache without registering recent use. Feasible if
+ * you do not want to chage the state of the cache, but only "peek" at it.
+ * Returns the entry associated with if found, or undefined if not found.
+ */
+LRUCache.prototype.find = function(key) {
+ return this._keymap[key];
+}
+
+/**
+ * Update the value of entry with . Returns the old value, or undefined if
+ * entry was not in the cache.
+ */
+LRUCache.prototype.set = function(key, value) {
+ var oldvalue, entry = this.get(key, true);
+ if (entry) {
+ oldvalue = entry.value;
+ entry.value = value;
+ } else {
+ oldvalue = this.put(key, value);
+ if (oldvalue) oldvalue = oldvalue.value;
+ }
+ return oldvalue;
+}
+
+/**
+ * Remove entry from cache and return its value. Returns undefined if not
+ * found.
+ */
+LRUCache.prototype.remove = function(key) {
+ var entry = this._keymap[key];
+ if (!entry) return;
+ delete this._keymap[entry.key]; // need to do delete unfortunately
+ if (entry.newer && entry.older) {
+ // relink the older entry with the newer entry
+ entry.older.newer = entry.newer;
+ entry.newer.older = entry.older;
+ } else if (entry.newer) {
+ // remove the link to us
+ entry.newer.older = undefined;
+ // link the newer entry to head
+ this.head = entry.newer;
+ } else if (entry.older) {
+ // remove the link to us
+ entry.older.newer = undefined;
+ // link the newer entry to head
+ this.tail = entry.older;
+ }
+ return entry.value;
+}
+
+/** Removes all entries */
+LRUCache.prototype.removeAll = function() {
+ // This should be safe, as we never expose strong refrences to the outside
+ this.head = this.tail = undefined;
+ this.size = 0;
+ this._keymap = {};
+}
+
+/**
+ * Return an array containing all keys of entries stored in the cache object, in
+ * arbitrary order.
+ */
+if (typeof Object.keys === 'function') {
+ LRUCache.prototype.keys = function() { return Object.keys(this._keymap); }
+} else {
+ LRUCache.prototype.keys = function() {
+ var keys = [];
+ for (var k in this._keymap) keys.push(k);
+ return keys;
+ }
+}
+
+/**
+ * Call `fun` for each entry. Starting with the newest entry if `desc` is a true
+ * value, otherwise starts with the oldest (head) enrty and moves towards the
+ * tail.
+ *
+ * `fun` is called with 3 arguments in the context `context`:
+ * `fun.call(context, Object key, Object value, LRUCache self)`
+ */
+LRUCache.prototype.forEach = function(fun, context, desc) {
+ if (context === true) { desc = true; context = undefined; }
+ else if (typeof context !== 'object') context = this;
+ if (desc) {
+ var entry = this.tail;
+ while (entry) {
+ fun.call(context, entry.key, entry.value, this);
+ entry = entry.older;
+ }
+ } else {
+ var entry = this.head;
+ while (entry) {
+ fun.call(context, entry.key, entry.value, this);
+ entry = entry.newer;
+ }
+ }
+}
+
+/** Returns a JSON (array) representation */
+LRUCache.prototype.toJSON = function() {
+ var s = [], entry = this.head;
+ while (entry) {
+ s.push({key:entry.key.toJSON(), value:entry.value.toJSON()});
+ entry = entry.newer;
+ }
+ return s;
+}
+
+/** Returns a String representation */
+LRUCache.prototype.toString = function() {
+ var s = '', entry = this.head;
+ while (entry) {
+ s += String(entry.key)+':'+entry.value;
+ if (entry = entry.newer)
+ s += ' < ';
+ }
+ return s;
+}
+
+// Export ourselves
+//if (typeof this === 'object') this.LRUCache = LRUCache;
+
+module.exports = LRUCache;
\ No newline at end of file
diff --git a/lib/cartodb/tile_cache.js b/lib/cartodb/tile_cache.js
new file mode 100644
index 00000000..60711528
--- /dev/null
+++ b/lib/cartodb/tile_cache.js
@@ -0,0 +1,111 @@
+// tile cache policies
+// it exports two cache types:
+// 'nocache' implements a pass-troguth cache
+// 'lru' implements a LRU cache
+
+var LRUCache = require('./lru'),
+ CacheValidator = require('./cache_validator')
+
+module.exports.NoCache = function() {
+
+ var me = {}
+
+ me.beforeTileRender = function(req, res, callback) {
+ callback(null);
+ }
+
+ me.afterTileRender = function(req, res, tile, headers, callback) {
+ callback(null, tile, headers);
+ }
+
+ return me;
+
+}
+
+module.exports.LRUcache = function(max_items, redis_opts) {
+ var cache_validator = CacheValidator(redis_opts);
+ return GenericCache(new LRUCache(max_items), cache_validator);
+}
+
+// implements a generic cache for tile
+// cache_policy should implement set and get methods and optionally getStats
+function GenericCache (cache_policy, cache_validator) {
+
+ var me = {
+ cache: cache_policy,
+ cache_validator: cache_validator,
+ cache_hits: 0,
+ cache_misses: 0,
+ current_items: 0,
+ cache_invalidated: 0,
+ max_items: 0
+ }
+
+ function cache_key(req) {
+ return req.url;
+ }
+
+ function update_items(n) {
+ me.current_items = n;
+ if(n > me.max_items) {
+ me.max_items = n;
+ }
+ }
+
+ me.beforeTileRender = function(req, res, callback) {
+ var key = cache_key(req);
+ var tile = me.cache.get(key);
+ if(tile) {
+ // validate the cache
+ me.cache_validator.getTimestamp(req.params.dbname, req.params.table, function(err, t) {
+ if(t != null && tile.timestamp < t) {
+ me.cache_misses++;
+ me.cache_invalidated++;
+ callback(null);
+ } else {
+ me.cache_hits++;
+ res.header('X-Cache-hit', 'true');
+ res.send(tile.tile, tile.headers, 200);
+ }
+ });
+ } else {
+ me.cache_misses++;
+ callback(null);
+ }
+ }
+
+ me.afterTileRender = function(req, res, tile, headers, callback) {
+ var timestamp = new Date().getTime()/1000.0;
+ me.cache.put(cache_key(req), { tile: tile, headers: headers, timestamp: timestamp});
+ update_items(me.cache.size || 0);
+ callback(null, tile, headers);
+ }
+
+ me.afterStateChange = function(req, data, callback) {
+ console.log("invalidating", req.params.dbname,", ", req.params.table);
+ me.cache_validator.setTimestamp(req.params.dbname, req.params.table, new Date().getTime()/1000.0, function(err, t) {
+ callback(err, data);
+ });
+ }
+
+ me.getStats = function() {
+ var total = me.cache_hits + me.cache_misses;
+ var mem = 0;
+ me.cache.forEach(function(key, value) {
+ if(value.tile.length !== undefined) {
+ mem += value.tile.length;
+ }
+ });
+ return {
+ cache_hits: me.cache_hits,
+ cache_misses: me.cache_misses,
+ current_items: me.current_items,
+ max_items: me.max_items,
+ memory: mem,
+ memory_per_item: total ? mem/total: 0,
+ ratio: total ? me.cache_hits/total: 0
+ };
+ }
+
+ return me;
+}
diff --git a/test/acceptance/cache.js b/test/acceptance/cache.js
new file mode 100644
index 00000000..4fac1f48
--- /dev/null
+++ b/test/acceptance/cache.js
@@ -0,0 +1,138 @@
+
+var assert = require('assert')
+ , tests = module.exports = {}
+ , _ = require('underscore')
+ , querystring = require('querystring')
+ , fs = require('fs')
+ , th = require(__dirname + '/../test_helper')
+ , CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft')
+ , http = require('http')
+ , Step = require('step')
+ , CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
+
+var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
+
+var cached_server = new CartodbWindshaft(serverOptions);
+
+tests["first time a tile is request should not be cached"] = function() {
+ assert.response(cached_server, {
+ url: '/tiles/gadm4/6/31/24.png?geom_type=polygon',
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+
+ }, function(res) {
+ assert.ok(res.header('X-Cache-hit') === undefined);
+
+ });
+
+}
+
+/*
+tests["second time a tile is request should be cached"] = function() {
+
+ var cached_server2 = new CartodbWindshaft(serverOptions);
+ var url= '/tiles/gadm4/6/31/24.png';
+ assert.response(cached_server2, {
+ url: url,
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+ }, function(res) {
+ assert.response(cached_server2, {
+ url: url,
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+ }, function(res) {
+ assert.ok(res.header('X-Cache-hit') !== undefined);
+ });
+ });
+}
+*/
+
+
+tests["LRU tile should be removed"] = function() {
+
+ var urls = ['/tiles/gadm4/6/31/24.png',
+ '/tiles/gadm4/6/31/25.png',
+ '/tiles/gadm4/6/31/26.png',
+ '/tiles/gadm4/6/31/27.png'];
+
+ //create another server to not take previos test stats into account
+ var so = _.clone(serverOptions);
+ _(so).extend({lru_cache: true, lru_cache_size: 3});
+
+ var _cached_server = new CartodbWindshaft(so);
+
+ function makeReq(url, callback) {
+ assert.response(_cached_server, {
+ url: url,
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+ }, callback);
+ }
+
+ Step(
+ function() {
+ makeReq(urls[0], this);
+ },
+ function() {
+ makeReq(urls[1], this);
+ },
+ function() {
+ makeReq(urls[2], this);
+ },
+ function() {
+ makeReq(urls[3], this);
+ }, function() {
+ assert.response(_cached_server, {
+ url: urls[0],
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+
+ }, function(res) {
+ assert.ok(res.header('X-Cache-hit') === undefined);
+ var st = so.cacheStats()
+ assert.eql(st.cache_hits, 0);
+ assert.eql(st.cache_misses, 5);
+ assert.eql(st.current_items, 3);
+ });
+ }
+ )
+
+}
+
+tests["cache should be invalidated"] = function() {
+
+ var url = '/tiles/gadm4/6/29/27.png';
+ var cache = CacheValidator(global.environment.redis);
+ assert.response(cached_server, {
+ url: url,
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+ }, function(res) {
+ cache.setTimestamp('cartodb_dev_user_3_db', 'gadm4', (new Date().getTime()/1000.0)+100, function() {
+ assert.response(cached_server, {
+ url: url,
+ headers: {host: 'vizzuality.localhost.lan'},
+ method: 'GET'
+ },{
+ status: 200
+ }, function(res) {
+ assert.ok(res.header('X-Cache-hit') === undefined);
+ });
+ });
+ });
+
+}
+
diff --git a/test/acceptance/cache_validator.js b/test/acceptance/cache_validator.js
new file mode 100644
index 00000000..55bd714e
--- /dev/null
+++ b/test/acceptance/cache_validator.js
@@ -0,0 +1,24 @@
+
+
+var assert = require('assert'),
+ tests = module.exports = {},
+ th = require(__dirname + '/../test_helper'),
+ CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator')
+
+var cache = CacheValidator(global.environment.redis);
+
+tests["should get the timestamp it sets"] = function() {
+ var d = new Date().getTime();
+ cache.setTimestamp('mydatabase', 'mytable', d, function() {
+ cache.getTimestamp('mydatabase', 'mytable', function(err, t) {
+ assert.eql(t, d);
+ })
+ });
+}
+
+tests["should get null when timestamp is not set"] = function() {
+ var d = new Date().getTime();
+ cache.getTimestamp('mydatabase', 'mytable2', function(err, t) {
+ assert.ok(t === null);
+ })
+}
diff --git a/test/acceptance/server.js b/test/acceptance/server.js
index bda87bc4..c33f6486 100644
--- a/test/acceptance/server.js
+++ b/test/acceptance/server.js
@@ -4,9 +4,9 @@ var _ = require('underscore');
var querystring = require('querystring');
require(__dirname + '/../test_helper');
-var Windshaft = require(__dirname + '/../../lib/windshaft');
+var CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft');
var serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
-var server = new Windshaft.Server(serverOptions);
+var server = new CartodbWindshaft(serverOptions);
tests['true'] = function() {
assert.ok(true);
@@ -41,7 +41,7 @@ tests["post'ing no style returns 400 with errors"] = function(){
method: 'POST'
},{
status: 400,
- body: '{"error":"must sent style information"}'
+ body: '{"error":"must send style information"}'
});
};
@@ -149,8 +149,6 @@ tests["get'ing completed infowindow with callback returns information with callb
});
};
-
-
tests["get'ing a tile with default style should return an image"] = function(){
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
@@ -162,6 +160,7 @@ tests["get'ing a tile with default style should return an image"] = function(){
});
};
+
tests["get'ing a json with default style should return an grid"] = function(){
assert.response(server, {
headers: {host: 'vizzuality.localhost.lan'},
@@ -211,4 +210,4 @@ tests["get'ing a tile with default style and complex sql should return a constra
status: 200,
headers: { 'Content-Type': 'image/png' }
});
-};
\ No newline at end of file
+};