From 54df010694ea58a0ded5e8881dd779f822b09f6f Mon Sep 17 00:00:00 2001 From: javi santana Date: Wed, 30 Nov 2011 19:57:26 +0100 Subject: [PATCH] removed old cache code --- lib/cartodb/lru.js | 252 ----------------------------- lib/cartodb/tile_cache.js | 178 -------------------- lib/cartodb/ttl.js | 47 ------ test/acceptance/cache.js | 214 ------------------------ test/acceptance/cache_validator.js | 24 --- 5 files changed, 715 deletions(-) delete mode 100644 lib/cartodb/lru.js delete mode 100644 lib/cartodb/tile_cache.js delete mode 100644 lib/cartodb/ttl.js delete mode 100644 test/acceptance/cache.js delete mode 100644 test/acceptance/cache_validator.js diff --git a/lib/cartodb/lru.js b/lib/cartodb/lru.js deleted file mode 100644 index adb8a8f5..00000000 --- a/lib/cartodb/lru.js +++ /dev/null @@ -1,252 +0,0 @@ -/** - * see original file: https://github.com/rsms/js-lru - * - * A doubly linked list-based Least Recently Used (LRU) cache. Will keep most - * recently used items while discarding least recently used items when its limit - * is reached. - * - * Licensed under MIT. Copyright (c) 2010 Rasmus Andersson - * See README.md for details. - * - * Illustration of the design: - * - * entry entry entry entry - * ______ ______ ______ ______ - * | head |.newer => | |.newer => | |.newer => | tail | - * | A | | B | | C | | D | - * |______| <= older.|______| <= older.|______| <= older.|______| - * - * removed <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- <-- added - */ -function LRUCache (limit) { - // Current size of the cache. (Read-only). - this.size = 0; - // Maximum number of items this cache can hold. - this.limit = limit; - this._keymap = {}; -} - -/** - * Put into the cache associated with . Returns the entry which was - * removed to make room for the new entry. Otherwise undefined is returned - * (i.e. if there was enough room already). - */ -LRUCache.prototype.put = function(key, value) { - var entry = {key:key, value:value}; - // Note: No protection agains replacing, and thus orphan entries. By design. - this._keymap[key] = entry; - if (this.tail) { - // link previous tail to the new tail (entry) - this.tail.newer = entry; - entry.older = this.tail; - } else { - // we're first in -- yay - this.head = entry; - } - // add new entry to the end of the linked list -- it's now the freshest entry. - this.tail = entry; - if (this.size === this.limit) { - // we hit the limit -- remove the head - return this.shift(); - } else { - // increase the size counter - this.size++; - } -} - -/** - * Purge the least recently used (oldest) entry from the cache. Returns the - * removed entry or undefined if the cache was empty. - * - * If you need to perform any form of finalization of purged items, this is a - * good place to do it. Simply override/replace this function: - * - * var c = new LRUCache(123); - * c.shift = function() { - * var entry = LRUCache.prototype.shift.call(this); - * doSomethingWith(entry); - * return entry; - * } - */ -LRUCache.prototype.shift = function() { - // todo: handle special case when limit == 1 - var entry = this.head; - if (entry) { - if (this.head.newer) { - this.head = this.head.newer; - this.head.older = undefined; - } else { - this.head = undefined; - } - // Remove last strong reference to and remove links from the purged - // entry being returned: - entry.newer = entry.older = undefined; - // delete is slow, but we need to do this to avoid uncontrollable growth: - delete this._keymap[entry.key]; - } - return entry; -} - -/** - * Get and register recent use of . Returns the value associated with - * or undefined if not in cache. - */ -LRUCache.prototype.get = function(key, returnEntry) { - // First, find our cache entry - var entry = this._keymap[key]; - if (entry === undefined) return; // Not cached. Sorry. - // As was found in the cache, register it as being requested recently - if (entry === this.tail) { - // Already the most recenlty used entry, so no need to update the list - return entry.value; - } - // HEAD--------------TAIL - // <.older .newer> - // <--- add direction -- - // A B C E - if (entry.newer) { - if (entry === this.head) - this.head = entry.newer; - entry.newer.older = entry.older; // C <-- E. - } - if (entry.older) - entry.older.newer = entry.newer; // C. --> E - entry.newer = undefined; // D --x - entry.older = this.tail; // D. --> E - if (this.tail) - this.tail.newer = entry; // E. <-- D - this.tail = entry; - return returnEntry ? entry : entry.value; -} - -// ---------------------------------------------------------------------------- -// Following code is optional and can be removed without breaking the core -// functionality. - -/** - * Check if is in the cache without registering recent use. Feasible if - * you do not want to chage the state of the cache, but only "peek" at it. - * Returns the entry associated with if found, or undefined if not found. - */ -LRUCache.prototype.find = function(key) { - return this._keymap[key]; -} - -/** - * Update the value of entry with . Returns the old value, or undefined if - * entry was not in the cache. - */ -LRUCache.prototype.set = function(key, value) { - var oldvalue, entry = this.get(key, true); - if (entry) { - oldvalue = entry.value; - entry.value = value; - } else { - oldvalue = this.put(key, value); - if (oldvalue) oldvalue = oldvalue.value; - } - return oldvalue; -} - -/** - * Remove entry from cache and return its value. Returns undefined if not - * found. - */ -LRUCache.prototype.remove = function(key) { - var entry = this._keymap[key]; - if (!entry) return; - delete this._keymap[entry.key]; // need to do delete unfortunately - if (entry.newer && entry.older) { - // relink the older entry with the newer entry - entry.older.newer = entry.newer; - entry.newer.older = entry.older; - } else if (entry.newer) { - // remove the link to us - entry.newer.older = undefined; - // link the newer entry to head - this.head = entry.newer; - } else if (entry.older) { - // remove the link to us - entry.older.newer = undefined; - // link the newer entry to head - this.tail = entry.older; - } else { - this.head = this.tail = undefined; - } - this.size--; - return entry.value; -} - -/** Removes all entries */ -LRUCache.prototype.removeAll = function() { - // This should be safe, as we never expose strong refrences to the outside - this.head = this.tail = undefined; - this.size = 0; - this._keymap = {}; -} - -/** - * Return an array containing all keys of entries stored in the cache object, in - * arbitrary order. - */ -if (typeof Object.keys === 'function') { - LRUCache.prototype.keys = function() { return Object.keys(this._keymap); } -} else { - LRUCache.prototype.keys = function() { - var keys = []; - for (var k in this._keymap) keys.push(k); - return keys; - } -} - -/** - * Call `fun` for each entry. Starting with the newest entry if `desc` is a true - * value, otherwise starts with the oldest (head) enrty and moves towards the - * tail. - * - * `fun` is called with 3 arguments in the context `context`: - * `fun.call(context, Object key, Object value, LRUCache self)` - */ -LRUCache.prototype.forEach = function(fun, context, desc) { - if (context === true) { desc = true; context = undefined; } - else if (typeof context !== 'object') context = this; - if (desc) { - var entry = this.tail; - while (entry) { - fun.call(context, entry.key, entry.value, this); - entry = entry.older; - } - } else { - var entry = this.head; - while (entry) { - fun.call(context, entry.key, entry.value, this); - entry = entry.newer; - } - } -} - -/** Returns a JSON (array) representation */ -LRUCache.prototype.toJSON = function() { - var s = [], entry = this.head; - while (entry) { - s.push({key:entry.key.toJSON(), value:entry.value.toJSON()}); - entry = entry.newer; - } - return s; -} - -/** Returns a String representation */ -LRUCache.prototype.toString = function() { - var s = '', entry = this.head; - while (entry) { - s += String(entry.key)+':'+entry.value; - if (entry = entry.newer) - s += ' < '; - } - return s; -} - -// Export ourselves -//if (typeof this === 'object') this.LRUCache = LRUCache; - -module.exports = LRUCache; diff --git a/lib/cartodb/tile_cache.js b/lib/cartodb/tile_cache.js deleted file mode 100644 index 92b1737e..00000000 --- a/lib/cartodb/tile_cache.js +++ /dev/null @@ -1,178 +0,0 @@ -// tile cache policies -// it exports two cache types: -// 'nocache' implements a pass-troguth cache -// 'lru' implements a LRU cache - -var LRUCache = require('./lru'), - CacheValidator = require('./cache_validator'), - TTL = require('./ttl'); - -var BLANK_TILE_SIZE = 334; // totally transparent tile size in bytes - -module.exports.NoCache = function() { - - var me = {} - - me.beforeTileRender = function(req, res, callback) { - callback(null); - } - - me.afterTileRender = function(req, res, tile, headers, callback) { - callback(null, tile, headers); - } - - return me; - -} - -module.exports.LRUcache = function(max_items, redis_opts, ttl_timeout) { - var cache_validator = CacheValidator(redis_opts); - return GenericCache(new LRUCache(max_items), cache_validator, ttl_timeout || 600); -} - -// implements a generic cache for tile -// cache_policy should implement set and get methods and optionally getStats -function GenericCache (cache_policy, cache_validator, ttl_timeout) { - - var me = { - cache: cache_policy, - cache_validator: cache_validator, - cache_hits: 0, - cache_misses: 0, - cache_not_modified: 0, - current_items: 0, - expired: 0, - cache_invalidated: 0, - max_items: 0 - }; - - // enable ttl - var ttl = TTL(function(key) { - me.remove(key); - me.expired++; - }, ttl_timeout || 60); - - function cache_key(req) { - return req.url; - } - - function update_items(n) { - me.current_items = n; - if(n > me.max_items) { - me.max_items = n; - } - } - - me.remove = function(key) { - ttl.remove(key); - me.cache.remove(key); - update_items(me.cache.size || 0); - } - - me.beforeTileRender = function(req, res, callback) { - req.windshaft_start = new Date().getTime(); - var key = cache_key(req); - var tile = me.cache.get(key); - if(tile) { - // validate the cache - me.cache_validator.getTimestamp(req.params.dbname, req.params.table, function(err, t) { - if(t !== null && tile.timestamp < t) { - me.cache_misses++; - me.cache_invalidated++; - callback(null); - } else { - // stats - me.cache_hits++; - var timestamp = new Date().getTime(); - var delta = timestamp - req.windshaft_start; - tile.cache_time = delta/1000.0; - tile.hits++; - res.header('X-Cache-hit', 'true'); - res.header('Last-Modified', new Date(tile.timestamp*1000).toGMTString()); - // check 304 - var modified_since = req.header('if-modified-since'); - if (modified_since) { - modified_since = Date.parse(modified_since); - if(modified_since && modified_since <= timestamp*1000) { - me.cache_not_modified++; - res.statusCode = 304; - res.end(); - return; - } - } - res.send(tile.tile, tile.headers, 200); - } - }); - } else { - me.cache_misses++; - callback(null); - } - } - - me.afterTileRender = function(req, res, tile, headers, callback) { - var timestamp = new Date().getTime(); - var delta = timestamp - req.windshaft_start; - var key = cache_key(req); - me.cache.put(key, { - tile: tile, - headers: headers, - timestamp: timestamp/1000.0, - render_time: delta/1000.0, - hits: 0}); - ttl.start(key); - update_items(me.cache.size || 0); - callback(null, tile, headers); - } - - me.afterStateChange = function(req, data, callback) { - me.cache_validator.setTimestamp(req.params.dbname, req.params.table, new Date().getTime()/1000.0, function(err, t) { - callback(err, data); - }); - } - - me.getStats = function(include_tile_info, sort_by) { - var total = me.cache_hits + me.cache_misses; - var mem = 0; - var blank_tile_count = 0; - var tile_info = [] - me.cache.forEach(function(key, value) { - if(value.tile.length !== undefined) { - mem += value.tile.length; - if(value.tile.length == BLANK_TILE_SIZE) { - blank_tile_count++; - } - } - if(include_tile_info) { - tile_info.push({ - key: key, - length: value.tile.length, - hits: value.hits, - render_time: value.render_time, - cache_time: value.cache_time - }); - } - }); - sort_by = sort_by || 'hits'; - tile_info.sort(function(a, b) { - return b[sort_by] - a[sort_by]; - }); - return { - cache_hits: me.cache_hits, - cache_misses: me.cache_misses, - cache_invalidated: me.cache_invalidated, - cache_not_modified: me.cache_not_modified, - current_items: me.current_items, - expired: me.expired, - max_items: me.max_items, - memory: mem, - memory_per_item: total ? mem/total: 0, - ratio: total ? me.cache_hits/total: 0, - blank_tile_count: blank_tile_count, - blank_tile_size: blank_tile_count*BLANK_TILE_SIZE, - blank_items_ratio: total? blank_tile_count/total: 0, - tiles: tile_info - }; - } - - return me; -} diff --git a/lib/cartodb/ttl.js b/lib/cartodb/ttl.js deleted file mode 100644 index 443911ba..00000000 --- a/lib/cartodb/ttl.js +++ /dev/null @@ -1,47 +0,0 @@ - -/* - ========================================================== - manages timeouts - * timeout in seconds - * on_delete will be called when time expired with the key as first param - - usage: - ttl = TTL(function(key) { - console.log("10 seconds expired on " + key"); - }, 10); - ========================================================== -*/ -function TTL(on_delete, timeout) { - - var me = { - timeouts: {}, - on_delete: on_delete, - timeout: timeout*1000 - }; - - me.start = function(key) { - var t = me.timeouts[key]; - if (t) { - clearTimeout(t); - } - me.timeouts[key] = setTimeout( - (function(k) { - return function() { - me.on_delete(k); - }; - })(key), - me.timeout); - } - - me.remove = function(key) { - var t = me.timeouts[key]; - if (t) { - clearTimeout(t); - delete me.timeouts[key]; - } - } - - return me; -} - -module.exports = TTL; diff --git a/test/acceptance/cache.js b/test/acceptance/cache.js deleted file mode 100644 index c37f7557..00000000 --- a/test/acceptance/cache.js +++ /dev/null @@ -1,214 +0,0 @@ - -var assert = require('assert') - , tests = module.exports = {} - , _ = require('underscore') - , querystring = require('querystring') - , fs = require('fs') - , th = require(__dirname + '/../test_helper') - , CartodbWindshaft = require(__dirname + '/../../lib/cartodb/cartodb_windshaft') - , http = require('http') - , Step = require('step') - , CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator') - -var serverOptions = require(__dirname + '/../../lib/cartodb/server_options'); - -var cached_server = new CartodbWindshaft(serverOptions); - -tests["first time a tile is request should not be cached"] = function() { - assert.response(cached_server, { - url: '/tiles/test_table_3/6/31/24.png?geom_type=polygon', - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - - }, function(res) { - assert.ok(res.header('X-Cache-hit') === undefined); - - }); - -} - -/* -tests["second time a tile is request should be cached"] = function() { - - var cached_server2 = new CartodbWindshaft(serverOptions); - var url= '/tiles/test_table_2/6/31/24.png'; - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.ok(res.header('X-Cache-hit') !== undefined); - }); - }); -} -*/ - - -tests["LRU tile should be removed"] = function() { - - var urls = ['/tiles/test_table_2/6/31/24.png', - '/tiles/test_table_2/6/31/25.png', - '/tiles/test_table_2/6/31/26.png', - '/tiles/test_table_2/6/31/27.png']; - - //create another server to not take previos test stats into account - var so = _.clone(serverOptions); - _(so).extend({lru_cache: true, lru_cache_size: 3}); - - var _cached_server = new CartodbWindshaft(so); - - function makeReq(url, callback) { - assert.response(_cached_server, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, callback); - } - - Step( - function() { - makeReq(urls[0], this); - }, - function() { - makeReq(urls[1], this); - }, - function() { - makeReq(urls[2], this); - }, - function() { - makeReq(urls[3], this); - }, function() { - assert.response(_cached_server, { - url: urls[0], - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - - }, function(res) { - assert.ok(res.header('X-Cache-hit') === undefined); - var st = so.cacheStats() - assert.eql(st.cache_hits, 0); - assert.eql(st.cache_misses, 5); - assert.eql(st.current_items, 3); - }); - } - ) - -} - -tests["cache should be invalidated"] = function() { - - var url = '/tiles/test_table_2/6/29/27.png'; - var cached_server2 = new CartodbWindshaft(serverOptions); - var cache = CacheValidator(global.environment.redis); - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - cache.setTimestamp('cartodb_test_user_1_db', 'test_table_2', (new Date().getTime()/1000.0)+100, function() { - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(r) { - assert.ok(r.header('X-Cache-hit') === undefined); - }); - }); - }); - -}; - -tests["Last-Modified header should be sent"] = function() { - var cached_server2 = new CartodbWindshaft(serverOptions); - var url= '/tiles/test_table/6/31/22.png'; - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.response(cached_server2, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.ok(res.header('X-Cache-hit') !== undefined); - var last_modified = res.header('Last-Modified'); - assert.ok(last_modified !== undefined); - assert.response(cached_server2, { - url: url, - headers: { - host: 'vizzuality.localhost.lan', - 'if-modified-since': last_modified - }, - method: 'GET' - }, { - status: 304 - }); - }); - }); -} - -tests["TTL should invalidate a tile"] = function() { - - var url = '/tiles/test_table_2/6/31/24.png'; - - //create another server to not take previos test stats into account - var so = _.clone(serverOptions); - _(so).extend({ttl_timeout: 1}); - - var _cached_server = new CartodbWindshaft(so); - // cache it - assert.response(_cached_server, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - console.log("WAIT A LITTLE BIT PLEASE"); - - // test before invalidating - setTimeout(function() { - var st = so.cacheStats(); - assert.eql(st.expired, 0); - }, 500); - - // test after invalidation - setTimeout(function() { - assert.response(_cached_server, { - url: url, - headers: {host: 'vizzuality.localhost.lan'}, - method: 'GET' - },{ - status: 200 - }, function(res) { - assert.ok(res.header('X-Cache-hit') === undefined); - var st = so.cacheStats(); - assert.eql(st.expired, 1); - }); - }, 2000); - }); -} diff --git a/test/acceptance/cache_validator.js b/test/acceptance/cache_validator.js deleted file mode 100644 index 55bd714e..00000000 --- a/test/acceptance/cache_validator.js +++ /dev/null @@ -1,24 +0,0 @@ - - -var assert = require('assert'), - tests = module.exports = {}, - th = require(__dirname + '/../test_helper'), - CacheValidator = require(__dirname + '/../../lib/cartodb/cache_validator') - -var cache = CacheValidator(global.environment.redis); - -tests["should get the timestamp it sets"] = function() { - var d = new Date().getTime(); - cache.setTimestamp('mydatabase', 'mytable', d, function() { - cache.getTimestamp('mydatabase', 'mytable', function(err, t) { - assert.eql(t, d); - }) - }); -} - -tests["should get null when timestamp is not set"] = function() { - var d = new Date().getTime(); - cache.getTimestamp('mydatabase', 'mytable2', function(err, t) { - assert.ok(t === null); - }) -}