first commit

test/README.md

@@ -0,0 +1,21 @@
+cartodb-sql-api tests
+---------------------
+Tests require you create a test database and set some redis keys before,
+you can execute prepare_db.sh script, it will create database, users
+and redis stuff for you. Be sure postgres and redis are running.
+
+> cd test && ./prepare_db.sh
+
+Note that "make check" from top-level dir will try to do everything
+needed to prepare & run the tests.
+
+
+Acceptance tests (need ctrl-C to exit)
+--------------------------------------
+> mocha -u tdd test/acceptance/app.test.js
+> mocha -u tdd test/acceptance/app.auth.test.js
+
+
+Unit tests 
+--------------------------------
+> mocha -u tdd test/unit/*.js (or run the tests individually)

test/acceptance/app-configuration.js

@@ -0,0 +1,169 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+
+
+describe('app-configuration', function() {
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    var expected_cache_control = 'no-cache,max-age=31536000,must-revalidate,public';
+    var expected_cache_control_persist = 'public,max-age=31536000';
+
+    it('GET /api/v1/version', function(done){
+        assert.response(server, {
+            url: '/api/v1/version',
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var parsed = JSON.parse(res.body);
+            var sqlapi_version = require(__dirname + '/../../package.json').version;
+            assert.ok(parsed.hasOwnProperty('cartodb_sql_api'), "No 'cartodb_sql_api' version in " + parsed);
+            assert.equal(parsed.cartodb_sql_api, sqlapi_version);
+            done();
+        });
+    });
+
+    it('GET /api/v1/sql', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{
+            status: 400
+        }, function(err, res) {
+            assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+            assert.deepEqual(res.headers['content-disposition'], 'inline');
+            assert.deepEqual(JSON.parse(res.body), {"error":["You must indicate a sql query"]});
+            done();
+        });
+    });
+
+    // Test base_url setting
+    it('GET /api/whatever/sql', function(done){
+        assert.response(server, {
+            url: '/api/whatever/sql?q=SELECT%201',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, done);
+    });
+
+    // Test CORS headers with GET
+    it('GET /api/whatever/sql', function(done){
+        assert.response(server, {
+            url: '/api/whatever/sql?q=SELECT%201',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            assert.equal(
+                res.headers['access-control-allow-headers'],
+                'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization'
+            );
+            assert.equal(res.headers['access-control-allow-origin'], '*');
+            done();
+        });
+    });
+
+    // Test that OPTIONS does not run queries
+    it('OPTIONS /api/x/sql', function(done){
+        assert.response(server, {
+            url: '/api/x/sql?q=syntax%20error',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'OPTIONS'
+        }, RESPONSE_OK, function(err, res) {
+            assert.equal(res.body, '');
+            assert.equal(
+                res.headers['access-control-allow-headers'],
+                'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization'
+            );
+            assert.equal(res.headers['access-control-allow-origin'], '*');
+            done();
+        });
+    });
+
+
+    it('cache_policy=persist', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=' +
+                'SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db&cache_policy=persist',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            // Check cache headers
+            assert.ok(res.headers.hasOwnProperty('x-cache-channel'));
+            // See https://github.com/CartoDB/CartoDB-SQL-API/issues/105
+            assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4');
+            assert.equal(res.headers['cache-control'], expected_cache_control_persist);
+            done();
+        });
+    });
+
+    // See https://github.com/CartoDB/CartoDB-SQL-API/issues/121
+    it('SELECT from user-specific database', function(done){
+        var backupDBHost = global.settings.db_host;
+        global.settings.db_host = '6.6.6.6';
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT+2+as+n',
+            headers: {host: 'cartodb250user.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            global.settings.db_host = backupDBHost;
+            try {
+                var parsed = JSON.parse(res.body);
+                assert.equal(parsed.rows.length, 1);
+                assert.equal(parsed.rows[0].n, 2);
+            } catch (e) {
+                return done(e);
+            }
+            done();
+        });
+    });
+
+    // See https://github.com/CartoDB/CartoDB-SQL-API/issues/120
+    it('SELECT with user-specific password', function(done){
+        var backupDBUserPass = global.settings.db_user_pass;
+        global.settings.db_user_pass = '<%= user_password %>';
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT+2+as+n&api_key=1234',
+            headers: {host: 'cartodb250user.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            global.settings.db_user_pass = backupDBUserPass;
+            try {
+            assert.equal(res.statusCode, 200, res.statusCode + ": " + res.body);
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.rows.length, 1);
+            assert.equal(parsed.rows[0].n, 2);
+            } catch (e) {
+            return done(e);
+            }
+            return done();
+        });
+    });
+
+    /**
+     * CORS
+     */
+    it('GET /api/v1/sql with SQL parameter on SELECT only should return CORS headers ', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            // Check cache headers
+            assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4');
+            assert.equal(res.headers['cache-control'], expected_cache_control);
+            assert.equal(res.headers['access-control-allow-origin'], '*');
+            assert.equal(
+                res.headers['access-control-allow-headers'],
+                "X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization"
+            );
+            done();
+        });
+    });
+
+});

test/acceptance/app.auth.test.js

@@ -0,0 +1,67 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+
+describe('app.auth', function() {
+
+    var scenarios = [
+        {
+            desc: 'no api key should fallback to default api key',
+            url: "/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4",
+            statusCode: 200
+        },
+        {
+            desc: 'invalid api key should return 401',
+            url: "/api/v1/sql?api_key=THIS_API_KEY_NOT_EXIST&q=SELECT%20*%20FROM%20untitle_table_4",
+            statusCode: 401
+        },
+        {
+            desc: 'valid api key should allow insert in protected tables',
+            url: "/api/v1/sql?api_key=1234&q=INSERT%20INTO%20private_table%20(name)%20VALUES%20('app_auth_test1')",
+            statusCode: 200
+        },
+        {
+            desc: 'valid api key should allow delete in protected tables',
+            url: "/api/v1/sql?api_key=1234&q=DELETE%20FROM%20private_table%20WHERE%20name%3d'app_auth_test1'",
+            statusCode: 200
+        },
+        {
+            desc: 'invalid api key should NOT allow insert in protected tables',
+            url: "/api/v1/sql?api_key=THIS_API_KEY_NOT_EXIST&q=INSERT%20INTO%20private_table%20(name)%20VALUES%20('R')",
+            statusCode: 401
+        },
+        {
+            desc: 'no api key should NOT allow insert in protected tables',
+            url: "/api/v1/sql?q=INSERT%20INTO%20private_table%20(name)%20VALUES%20('RAMBO')",
+            statusCode: 403
+        },
+        {
+            desc: 'no api key should NOT allow insert in public tables',
+            url: "/api/v1/sql?q=INSERT%20INTO%20untitle_table_4%20(name)%20VALUES%20('RAMBO')",
+            statusCode: 403
+        }
+    ];
+
+    scenarios.forEach(function(scenario) {
+        it(scenario.desc, function(done) {
+            assert.response(server, {
+                    // view prepare_db.sh to find public table name and structure
+                    url: scenario.url,
+                    headers: {
+                        host: 'vizzuality.cartodb.com'
+                    },
+                    method: 'GET'
+                },
+                {},
+                function(err, res) {
+                    assert.equal(res.statusCode, scenario.statusCode, res.statusCode + ': ' + res.body);
+                    done();
+                }
+            );
+        });
+    });
+
+});

test/acceptance/app.test.js

@@ -0,0 +1,885 @@
+'use strict';
+
+/**
+ *
+ * Requires the database and tables setup in config/environments/test.js to exist
+ * Ensure the user is present in the pgbouncer auth file too
+ * TODO: Add OAuth tests.
+ *
+ * To run this test, ensure that cartodb_test_user_1_db metadata exists
+ * in Redis for the vizzuality.cartodb.com domain
+ *
+ * SELECT 5
+ * HSET rails:users:vizzuality id 1
+ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db
+ *
+ */
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var _ = require('underscore');
+var step = require('step');
+
+
+describe('app.test', function() {
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+var expected_cache_control = 'no-cache,max-age=31536000,must-revalidate,public';
+var expected_rw_cache_control = 'no-cache,max-age=0,must-revalidate,public';
+
+it('GET /api/v1/sql with SQL parameter on SELECT only. No oAuth included ', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&database=cartodb_test_user_1_db',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        // Check cache headers
+        assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4');
+        assert.equal(res.headers['cache-control'], expected_cache_control);
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        done();
+    });
+});
+
+it('GET /user/vizzuality/api/v1/sql with SQL parameter on SELECT only', function(done){
+    assert.response(server, {
+        url: '/user/vizzuality/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers. Authenticated.',
+function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20cartodb_id*2%20FROM%20untitle_table_4&api_key=1234',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        // Check cache headers
+        assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4');
+        assert.equal(res.headers['cache-control'], expected_cache_control);
+        done();
+    });
+});
+
+it('POST /api/v1/sql with SQL parameter on SELECT only. no database param, just id using headers', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({q: "SELECT * FROM untitle_table_4"}),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        done();
+    });
+});
+
+it('GET /api/v1/sql with INSERT. oAuth not used, so public user - should fail', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?q=INSERT%20INTO%20untitle_table_4%20(cartodb_id)%20VALUES%20(1e4)" +
+            "&database=cartodb_test_user_1_db",
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{
+    }, function(err, res) {
+        assert.equal(res.statusCode, 403, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.ok(JSON.parse(res.body).error[0].match(/permission denied for .+? untitle_table_4/));
+        done();
+    });
+});
+
+it('GET /api/v1/sql with DROP TABLE. oAuth not used, so public user - should fail', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?q=DROP%20TABLE%20untitle_table_4&database=cartodb_test_user_1_db",
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{
+    }, function(err, res) {
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.ok(JSON.parse(res.body).error[0].match(/must be owner of.+? untitle_table_4/));
+        done();
+    });
+});
+
+it('GET /api/v1/sql with INSERT. header based db - should fail', function (done) {
+    assert.response(server, {
+        url: "/api/v1/sql?q=INSERT%20INTO%20untitle_table_4%20(id)%20VALUES%20(1)",
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    }, {
+        status: 400
+    }, done);
+});
+
+it('GET /api/v1/sql with SQL parameter on DROP TABLE. should fail', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?q=DROP%20TABLE%20untitle_table_4",
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.ok(JSON.parse(res.body).error[0].match(/must be owner of.+? untitle_table_4/));
+        done();
+    });
+});
+
+// Check X-Cache-Channel when querying "updated_at" fields
+//
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/99
+it('Field name is not confused with UPDATE operation', function(done){
+    assert.response(server, {
+        // view prepare_db.sh to see where to set api_key
+        url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+          "SELECT min(updated_at) FROM private_table"
+        }),
+        headers: {host: 'vizzuality.localhost.lan:8080' },
+        method: 'GET'
+    },{}, function(err, res) {
+        assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+        assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.private_table');
+        done();
+    });
+});
+
+it('CREATE TABLE with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'CREATE TABLE test_table(a int)',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      // Check cache headers
+      // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      done();
+    });
+});
+
+it('ALTER TABLE with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'ALTER TABLE test_table ADD b int',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      // Check cache headers
+      // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      done();
+    });
+});
+
+it('multistatement insert, alter, select, begin, commit', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'BEGIN; DELETE FROM test_table; COMMIT; BEGIN; INSERT INTO test_table(b) values (5); COMMIT; ' +
+              'ALTER TABLE test_table ALTER b TYPE float USING b::float/2; SELECT b FROM test_table;',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      var parsedBody = JSON.parse(res.body);
+      assert.equal(parsedBody.total_rows, 1);
+      assert.deepEqual(parsedBody.rows[0], {b:2.5});
+      done();
+    });
+});
+
+it('TRUNCATE TABLE with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'TRUNCATE TABLE test_table',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      var pbody = JSON.parse(res.body);
+      assert.equal(pbody.rows.length, 0);
+      assert.response(server, {
+          url: "/api/v1/sql?" + querystring.stringify({
+            q: 'SELECT count(*) FROM test_table',
+            api_key: 1234
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      },{}, function(err, res) {
+        assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+        // table should not get a cache channel as it won't get invalidated
+        assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
+        assert.equal(res.headers['cache-control'], expected_cache_control);
+        var pbody = JSON.parse(res.body);
+        assert.equal(pbody.total_rows, 1);
+        assert.equal(pbody.rows[0].count, 0);
+        done();
+      });
+    });
+});
+
+it('REINDEX TABLE with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: ' ReINdEX TABLE test_table',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      var pbody = JSON.parse(res.body);
+      assert.equal(pbody.rows.length, 0);
+      done();
+    });
+});
+
+it('DROP TABLE with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'DROP TABLE test_table',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      // Check cache headers
+      // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      done();
+    });
+});
+
+it('CREATE FUNCTION with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'CREATE FUNCTION create_func_test(a int) RETURNS INT AS \'SELECT 1\' LANGUAGE \'sql\'',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      // Check cache headers
+      // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      done();
+    });
+});
+
+it('DROP FUNCTION with GET and auth', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({
+          q: 'DROP FUNCTION create_func_test(a int)',
+          api_key: 1234
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+      assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+      // Check cache headers
+      // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+      assert.ok(!res.hasOwnProperty('x-cache-channel'));
+      assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+      done();
+    });
+});
+
+it('sends a 400 when an unsupported format is requested', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=unknown',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 400, res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.deepEqual(JSON.parse(res.body), {"error":[ "Invalid format: unknown" ]});
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var ct = res.headers['content-type'];
+        assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^inline/.test(cd), 'Default format is not disposed inline: ' + cd);
+        assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
+        done();
+    });
+});
+
+it('POST /api/v1/sql with SQL parameter and no format, ensuring content-disposition set to json', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({q: "SELECT * FROM untitle_table_4" }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var ct = res.headers['content-type'];
+        assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^inline/.test(cd), 'Default format is not disposed inline: ' + cd);
+        assert.equal(true, /filename=cartodb-query.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SQL parameter and no format, but a filename', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&filename=x',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var ct = res.headers['content-type'];
+        assert.ok(/json/.test(ct), 'Default format is not JSON: ' + ct);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'Format with filename is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=x.json/gi.test(cd), 'Unexpected JSON filename: ' + cd);
+        done();
+    });
+});
+
+it('GET /api/v1/sql ensure cross domain set on errors', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*gadfgadfg%20FROM%20untitle_table_4',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{
+        status: 400
+    }, function(err, res){
+        var cd = res.headers['access-control-allow-origin'];
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.equal(cd, '*');
+        done();
+    });
+});
+
+it('GET decent error if domain is incorrect', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+        headers: {host: 'vizzualinot.cartodb.com'},
+        method: 'GET'
+    }, {}, function(err, res){
+        assert.equal(res.statusCode, 404, res.statusCode + ( res.statusCode !== 200 ? ( ': ' + res.body ) : ''));
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        var result = JSON.parse(res.body);
+        assert.equal(
+            result.error[0],
+            "Sorry, we can't find CARTO user 'vizzualinot'. Please check that you have entered the correct domain."
+        );
+        done();
+    });
+});
+
+// this test does not make sense with the current CDB_QueryTables implementation
+it('GET decent error if SQL is broken', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({q:
+          'SELECT star FROM this and that'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res){
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        var result = JSON.parse(res.body);
+        // NOTE: actual error message may be slighly different, possibly worth a regexp here
+        assert.equal(result.error[0], 'syntax error at or near "and"');
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/88
+it('numeric arrays are rendered as such', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?" + querystring.stringify({q:
+          "SELECT ARRAY[8.7,4.3]::numeric[] as x"
+        }),
+        headers: {host: 'vizzuality.localhost.lan:8080' },
+        method: 'GET'
+    },{}, function(err, res) {
+        assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+        var out = JSON.parse(res.body);
+        assert.ok(out.hasOwnProperty('time'));
+        assert.equal(out.total_rows, 1);
+        assert.equal(out.rows.length, 1);
+        assert.ok(out.rows[0].hasOwnProperty('x'));
+        assert.equal(out.rows[0].x.length, 2);
+        assert.equal(out.rows[0].x[0], '8.7');
+        assert.equal(out.rows[0].x[1], '4.3');
+        assert.equal(res.headers.hasOwnProperty('x-cache-channel'), false);
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/97
+it('field names and types are exposed', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: "SELECT 1::int as a, 2::float8 as b, 3::varchar as c, " +
+             "4::char as d, now() as e, 'a'::text as f" +
+             ", 1::bool as g" +
+             ", 'POINT(0 0)'::geometry as h" +
+             // See https://github.com/CartoDB/CartoDB-SQL-API/issues/117
+             ", now()::date as i" +
+             ", '1'::numeric as j" +
+             " LIMIT 0"
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        var parsedBody = JSON.parse(res.body);
+        assert.equal(_.keys(parsedBody.fields).length, 10);
+        assert.equal(parsedBody.fields.a.type, 'number');
+        assert.equal(parsedBody.fields.b.type, 'number');
+        assert.equal(parsedBody.fields.c.type, 'string');
+        assert.equal(parsedBody.fields.d.type, 'string');
+        assert.equal(parsedBody.fields.e.type, 'date');
+        assert.equal(parsedBody.fields.f.type, 'string');
+        assert.equal(parsedBody.fields.g.type, 'boolean');
+        assert.equal(parsedBody.fields.h.type, 'geometry');
+        assert.equal(parsedBody.fields.i.type, 'date');
+        assert.equal(parsedBody.fields.j.type, 'number');
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/100
+it('numeric fields are rendered as numbers in JSON', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: "WITH inp AS ( SELECT 1::int2 as a, 2::int4 as b, " +
+                                  "3::int8 as c, 4::float4 as d, " +
+                                  "5::float8 as e, 6::numeric as f" +
+              ") SELECT a,b,c,d,e,f," +
+              " ARRAY[a] AS _a, " +
+              " ARRAY[b] AS _b, " +
+              " ARRAY[c] AS _c, " +
+              " ARRAY[d] AS _d, " +
+              " ARRAY[e] AS _e, " +
+              " ARRAY[f] AS _f " +
+              "FROM inp"
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        var parsedBody = JSON.parse(res.body);
+        var row = parsedBody.rows[0];
+        assert.equal(typeof(row.a), 'number');
+        assert.equal(typeof(row.b), 'number');
+        assert.equal(typeof(row.c), 'number');
+        assert.equal(typeof(row.d), 'number');
+        assert.equal(typeof(row.e), 'number');
+        assert.equal(typeof(row.f), 'number');
+        assert.equal(typeof(row._a[0]), 'number');
+        assert.equal(typeof(row._b[0]), 'number');
+        assert.equal(typeof(row._c[0]), 'number');
+        assert.equal(typeof(row._d[0]), 'number');
+        assert.equal(typeof(row._e[0]), 'number');
+        assert.equal(typeof(row._f[0]), 'number');
+        done();
+    });
+});
+
+// Timezone information is retained with JSON output
+//
+// NOTE: results of these tests rely on the TZ env variable
+//       being set to 'Europe/Rome'. The env variable cannot
+//       be set within this test in a reliable way, see
+//       https://github.com/joyent/node/issues/3286
+//
+// FIXME: we'd like to also test UTC outputs of these
+//        numbers, but it'd currently take running the
+//        test again (new mocha run) with a different TZ
+//
+it('timezone info in JSON output', function(done){
+  step(
+    function testEuropeRomeExplicit() {
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET timezone TO 'Europe/Rome'; SELECT '2000-01-01T00:00:00+01'::timestamptz as d"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      },{ }, function(err, res) {
+          try {
+            assert.equal(res.statusCode, 200, res.body);
+            var parsedBody = JSON.parse(res.body);
+            assert.equal(parsedBody.rows[0].d, '2000-01-01T00:00:00+0100');
+            next();
+          } catch (err) {
+            next(err);
+          }
+      });
+    },
+    function testEuropeRomeImplicit(err) {
+      assert.ifError(err);
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET timezone TO 'Europe/Rome'; SELECT '2000-01-01T00:00:00'::timestamp as d"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      },{ }, function(err, res) {
+          try {
+            assert.equal(res.statusCode, 200, res.body);
+            var parsedBody = JSON.parse(res.body);
+            assert.equal(parsedBody.rows[0].d, '2000-01-01T00:00:00+0100');
+            next();
+          } catch (err) {
+            next(err);
+          }
+      });
+    },
+    function testUTCExplicit(err) {
+      assert.ifError(err);
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET timezone TO 'UTC'; SELECT '2000-01-01T00:00:00+00'::timestamptz as d"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      },{ }, function(err, res) {
+          try {
+            assert.equal(res.statusCode, 200, res.body);
+            var parsedBody = JSON.parse(res.body);
+            assert.equal(parsedBody.rows[0].d, '2000-01-01T01:00:00+0100');
+            next();
+          } catch (err) {
+            next(err);
+          }
+      });
+    },
+    function testUTCImplicit(err) {
+      assert.ifError(err);
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET timezone TO 'UTC'; SELECT '2000-01-01T00:00:00'::timestamp as d"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      },{ }, function(err, res) {
+          try {
+            assert.equal(res.statusCode, 200, res.body);
+            var parsedBody = JSON.parse(res.body);
+            assert.equal(parsedBody.rows[0].d, '2000-01-01T00:00:00+0100');
+            next();
+          } catch (err) {
+            next(err);
+          }
+      });
+    },
+    function finish(err) {
+      done(err);
+    }
+  );
+});
+
+// WARNING and NOTICE in JSON output
+// See https://github.com/CartoDB/CartoDB-SQL-API/issues/104
+it('notice and warning info in JSON output', function(done){
+  step(
+    function addRaiseFunction() {
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "create or replace function raise(lvl text, msg text) returns void as $$ begin if lvl = 'notice' " +
+                "then raise notice '%', msg; elsif lvl = 'warning' then raise warning '%', msg; " +
+                "else raise exception '%', msg; end if; end; $$ language plpgsql;",
+            api_key: '1234'
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      }, RESPONSE_OK, this);
+    },
+    function raiseNotice(err) {
+      assert.ifError(err);
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET client_min_messages TO 'notice'; select raise('notice', 'hello notice')"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      }, RESPONSE_OK, function(err, res) {
+          try {
+            var parsedBody = JSON.parse(res.body);
+            assert.ok(parsedBody.hasOwnProperty('notices'), 'Missing notices from result');
+            assert.equal(parsedBody.notices.length, 1);
+            assert.equal(parsedBody.notices[0], 'hello notice');
+          } catch (e) {
+            return next(e);
+          }
+          next(err);
+      });
+    },
+    function raiseWarning(err) {
+      assert.ifError(err);
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET client_min_messages TO 'notice'; select raise('warning', 'hello warning')"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      }, RESPONSE_OK, function(err, res) {
+          try {
+            var parsedBody = JSON.parse(res.body);
+            assert.ok(parsedBody.hasOwnProperty('warnings'), 'Missing warnings from result');
+            assert.equal(parsedBody.warnings.length, 1);
+            assert.equal(parsedBody.warnings[0], 'hello warning');
+          } catch (e) {
+              return next(e);
+          }
+          next(err);
+      });
+    },
+    function raiseBothWarningAndNotice(err) {
+      assert.ifError(err);
+      var next = this;
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "SET client_min_messages TO 'notice'; select raise('warning', 'hello again warning'), " +
+                "raise('notice', 'hello again notice');"
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      }, RESPONSE_OK, function(err, res) {
+          try {
+            var parsedBody = JSON.parse(res.body);
+            assert.ok(parsedBody.hasOwnProperty('warnings'), 'Missing warnings from result');
+            assert.equal(parsedBody.warnings.length, 1);
+            assert.equal(parsedBody.warnings[0], 'hello again warning');
+            assert.ok(parsedBody.hasOwnProperty('notices'), 'Missing notices from result');
+            assert.equal(parsedBody.notices.length, 1);
+            assert.equal(parsedBody.notices[0], 'hello again notice');
+          } catch (e) {
+              return next(e);
+          }
+          next(err);
+      });
+    },
+    function delRaiseFunction() {
+      assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify({
+            q: "DROP function raise(text, text)",
+            api_key: '1234'
+          }),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+      }, RESPONSE_OK, function(err, res) {
+          try {
+            assert.equal(res.statusCode, 200, res.body);
+            JSON.parse(res.body);
+          } catch (e) {
+              err = new Error(err + ',' + e);
+          }
+          done(err);
+      });
+    }
+  );
+});
+
+it('GET with callback param returns wrapped result set with callback as jsonp', function(done) {
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&callback=foo_jsonp',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res) {
+        assert.equal(res.statusCode, 200, res.body);
+        assert.ok(res.body.match(/foo\_jsonp\(.*\)/));
+        done();
+    });
+});
+
+it('GET with callback must return 200 status error even if it is an error', function(done){
+    assert.response(server, {
+        url: "/api/v1/sql?q=DROP%20TABLE%20untitle_table_4&callback=foo_jsonp",
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{}, function(err, res) {
+        assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+        var didRunJsonCallback = false;
+        // jshint ignore:start
+        function foo_jsonp(body) {
+            assert.ok(body.error[0].match(/must be owner of.+? untitle_table_4/));
+            didRunJsonCallback = true;
+        }
+        eval(res.body);
+        // jshint ignore:end
+        assert.ok(didRunJsonCallback);
+        done();
+    });
+});
+
+it('GET with slow query exceeding statement timeout returns proper error message', function(done){
+    assert.response(server, {
+            url: "/api/v1/sql?q=select%20pg_sleep(2.1)%20as%20sleep",
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        {
+            // status: 429,  ---> Both 200 and 429 are valid
+            headers: {
+                'Content-Type': 'application/json; charset=utf-8'
+            }
+        },
+        function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error.error, [
+                    'You are over platform\'s limits: SQL query timeout error.' +
+                    ' Refactor your query before running again or contact CARTO support for more details.'
+                ]);
+
+            done();
+        });
+});
+
+it('GET with slow query exceeding statement timeout returns proper error message (streaming)', function(done){
+    assert.response(server, {
+            url: "/api/v1/sql?q=SELECT%20pg_sleep(generate_series(2,10)/10.0)",
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        {
+            // status: 429,  ---> Both 200 and 429 are valid
+            headers: {
+                'Content-Type': 'application/json; charset=utf-8'
+            }
+        },
+        function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error.error, [
+                    'You are over platform\'s limits: SQL query timeout error.' +
+                    ' Refactor your query before running again or contact CARTO support for more details.'
+                ]);
+
+            done();
+        });
+});
+
+it('GET with slow python script exceeding statement timeout returns proper error message', function(done){
+    assert.response(server, {
+            url: "/api/v1/sql?q=select%20py_sleep(2.1)",
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        {
+            // status: 429,  ---> Both 200 and 429 are valid
+            headers: {
+                'Content-Type': 'application/json; charset=utf-8'
+            }
+        },
+        function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error.error, [
+                    'You are over platform\'s limits: SQL query timeout error.' +
+                    ' Refactor your query before running again or contact CARTO support for more details.'
+                ]);
+
+            done();
+        });
+});
+
+    it('too large rows get into error log', function(done){
+
+        var dbMaxRowSize = global.settings.db_max_row_size;
+        global.settings.db_max_row_size = 4;
+
+        var consoleErrorFn = console.error;
+        var hit = false;
+        var consoleError;
+        console.error = function(what) {
+            hit = true;
+            consoleError = what;
+        };
+        assert.response(
+            server,
+            {
+                url: "/api/v1/sql?" + querystring.stringify({
+                    q: "SELECT * FROM untitle_table_4"
+                }),
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                method: 'GET'
+            },
+            {
+                status: 400
+            },
+            function() {
+                assert.equal(hit, true);
+                var parsedError = JSON.parse(consoleError);
+                assert.ok(parsedError.error.match(/^row too large.*/i), "Expecting row size limit error");
+                assert.equal(parsedError.username, 'vizzuality');
+                assert.equal(parsedError.type, 'row_size_limit_exceeded');
+
+                global.settings.db_max_row_size = dbMaxRowSize;
+                console.error = consoleErrorFn;
+
+                done();
+            }
+        );
+    });
+
+
+});

test/acceptance/auth-api.js

@@ -0,0 +1,275 @@
+'use strict';
+
+const assert = require('../support/assert');
+const TestClient = require('../support/test-client');
+const BatchTestClient = require('../support/batch-test-client');
+const JobStatus = require('../../batch/job_status');
+
+describe('Auth API', function () {
+    const publicSQL = 'select * from untitle_table_4';
+    const scopedSQL = 'select * from scoped_table_1';
+    const privateSQL = 'select * from private_table';
+    const systemSQL = 'select * from information_schema.tables';
+
+    it('should get result from query using the default API key', function (done) {
+        this.testClient = new TestClient();
+        this.testClient.getResult(publicSQL, (err, result) => {
+            assert.ifError(err);
+            assert.equal(result.length, 6);
+            done();
+        });
+    });
+
+    it('should fail when using a wrong API key', function (done) {
+        this.testClient = new TestClient({ apiKey: 'THIS_API_KEY_DOES_NOT_EXIST' });
+
+        const expectedResponse = {
+            response: {
+                status: 401
+            }
+        };
+
+        this.testClient.getResult(publicSQL, expectedResponse, (err, result) => {
+            assert.ifError(err);
+            assert.equal(result.error, 'Unauthorized');
+            done();
+        });
+    });
+
+    it('should fail while fetching data (private dataset) and using the default API key', function (done) {
+        this.testClient = new TestClient();
+        const expectedResponse = {
+            response: {
+                status: 403
+            },
+            anonymous: true
+        };
+        this.testClient.getResult(privateSQL, expectedResponse, (err, result) => {
+            assert.ifError(err);
+            assert.ok(result.error[0].match(/permission denied for .+? private_table/));
+            done();
+        });
+    });
+
+    it('should get result from query using the master API key and public dataset', function (done) {
+        this.testClient = new TestClient({ apiKey: 1234 });
+        this.testClient.getResult(publicSQL, (err, result) => {
+            assert.ifError(err);
+            assert.equal(result.length, 6);
+            done();
+        });
+    });
+
+    it('should get result from query using the master API key and private dataset', function (done) {
+        this.testClient = new TestClient({ apiKey: 1234 });
+        this.testClient.getResult(privateSQL, (err, result) => {
+            assert.ifError(err);
+            assert.equal(result.length, 5);
+            done();
+        });
+    });
+
+    it('should get result from query using the regular API key and scoped dataset', function (done) {
+        this.testClient = new TestClient({ apiKey: 'regular1' });
+        this.testClient.getResult(scopedSQL, (err, result) => {
+            assert.ifError(err);
+            assert.equal(result.length, 4);
+            done();
+        });
+    });
+
+    it('should fail while fetching data (scoped dataset) and using regular API key', function (done) {
+        this.testClient = new TestClient({ apiKey: 'regular2' });
+        const expectedResponse = {
+            response: {
+                status: 403
+            }
+        };
+
+        this.testClient.getResult(scopedSQL, expectedResponse, (err, result) => {
+            assert.ifError(err);
+            assert.ok(result.error[0].match(/permission denied for .+? scoped_table_1/));
+            done();
+        });
+    });
+
+    describe('Batch API', function () {
+        it('should create a job with master api key and get it done', function (done) {
+            this.testClient = new BatchTestClient({ apiKey: '1234' });
+
+            this.testClient.createJob({ query: scopedSQL }, (err, jobResult) => {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult.getStatus(function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.DONE);
+
+                    done();
+                });
+            });
+        });
+
+        it('should create a job with regular api key and get it failed', function (done) {
+            this.testClient = new BatchTestClient({ apiKey: 'regular1' });
+
+            this.testClient.createJob({ query: privateSQL }, { response: 403 }, (err, response) => {
+                if (err) {
+                    return done(err);
+                }
+
+                const body = JSON.parse(response.body);
+                assert.equal(body.error, 'permission denied');
+                done();
+            });
+        });
+
+        it('should create a job with default public api key and get it failed', function (done) {
+            this.testClient = new BatchTestClient({ apiKey: 'default_public' });
+
+            this.testClient.createJob({ query: publicSQL }, { response: 403 }, (err, response) => {
+                if (err) {
+                    return done(err);
+                }
+
+                const body = JSON.parse(response.body);
+                assert.equal(body.error, 'permission denied');
+                done();
+            });
+        });
+
+        it('should create a job with fallback default public api key and get it failed', function (done) {
+            this.testClient = new BatchTestClient();
+
+            this.testClient.createJob({ query: publicSQL }, { response: 403, anonymous: true }, (err, response) => {
+                if (err) {
+                    return done(err);
+                }
+
+                const body = JSON.parse(response.body);
+                assert.equal(body.error, 'permission denied');
+                done();
+            });
+        });
+
+        afterEach(function (done) {
+            this.testClient.drain(done);
+        });
+    });
+
+    describe('Basic Auth', function () {
+        it('should get result from query using the regular API key and scoped dataset', function (done) {
+            this.testClient = new TestClient({ authorization: 'vizzuality:regular1' });
+
+            this.testClient.getResult(scopedSQL, { anonymous: true }, (err, result) => {
+                assert.ifError(err);
+                assert.equal(result.length, 4);
+                done();
+            });
+        });
+
+        it('should fail while fetching data (scoped dataset) and using regular API key', function (done) {
+            this.testClient = new TestClient({ authorization: 'vizzuality:regular2' });
+            const expectedResponse = {
+                response: {
+                    status: 403
+                },
+                anonymous: true
+            };
+
+            this.testClient.getResult(scopedSQL, expectedResponse, (err, result) => {
+                assert.ifError(err);
+                assert.ok(result.error[0].match(/permission denied for .+? scoped_table_1/));
+                done();
+            });
+        });
+
+
+        it('should fail while fetching information schema and using default API key', function (done) {
+            this.testClient = new TestClient({ authorization: 'vizzuality:default_public' });
+            const expectedResponse = {
+                response: {
+                    status: 403
+                },
+                anonymous: true
+            };
+
+            this.testClient.getResult(systemSQL, expectedResponse, (err, result) => {
+                assert.ifError(err);
+                assert.equal(result.error, 'system tables are forbidden');
+                done();
+            });
+        });
+
+        it('should fail when basic auth name does not match with user\'s', function (done) {
+            this.testClient = new TestClient({ authorization: 'wadus:regular2' });
+            const expectedResponse = {
+                response: {
+                    status: 403
+                },
+                anonymous: true
+            };
+
+            this.testClient.getResult(scopedSQL, expectedResponse, (err, result) => {
+                assert.ifError(err);
+                assert.equal(result.error, 'permission denied');
+                done();
+            });
+        });
+
+        it('should fail when querying using a wrong API key', function (done) {
+            this.testClient = new TestClient({ authorization: 'vizzuality:THIS_API_KEY_DOES_NOT_EXIST' });
+
+            const expectedResponse = {
+                response: {
+                    status: 401
+                },
+                anonymous: true
+            };
+
+            this.testClient.getResult(publicSQL, expectedResponse, (err, result) => {
+                assert.ifError(err);
+                assert.equal(result.error, 'Unauthorized');
+                done();
+            });
+        });
+
+        describe('Batch API', function () {
+            it('should create a job with regular api key and get it failed', function (done) {
+                this.testClient = new BatchTestClient({ authorization: 'vizzuality:regular1', response: 403 });
+
+                this.testClient.createJob({ query: scopedSQL }, { anonymous: true }, (err, response) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    const body = JSON.parse(response.body);
+                    assert.equal(body.error, 'permission denied');
+                    done();
+                });
+            });
+
+            it('should create a job with default api key and get it failed', function (done) {
+                this.testClient = new BatchTestClient({ authorization: 'vizzuality:default_public', response: 403 });
+
+                this.testClient.createJob({ query: privateSQL }, { anonymous: true }, (err, response) => {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    const body = JSON.parse(response.body);
+                    assert.equal(body.error, 'permission denied');
+                    done();
+                });
+            });
+
+            afterEach(function (done) {
+                this.testClient.drain(done);
+            });
+        });
+    });
+});

test/acceptance/backend_crash.js

@@ -0,0 +1,83 @@
+'use strict';
+
+require('../helper');
+
+var assert = require('../support/assert');
+var step = require('step');
+var net = require('net');
+
+var sql_server_port = 5540;
+var sql_server = net.createServer(function(c) {
+  console.log('server connected');
+  c.destroy();
+  console.log('server socket destroyed.');
+  sql_server.close(function() {
+  console.log('server closed');
+  });
+});
+
+describe('backend crash', function() {
+
+before(function(done){
+  sql_server.listen(sql_server_port, done);
+});
+
+// See https://github.com/CartoDB/CartoDB-SQL-API/issues/135
+it('does not hang server', function(done){
+//console.log("settings:"); console.dir(global.settings);
+  var db_host_backup = global.settings.db_host;
+  var db_port_backup = global.settings.db_port;
+  global.settings.db_host = 'localhost';
+  global.settings.db_port = sql_server_port;
+  var server = require('../../app/server')();
+  step(
+    function sendQuery() {
+      assert.response(server, {
+          url: '/api/v1/sql?q=SELECT+1',
+          method: 'GET',
+          headers: {host: 'vizzuality.localhost' }
+      },{}, this);
+    },
+    function checkResponse(err, res) {
+      assert.ifError(err);
+      assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
+      var parsed = JSON.parse(res.body);
+      assert.ok(parsed.error);
+      var msg = parsed.error[0];
+      assert.ok(msg.match(/unexpected.*end/), msg);
+      return null;
+    },
+    function sendAnotherQuery() {
+      assert.response(server, {
+          url: '/api/v1/sql?q=SELECT+2',
+          method: 'GET',
+          headers: {host: 'vizzuality.localhost' }
+      },{}, this);
+    },
+    function checkResponse(err, res) {
+      assert.ifError(err);
+      assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
+      var parsed = JSON.parse(res.body);
+      assert.ok(parsed.error);
+      var msg = parsed.error[0];
+      assert.ok(msg.match(/connect/), msg);
+      return null;
+    },
+    function finish(err) {
+      global.settings.db_host = db_host_backup;
+      global.settings.db_port = db_port_backup;
+      done(err);
+    }
+  );
+});
+
+after(function(done) {
+  // be sure the sql_server is closed
+  if (sql_server.listening) {
+    return sql_server.close(done);
+  }
+
+  done();
+});
+
+});

test/acceptance/batch/batch-drain.test.js

@@ -0,0 +1,83 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+var batchFactory = require('../../../batch/index');
+
+var JobPublisher = require('../../../batch/pubsub/job-publisher');
+var JobQueue = require('../../../batch/job_queue');
+var JobBackend = require('../../../batch/job_backend');
+var JobService = require('../../../batch/job_service');
+var JobCanceller = require('../../../batch/job_canceller');
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+
+describe('batch module', function() {
+    var dbInstance = 'localhost';
+    var username = 'vizzuality';
+    var pool = redisUtils.getPool();
+    var jobPublisher = new JobPublisher(pool);
+    var jobQueue =  new JobQueue(metadataBackend, jobPublisher);
+    var jobBackend = new JobBackend(metadataBackend, jobQueue);
+    var jobCanceller = new JobCanceller();
+    var jobService = new JobService(jobBackend, jobCanceller);
+
+    before(function (done) {
+        this.batch = batchFactory(metadataBackend, pool);
+        this.batch.start();
+        this.batch.on('ready', done);
+    });
+
+    after(function (done) {
+        this.batch.stop();
+        redisUtils.clean('batch:*', done);
+    });
+
+    function createJob(sql, done) {
+        var data = {
+            user: username,
+            query: sql,
+            host: dbInstance,
+            dbname: 'cartodb_test_user_1_db',
+            dbuser: 'test_cartodb_user_1',
+            port: 5432,
+            pass: 'test_cartodb_user_1_pass',
+        };
+
+        jobService.create(data, function (err, job) {
+            if (err) {
+                return done(err);
+            }
+
+            done(null, job.serialize());
+        });
+    }
+
+    it('should drain the current job', function (done) {
+        var self = this;
+        createJob('select pg_sleep(3)', function (err, job) {
+            if (err) {
+                return done(err);
+            }
+            setTimeout(function () {
+                jobBackend.get(job.job_id, function (err, job) {
+                    if (err) {
+                        done(err);
+                    }
+                    assert.equal(job.status, 'running');
+
+                    self.batch.drain(function () {
+                        jobBackend.get(job.job_id, function (err, job) {
+                            if (err) {
+                                done(err);
+                            }
+                            assert.equal(job.status, 'pending');
+                            done();
+                        });
+                    });
+                });
+            }, 50);
+        });
+    });
+
+});

test/acceptance/batch/batch-limits.test.js

@@ -0,0 +1,52 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+var redisUtils = require('../../support/redis_utils');
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+const db_utils = require('../../support/db_utils');
+
+describe('batch query statement_timeout limit', function() {
+
+    before(function(done) {
+        this.batchTestClient = new BatchTestClient();
+        this.batchQueryTimeout = global.settings.batch_query_timeout;
+        global.settings.batch_query_timeout = 15000;
+        metadataBackend.redisCmd(5, 'HMSET', ['limits:batch:vizzuality', 'timeout', 100], done);
+    });
+    before(db_utils.resetPgBouncerConnections);
+    after(function(done) {
+        global.settings.batch_query_timeout = this.batchQueryTimeout;
+        redisUtils.clean('limits:batch:*', function() {
+            this.batchTestClient.drain(done);
+        }.bind(this));
+    });
+    after(db_utils.resetPgBouncerConnections);
+
+    function jobPayload(query) {
+        return {
+            query: query
+        };
+    }
+
+    it('should cancel with user statement_timeout limit', function (done) {
+        var payload = jobPayload('select pg_sleep(10)');
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                assert.ok(job.failed_reason.match(/statement.*timeout/));
+                return done();
+            });
+        });
+    });
+
+});

test/acceptance/batch/batch.multiquery.test.js

@@ -0,0 +1,236 @@
+'use strict';
+
+require('../../helper');
+
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+var assert = require('../../support/assert');
+var queue = require('queue-async');
+
+describe('batch multiquery', function() {
+    function jobPayload(query) {
+        return {
+            query: query
+        };
+    }
+
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function (done) {
+        this.batchTestClient.drain(done);
+    });
+
+    it('should perform one multiquery job with two queries', function (done) {
+        var queries = [
+            'select pg_sleep(0)',
+            'select pg_sleep(0)'
+        ];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                return done();
+            });
+        });
+    });
+
+    it('should perform one multiquery job with two queries and fail on last one', function (done) {
+        var queries = [
+            'select pg_sleep(0)',
+            'select shouldFail()'
+        ];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                return done();
+            });
+        });
+    });
+
+    it('should perform one multiquery job with three queries and fail on last one', function (done) {
+        var queries = [
+            'select pg_sleep(0)',
+            'select pg_sleep(0)',
+            'select shouldFail()'
+        ];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                return done();
+            });
+        });
+    });
+
+
+    it('should perform one multiquery job with three queries and fail on second one', function (done) {
+        var queries = [
+            'select pg_sleep(0)',
+            'select shouldFail()',
+            'select pg_sleep(0)'
+        ];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                return done();
+            });
+        });
+    });
+
+    it('should perform two multiquery job with two queries for each one', function (done) {
+        var self = this;
+
+        var jobs = [
+            [
+                'select pg_sleep(0)',
+                'select pg_sleep(0)'
+            ],
+            [
+                'select pg_sleep(0)',
+                'select pg_sleep(0)'
+            ]
+        ];
+
+        var jobsQueue = queue(1);
+
+        jobs.forEach(function(job) {
+            jobsQueue.defer(function(payload, done) {
+                self.batchTestClient.createJob(payload, function(err, jobResult) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult.getStatus(done);
+                });
+            }, jobPayload(job));
+        });
+
+        jobsQueue.awaitAll(function (err, jobsCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            jobsCreated.forEach(function(job) {
+                assert.equal(job.status, JobStatus.DONE);
+            });
+
+            return done();
+        });
+    });
+
+    it('should perform two multiquery job with two queries for each one and fail the first one', function (done) {
+        var self = this;
+
+        var jobs = [
+            [
+                'select pg_sleep(0)',
+                'select shouldFail()'
+            ],
+            [
+                'select pg_sleep(0)',
+                'select pg_sleep(0)'
+            ]
+        ];
+
+        var expectedStatus = [JobStatus.FAILED, JobStatus.DONE];
+        var jobsQueue = queue(1);
+
+        jobs.forEach(function(job) {
+            jobsQueue.defer(function(payload, done) {
+                self.batchTestClient.createJob(payload, function(err, jobResult) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult.getStatus(done);
+                });
+            }, jobPayload(job));
+        });
+
+        jobsQueue.awaitAll(function (err, jobsCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            var statuses = jobsCreated.map(function(job) {
+                return job.status;
+            });
+            assert.deepEqual(statuses, expectedStatus);
+
+            return done();
+        });
+    });
+
+    it('should perform two multiquery job with two queries for each one and fail the second one', function (done) {
+        var self = this;
+
+        var jobs = [
+            [
+                'select pg_sleep(0)',
+                'select pg_sleep(0)'
+            ],
+            [
+                'select pg_sleep(0)',
+                'select shouldFail()'
+            ]
+        ];
+
+        var expectedStatus = [JobStatus.DONE, JobStatus.FAILED];
+        var jobsQueue = queue(1);
+
+        jobs.forEach(function(job) {
+            jobsQueue.defer(function(payload, done) {
+                self.batchTestClient.createJob(payload, function(err, jobResult) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult.getStatus(done);
+                });
+            }, jobPayload(job));
+        });
+
+        jobsQueue.awaitAll(function (err, jobsCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            var statuses = jobsCreated.map(function(job) {
+                return job.status;
+            });
+            assert.deepEqual(statuses, expectedStatus);
+
+            return done();
+        });
+    });
+});

test/acceptance/batch/batch.test.js

@@ -0,0 +1,197 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var queue = require('queue-async');
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('batch happy cases', function() {
+
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function(done) {
+        this.batchTestClient.drain(done);
+    });
+
+    function jobPayload(query) {
+        return {
+            query: query
+        };
+    }
+
+    it('should perform job with select', function (done) {
+        var payload = jobPayload('select * from private_table');
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                return done();
+            });
+        });
+    });
+
+    it('should perform job with select into', function (done) {
+        var payload = jobPayload('select * into batch_test_table from (select * from private_table) as job');
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                return done();
+            });
+        });
+    });
+
+    it('should perform job with select from result table', function (done) {
+        var payload = jobPayload('select * from batch_test_table');
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                return done();
+            });
+        });
+    });
+
+    it('should perform all enqueued jobs', function (done) {
+        var self = this;
+
+        var jobs = [
+            'select * from private_table',
+            'select * from private_table',
+            'select * from private_table',
+        ];
+
+        var jobsQueue = queue(1);
+
+        jobs.forEach(function(job) {
+            jobsQueue.defer(function(payload, done) {
+                self.batchTestClient.createJob(payload, function(err, jobResult) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult.getStatus(done);
+                });
+            }, jobPayload(job));
+        });
+
+        jobsQueue.awaitAll(function (err, jobsCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            jobsCreated.forEach(function(job) {
+                assert.equal(job.status, JobStatus.DONE);
+            });
+
+            return done();
+        });
+    });
+
+    it('should set all job as failed', function (done) {
+        var self = this;
+
+        var jobs = [
+            'select * from unexistent_table',
+            'select * from unexistent_table',
+            'select * from unexistent_table'
+        ];
+
+        var jobsQueue = queue(1);
+
+        jobs.forEach(function(job) {
+            jobsQueue.defer(function(payload, done) {
+                self.batchTestClient.createJob(payload, function(err, jobResult) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult.getStatus(done);
+                });
+            }, jobPayload(job));
+        });
+
+        jobsQueue.awaitAll(function (err, jobsCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            jobsCreated.forEach(function(job) {
+                assert.equal(job.status, JobStatus.FAILED);
+            });
+
+            return done();
+        });
+    });
+
+    it('should perform job with array of select', function (done) {
+        var queries = ['select * from private_table limit 1', 'select * from private_table'];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                return done();
+            });
+        });
+    });
+
+    it('should set job as failed if last query fails', function (done) {
+        var queries = ['select * from private_table', 'select * from undefined_table'];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                return done();
+            });
+        });
+    });
+
+    it('should set job as failed if first query fails', function (done) {
+        var queries = ['select * from undefined_table', 'select * from private_table'];
+
+        var payload = jobPayload(queries);
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                return done();
+            });
+        });
+    });
+});

test/acceptance/batch/batch.wip.test.js

@@ -0,0 +1,99 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('batch work in progress endpoint happy cases', function() {
+
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function(done) {
+        this.batchTestClient.drain(done);
+    });
+
+    function jobPayload(query) {
+        return {
+            query: query
+        };
+    }
+
+    it('should get a list of work in progress jobs group by user', function (done) {
+        var self = this;
+        var user = 'vizzuality';
+        var queries = ['select pg_sleep(3)'];
+        var payload = jobPayload(queries);
+
+        self.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.RUNNING, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                self.batchTestClient.getWorkInProgressJobs(function (err, workInProgressJobs) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    if (!workInProgressJobs[user]) {
+                        return done(new Error('User should be in work-in-progress list'));
+                    }
+
+                    assert.ok(Array.isArray(workInProgressJobs[user]));
+                    assert.ok(workInProgressJobs[user].length >= 1);
+                    for (var i = 0; i < workInProgressJobs[user].length; i++) {
+                        if (workInProgressJobs[user][i] === jobResult.job.job_id) {
+                            return jobResult.cancel(done);
+                        }
+                    }
+
+                    return done(new Error('Job should not be in work-in-progress list'));
+                });
+            });
+        });
+    });
+
+    it('should get a list of work in progress jobs w/o the finished ones', function (done) {
+        var self = this;
+        var user = 'vizzuality';
+        var queries = ['select pg_sleep(0)'];
+        var payload = jobPayload(queries);
+
+        self.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                self.batchTestClient.getWorkInProgressJobs(function (err, workInProgressJobs) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    if (workInProgressJobs[user]) {
+                        assert.ok(Array.isArray(workInProgressJobs[user]));
+                        assert.ok(workInProgressJobs[user].length >= 1);
+                        for (var i = 0; i < workInProgressJobs[user].length; i++) {
+                            if (workInProgressJobs[user][i] === jobResult.job.job_id) {
+                                return done(new Error('Job should not be in work-in-progress list'));
+                            }
+                        }
+                    }
+                    done();
+                });
+            });
+        });
+    });
+});

test/acceptance/batch/job.callback-template.test.js

@@ -0,0 +1,126 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var TestClient = require('../../support/test-client');
+var JobStatus = require('../../../batch/job_status');
+var BatchTestClient = require('../../support/batch-test-client');
+
+describe('Batch API callback templates', function () {
+    before(function () {
+        this.batchTestClient = new BatchTestClient();
+        this.testClient = new TestClient();
+    });
+
+    after(function (done) {
+        this.batchTestClient.drain(done);
+    });
+
+    it('should use templates for error_message and job_id onerror callback' +
+        ' and keep the original templated query but use the error message', function (done) {
+        var self = this;
+        var payload = {
+            "query": {
+                "query": [
+                    {
+                        "query": "SELECT * FROM invalid_table",
+                        "onerror": "INSERT INTO test_batch_errors " +
+                        "values ('<%= job_id %>', '<%= error_message %>')"
+                    }
+                ]
+            }
+        };
+        var expectedQuery = {
+            query: [
+                {
+                    "query": "SELECT * FROM invalid_table",
+                    "onerror": "INSERT INTO test_batch_errors values ('<%= job_id %>', '<%= error_message %>')",
+                    status: 'failed',
+                    fallback_status: 'done'
+                }
+            ]
+        };
+
+        self.testClient.getResult(
+            'create table test_batch_errors (job_id text, error_message text)', function (err) {
+            if (err) {
+                return done(err);
+            }
+
+            self.batchTestClient.createJob(payload, function(err, jobResult) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult.getStatus(JobStatus.FAILED, function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult.validateExpectedResponse(expectedQuery);
+                    self.testClient.getResult('select * from test_batch_errors', function(err, rows) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.equal(rows[0].job_id, job.job_id);
+                        assert.equal(rows[0].error_message, 'relation "invalid_table" does not exist');
+                        self.testClient.getResult('drop table test_batch_errors', done);
+                    });
+                });
+            });
+        });
+    });
+
+    it('should use template for job_id onsuccess callback ' +
+        'and keep the original templated query but use the job_id', function (done) {
+        var self = this;
+        var payload = {
+            "query": {
+                "query": [
+                    {
+                        query: "create table batch_jobs (job_id text)"
+                    },
+                    {
+                        "query": "SELECT 1",
+                        "onsuccess": "INSERT INTO batch_jobs values ('<%= job_id %>')"
+                    }
+                ]
+            }
+        };
+        var expectedQuery = {
+            query: [
+                {
+                    query: "create table batch_jobs (job_id text)",
+                    status: 'done'
+                },
+                {
+                    query: "SELECT 1",
+                    onsuccess: "INSERT INTO batch_jobs values ('<%= job_id %>')",
+                    status: 'done',
+                    fallback_status: 'done'
+                }
+            ]
+        };
+
+        self.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult.validateExpectedResponse(expectedQuery);
+                self.testClient.getResult('select * from batch_jobs', function(err, rows) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.equal(rows[0].job_id, job.job_id);
+
+                    self.testClient.getResult('drop table batch_jobs', done);
+                });
+            });
+        });
+    });
+});

test/acceptance/batch/job.fallback.test.js

@@ -0,0 +1,934 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var JobStatus = require('../../../batch/job_status');
+var BatchTestClient = require('../../support/batch-test-client');
+
+describe('Batc API fallback job', function () {
+
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function(done) {
+        this.batchTestClient.drain(done);
+    });
+
+    it('"onsuccess" on first query should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 1",
+                "status": "done",
+                "fallback_status": "done"
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onerror" on first query should not be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4",
+                    onerror: "SELECT * FROM untitle_table_4 limit 1"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4",
+                "onerror": "SELECT * FROM untitle_table_4 limit 1",
+                "status": "done",
+                "fallback_status": "skipped"
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onerror" on first query should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM nonexistent_table /* query should fail */",
+                    onerror: "SELECT * FROM untitle_table_4 limit 1"
+                }]
+            }
+        };
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM nonexistent_table /* query should fail */',
+                onerror: 'SELECT * FROM untitle_table_4 limit 1',
+                status: 'failed',
+                fallback_status: 'done',
+                failed_reason: 'relation "nonexistent_table" does not exist'
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" on first query should not be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM nonexistent_table /* query should fail */",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+                }]
+            }
+        };
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM nonexistent_table /* query should fail */',
+                onsuccess: 'SELECT * FROM untitle_table_4 limit 1',
+                status: 'failed',
+                fallback_status: 'skipped',
+                failed_reason: 'relation "nonexistent_table" does not exist'
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4",
+                }],
+                onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4",
+                "status": "done"
+            }],
+            "onsuccess": "SELECT * FROM untitle_table_4 limit 1"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                assert.equal(job.fallback_status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" should not be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM nonexistent_table /* query should fail */",
+                }],
+                onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM nonexistent_table /* query should fail */",
+                "status": "failed",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }],
+            "onsuccess": "SELECT * FROM untitle_table_4 limit 1"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                assert.equal(job.fallback_status, JobStatus.SKIPPED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onerror" should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM nonexistent_table /* query should fail */"
+                }],
+                onerror: "SELECT * FROM untitle_table_4 limit 1"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM nonexistent_table /* query should fail */",
+                "status": "failed",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }],
+            "onerror": "SELECT * FROM untitle_table_4 limit 1"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                assert.equal(job.fallback_status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onerror" should not be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4",
+                }],
+                onerror: "SELECT * FROM untitle_table_4 limit 1"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4",
+                "status": "done"
+            }],
+            "onerror": "SELECT * FROM untitle_table_4 limit 1"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                assert.equal(job.fallback_status, JobStatus.SKIPPED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" & "onsuccess" on query should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+                }],
+                onsuccess: "SELECT * FROM untitle_table_4 limit 2"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 1",
+                "status": "done",
+                "fallback_status": "done"
+            }],
+            "onsuccess": "SELECT * FROM untitle_table_4 limit 2"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                assert.equal(job.fallback_status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for each query should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 2",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 3"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 1",
+                "status": "done",
+                "fallback_status": "done"
+            }, {
+                "query": "SELECT * FROM untitle_table_4 limit 2",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 3",
+                "status": "done",
+                "fallback_status": "done"
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for each query should not be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM nonexistent_table /* should fail */",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 2",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 3"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM nonexistent_table /* should fail */",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 1",
+                "status": "failed",
+                "fallback_status": "skipped",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }, {
+                "query": "SELECT * FROM untitle_table_4 limit 2",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 3",
+                "status": "skipped",
+                "fallback_status": "skipped"
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for second query should not be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 2",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+                }, {
+                    query: "SELECT * FROM nonexistent_table /* should fail */",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 3"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4 limit 2",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 1",
+                "status": "done",
+                "fallback_status": "done"
+            }, {
+                "query": "SELECT * FROM nonexistent_table /* should fail */",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 3",
+                "status": "failed",
+                "fallback_status": "skipped",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onerror" should not be triggered for any query and "skipped"', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1",
+                    onerror: "SELECT * FROM untitle_table_4 limit 2"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 3",
+                    onerror: "SELECT * FROM untitle_table_4 limit 4"
+                }]
+            }
+        };
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM untitle_table_4 limit 1',
+                onerror: 'SELECT * FROM untitle_table_4 limit 2',
+                status: 'done',
+                fallback_status: 'skipped'
+            }, {
+                query: 'SELECT * FROM untitle_table_4 limit 3',
+                onerror: 'SELECT * FROM untitle_table_4 limit 4',
+                status: 'done',
+                fallback_status: 'skipped'
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" should be "skipped"', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 2"
+                }]
+            }
+        };
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */',
+                onsuccess: 'SELECT * FROM untitle_table_4 limit 2',
+                status: 'failed',
+                fallback_status: 'skipped',
+                failed_reason: 'syntax error at end of input'
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" should not be triggered and "skipped"', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */",
+                }],
+                onsuccess: "SELECT * FROM untitle_table_4 limit 2"
+            }
+        };
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */',
+                status: 'failed',
+                failed_reason: 'syntax error at end of input'
+            }],
+            onsuccess: 'SELECT * FROM untitle_table_4 limit 2'
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for first query should fail', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1",
+                    onsuccess: "SELECT * FROM nonexistent_table /* should fail */"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 2",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 3"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4 limit 1",
+                "onsuccess": "SELECT * FROM nonexistent_table /* should fail */",
+                "status": "done",
+                "fallback_status": "failed",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }, {
+                "query": "SELECT * FROM untitle_table_4 limit 2",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 3",
+                "status": "done",
+                "fallback_status": "done"
+            }]
+        };
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for second query should fail', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 2"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 3",
+                    onsuccess: "SELECT * FROM nonexistent_table /* should fail */"
+                }]
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4 limit 1",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 2",
+                "status": "done",
+                "fallback_status": "done"
+            }, {
+                "query": "SELECT * FROM untitle_table_4 limit 3",
+                "onsuccess": "SELECT * FROM nonexistent_table /* should fail */",
+                "status": "done",
+                "fallback_status": "failed",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for job & "onsuccess" for each query should be triggered', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 2"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 3",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 4"
+                }],
+                onsuccess: "SELECT * FROM untitle_table_4 limit 5"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4 limit 1",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 2",
+                "status": "done",
+                "fallback_status": "done"
+            }, {
+                "query": "SELECT * FROM untitle_table_4 limit 3",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 4",
+                "status": "done",
+                "fallback_status": "done"
+            }],
+            onsuccess: "SELECT * FROM untitle_table_4 limit 5"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                assert.equal(job.fallback_status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for job & "onsuccess" for each query should be triggered ' +
+    '(even second "onsuccess" fails job should be done)', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT * FROM untitle_table_4 limit 1",
+                    onsuccess: "SELECT * FROM untitle_table_4 limit 2"
+                }, {
+                    query: "SELECT * FROM untitle_table_4 limit 3",
+                    onsuccess: "SELECT * FROM nonexistent_table /* should fail */"
+                }],
+                onsuccess: "SELECT * FROM untitle_table_4 limit 5"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT * FROM untitle_table_4 limit 1",
+                "onsuccess": "SELECT * FROM untitle_table_4 limit 2",
+                "status": "done",
+                "fallback_status": "done"
+            }, {
+                "query": "SELECT * FROM untitle_table_4 limit 3",
+                "onsuccess": "SELECT * FROM nonexistent_table /* should fail */",
+                "status": "done",
+                "fallback_status": "failed",
+                "failed_reason": 'relation "nonexistent_table" does not exist'
+            }],
+            "onsuccess": "SELECT * FROM untitle_table_4 limit 5"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                assert.equal(job.fallback_status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('should fail first "onerror" and job "onerror" and skip the other ones', function (done) {
+        var payload = {
+            "query": {
+                "query": [{
+                    "query": "SELECT * FROM atm_madrid limit 1, should fail",
+                    "onerror": "SELECT * FROM atm_madrid limit 2"
+                }, {
+                    "query": "SELECT * FROM atm_madrid limit 3",
+                    "onerror": "SELECT * FROM atm_madrid limit 4"
+                }],
+                "onerror": "SELECT * FROM atm_madrid limit 5"
+            }
+        };
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM atm_madrid limit 1, should fail',
+                onerror: 'SELECT * FROM atm_madrid limit 2',
+                status: 'failed',
+                fallback_status: 'failed',
+                failed_reason: 'relation "atm_madrid" does not exist'
+            }, {
+                query: 'SELECT * FROM atm_madrid limit 3',
+                onerror: 'SELECT * FROM atm_madrid limit 4',
+                status: 'skipped',
+                fallback_status: 'skipped'
+            }],
+            onerror: 'SELECT * FROM atm_madrid limit 5'
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                assert.equal(job.fallback_status, JobStatus.FAILED);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('should run first "onerror" and job "onerror" and skip the other ones', function (done) {
+        var payload = {
+            "query": {
+                "query": [{
+                    "query": "SELECT * FROM untitle_table_4 limit 1, should fail",
+                    "onerror": "SELECT * FROM untitle_table_4 limit 2"
+                }, {
+                    "query": "SELECT * FROM untitle_table_4 limit 3",
+                    "onerror": "SELECT * FROM untitle_table_4 limit 4"
+                }],
+                "onerror": "SELECT * FROM untitle_table_4 limit 5"
+            }
+        };
+
+        var expectedQuery = {
+            "query": [
+              {
+                "query": "SELECT * FROM untitle_table_4 limit 1, should fail",
+                "onerror": "SELECT * FROM untitle_table_4 limit 2",
+                "status": "failed",
+                "fallback_status": "done",
+                "failed_reason": "LIMIT #,# syntax is not supported"
+              },
+              {
+                "query": "SELECT * FROM untitle_table_4 limit 3",
+                "onerror": "SELECT * FROM untitle_table_4 limit 4",
+                "status": "skipped",
+                "fallback_status": "skipped"
+              }
+            ],
+            "onerror": "SELECT * FROM untitle_table_4 limit 5"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                assert.equal(job.fallback_status, JobStatus.DONE);
+                jobResult.validateExpectedResponse(expectedQuery);
+                return done();
+            });
+        });
+    });
+
+    it('"onsuccess" for job & "onsuccess" for each query should not be triggered ' +
+    ' because it has been cancelled', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT pg_sleep(3)",
+                    onsuccess: "SELECT pg_sleep(0)"
+                }],
+                onsuccess: "SELECT pg_sleep(0)"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT pg_sleep(3)",
+                "onsuccess": "SELECT pg_sleep(0)",
+                "status": "cancelled",
+                "fallback_status": "skipped"
+            }],
+            "onsuccess": "SELECT pg_sleep(0)"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.status, JobStatus.RUNNING);
+
+                jobResult.cancel(function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.CANCELLED);
+                    assert.equal(job.fallback_status, JobStatus.SKIPPED);
+                    jobResult.validateExpectedResponse(expectedQuery);
+                    return done();
+                });
+            });
+        });
+    });
+
+    it('first "onsuccess" should be triggered and it will be cancelled', function (done) {
+        var payload = {
+            query: {
+                query: [{
+                    query: "SELECT pg_sleep(0)",
+                    onsuccess: "SELECT pg_sleep(3)"
+                }],
+                onsuccess: "SELECT pg_sleep(0)"
+            }
+        };
+        var expectedQuery = {
+            "query": [{
+                "query": "SELECT pg_sleep(0)",
+                "onsuccess": "SELECT pg_sleep(3)",
+                "status": "done",
+                "fallback_status": "cancelled"
+            }],
+            "onsuccess": "SELECT pg_sleep(0)"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.status, JobStatus.RUNNING);
+
+                jobResult.cancel(function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.CANCELLED);
+                    assert.equal(job.fallback_status, JobStatus.SKIPPED);
+                    jobResult.validateExpectedResponse(expectedQuery);
+                    return done();
+                });
+            });
+        });
+    });
+});

test/acceptance/batch/job.query.limit.test.js

@@ -0,0 +1,125 @@
+'use strict';
+
+/**
+ *
+ * Requires the database and tables setup in config/environments/test.js to exist
+ * Ensure the user is present in the pgbouncer auth file too
+ * TODO: Add OAuth tests.
+ *
+ * To run this test, ensure that cartodb_test_user_1_db metadata exists
+ * in Redis for the vizzuality.cartodb.com domain
+ *
+ * SELECT 5
+ * HSET rails:users:vizzuality id 1
+ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db
+ *
+ */
+require('../../helper');
+var JobController = require('../../../app/controllers/job_controller');
+var redisUtils = require('../../support/redis_utils');
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var querystring = require('qs');
+
+function payload(query) {
+    return JSON.stringify({query: query});
+}
+function payloadSize(query) {
+    return payload(query).length;
+}
+
+var minPayloadSize = payloadSize('');
+var queryMaxSize = new Array(JobController.MAX_LIMIT_QUERY_SIZE_IN_BYTES - minPayloadSize + 1).join('a');
+var queryTooLong = queryMaxSize.concat('a');
+
+describe('job query limit', function() {
+
+    function expectedErrorMessage(query) {
+        return JobController.getMaxSizeErrorMessage(payload(query));
+    }
+
+    after(function (done) {
+        redisUtils.clean('batch:*', done);
+    });
+
+    it('POST /api/v2/sql/job with a invalid query size  should respond with 400 query too long', function (done){
+
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: queryTooLong
+            })
+        }, {
+            status: 400
+        }, function (err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: [expectedErrorMessage(queryTooLong)] });
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job with a valid query size should respond with 201 created', function (done){
+
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: queryMaxSize
+            })
+        }, {
+            status: 201
+        }, function (err, res) {
+            var job = JSON.parse(res.body);
+            assert.ok(job.job_id);
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job with a invalid query size should consider multiple queries', function (done){
+        var queries = [queryTooLong, 'select 1'];
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: queries
+            })
+        }, {
+            status: 400
+        }, function (err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: [expectedErrorMessage(queries)] });
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job with a invalid query size should consider fallback queries/callbacks', function (done){
+        var fallbackQueries = {
+            query: [{
+                query: queryTooLong,
+                onsuccess: "SELECT * FROM untitle_table_4 limit 1"
+            }, {
+                query: "SELECT * FROM untitle_table_4 limit 2",
+                onsuccess: "SELECT * FROM untitle_table_4 limit 3"
+            }]
+        };
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: fallbackQueries
+            })
+        }, {
+            status: 400
+        }, function (err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: [expectedErrorMessage(fallbackQueries)] });
+            done();
+        });
+    });
+
+});

test/acceptance/batch/job.query.order.test.js

@@ -0,0 +1,59 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('job query order', function() {
+
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function (done) {
+        return this.batchTestClient.drain(done);
+    });
+
+    function createJob(queries) {
+        return {
+            query: queries
+        };
+    }
+
+    it('should run job queries in order (single consumer)', function (done) {
+        var jobRequest1 = createJob(["select 1", "select 2"]);
+        var jobRequest2 = createJob(["select 3"]);
+
+        this.batchTestClient.createJob(jobRequest1, function(err, jobResult1) {
+            if (err) {
+                return done(err);
+            }
+            this.batchTestClient.createJob(jobRequest2, function(err, jobResult2) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult1.getStatus(function (err, job1) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult2.getStatus(function(err, job2) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.equal(job1.status, JobStatus.DONE);
+                        assert.equal(job2.status, JobStatus.DONE);
+                        assert.ok(
+                            new Date(job1.updated_at).getTime() < new Date(job2.updated_at).getTime(),
+                            'job1 (' + job1.updated_at + ') should finish before job2 (' + job2.updated_at + ')'
+                        );
+                        done();
+                    });
+                });
+            });
+        }.bind(this));
+    });
+
+});

test/acceptance/batch/job.query.timeout.test.js

@@ -0,0 +1,99 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('job query timeout', function() {
+
+    before(function() {
+        this.batchQueryTimeout = global.settings.batch_query_timeout;
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function (done) {
+        global.settings.batch_query_timeout = this.batchQueryTimeout;
+        return this.batchTestClient.drain(done);
+    });
+
+    function createTimeoutQuery(query, timeout) {
+        return {
+            query: {
+                query: [
+                    {
+                        timeout: timeout,
+                        query: query
+                    }
+                ]
+            }
+        };
+    }
+
+    it('should run query with higher user timeout', function (done) {
+        var jobRequest = createTimeoutQuery("select pg_sleep(0.1)", 200);
+        this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function(err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.DONE);
+                done();
+            });
+        });
+    });
+
+    it('should fail to run query with lower user timeout', function (done) {
+        var jobRequest = createTimeoutQuery("select pg_sleep(0.1)", 50);
+        this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function(err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                done();
+            });
+        });
+    });
+
+    it('should fail to run query with user timeout if it is higher than config', function (done) {
+        global.settings.batch_query_timeout = 100;
+        var jobRequest = createTimeoutQuery("select pg_sleep(1)", 2000);
+        this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function(err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                done();
+            });
+        });
+    });
+
+    it('should fail to run query with user timeout if set to 0 (ignored timeout)', function (done) {
+        global.settings.batch_query_timeout = 100;
+        var jobRequest = createTimeoutQuery("select pg_sleep(1)", 0);
+        this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+            jobResult.getStatus(function(err, job) {
+                if (err) {
+                    return done(err);
+                }
+                assert.equal(job.status, JobStatus.FAILED);
+                done();
+            });
+        });
+    });
+});

test/acceptance/batch/job.test.js

@@ -0,0 +1,219 @@
+'use strict';
+
+/**
+ *
+ * Requires the database and tables setup in config/environments/test.js to exist
+ * Ensure the user is present in the pgbouncer auth file too
+ * TODO: Add OAuth tests.
+ *
+ * To run this test, ensure that cartodb_test_user_1_db metadata exists
+ * in Redis for the vizzuality.cartodb.com domain
+ *
+ * SELECT 5
+ * HSET rails:users:vizzuality id 1
+ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db
+ *
+ */
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+var querystring = require('querystring');
+
+describe('job module', function() {
+    var job = {};
+
+    after(function (done) {
+        redisUtils.clean('batch:*', done);
+    });
+
+    it('POST /api/v2/sql/job should respond with 200 and the created job', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: "SELECT * FROM untitle_table_4"
+            })
+        }, {
+            status: 201
+        }, function(err, res) {
+            job = JSON.parse(res.body);
+            assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+            assert.ok(job.job_id);
+            assert.equal(job.query, "SELECT * FROM untitle_table_4");
+            assert.equal(job.user, "vizzuality");
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job without query should respond with 400 and the corresponding message of error',
+    function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({})
+        }, {
+            status: 400
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] });
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                q: "SELECT * FROM untitle_table_4"
+            })
+        }, {
+            status: 400
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] });
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job with wrong api key should respond with 401 permission denied', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=wrong',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: "SELECT * FROM untitle_table_4"
+            })
+        }, {
+            status: 401
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: [ 'Unauthorized' ] });
+            done();
+        });
+    });
+
+    it('POST /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job?api_key=wrong',
+            headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'POST',
+            data: querystring.stringify({
+                query: "SELECT * FROM untitle_table_4"
+            })
+        }, {
+            status: 404
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, {
+                error: [
+                    'Sorry, we can\'t find CARTO user \'wrong-host\'. ' +
+                    'Please check that you have entered the correct domain.'
+                ]
+            });
+            done();
+        });
+    });
+
+    it('GET /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'GET'
+        }, {
+            status: 200
+        }, function(err, res) {
+            var jobGot = JSON.parse(res.body);
+            assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+            assert.equal(jobGot.query, "SELECT * FROM untitle_table_4");
+            assert.equal(jobGot.user, "vizzuality");
+            done();
+        });
+    });
+
+    it('GET /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'GET'
+        }, {
+            status: 401
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: ['Unauthorized'] });
+            done();
+        });
+    });
+
+    it('GET /api/v2/sql/job/:jobId with wrong jobId header respond with 400 and an error', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job/irrelevantJob?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'GET'
+        }, {
+            status: 400
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error , {
+                error: ['Job with id irrelevantJob not found']
+            });
+            done();
+        });
+    });
+
+    it('DELETE /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'DELETE'
+        }, {
+            status: 200
+        }, function(err, res) {
+            var jobCancelled = JSON.parse(res.body);
+            assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+            assert.equal(jobCancelled.job_id, job.job_id);
+            assert.equal(jobCancelled.query, "SELECT * FROM untitle_table_4");
+            assert.equal(jobCancelled.user, "vizzuality");
+            assert.equal(jobCancelled.status, "cancelled");
+            done();
+        });
+    });
+
+    it('DELETE /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong',
+            headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'DELETE'
+        }, {
+            status: 401
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error, { error: ['Unauthorized'] });
+            done();
+        });
+    });
+
+    it('DELETE /api/v2/sql/job/ with wrong host header respond with 404 not found', function (done){
+        assert.response(server, {
+            url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
+            headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+            method: 'DELETE'
+        }, {
+            status: 404
+        }, function(err, res) {
+            var error = JSON.parse(res.body);
+            assert.deepEqual(error , {
+                error: [
+                    'Sorry, we can\'t find CARTO user \'wrong-host\'. ' +
+                    'Please check that you have entered the correct domain.'
+                ]
+            });
+            done();
+        });
+    });
+});

test/acceptance/batch/job.timing.test.js

@@ -0,0 +1,115 @@
+'use strict';
+
+require('../../helper');
+
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('Batch API query timing', function () {
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function(done) {
+        this.batchTestClient.drain(done);
+    });
+
+    it('should report start and end time for each query with fallback queries' +
+        'and expose started_at and ended_at for all queries with fallback mechanism', function (done) {
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM untitle_table_4 limit 1',
+                onerror: 'SELECT * FROM untitle_table_4 limit 2',
+                status: 'done',
+                fallback_status: 'skipped'
+            }, {
+                query: 'SELECT * FROM untitle_table_4 limit 3',
+                onerror: 'SELECT * FROM untitle_table_4 limit 4',
+                status: 'done',
+                fallback_status: 'skipped'
+            }],
+            onerror: 'SELECT * FROM untitle_table_4 limit 5'
+        };
+
+        var payload = {
+            "query": {
+                "query": [
+                    {
+                        "query": "SELECT * FROM untitle_table_4 limit 1",
+                        "onerror": "SELECT * FROM untitle_table_4 limit 2"
+                    },
+                    {
+                        "query": "SELECT * FROM untitle_table_4 limit 3",
+                        "onerror": "SELECT * FROM untitle_table_4 limit 4"
+                    }
+                ],
+                "onerror": "SELECT * FROM untitle_table_4 limit 5"
+            }
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult.validateExpectedResponse(expectedQuery);
+                done();
+            });
+        });
+    });
+
+
+    it('should report start and end time for each query also for failing queries' +
+        'and expose started_at and ended_at for all queries with fallback mechanism (failed)', function (done) {
+        var expectedQuery = {
+            query: [{
+                query: 'SELECT * FROM untitle_table_4 limit 1',
+                onerror: 'SELECT * FROM untitle_table_4 limit 2',
+                status: 'done',
+                fallback_status: 'skipped'
+            }, {
+                query: 'SELECT * FROM untitle_table_4 limit 3 failed',
+                onerror: 'SELECT * FROM untitle_table_4 limit 4',
+                status: 'failed',
+                fallback_status: 'done'
+            }],
+            onerror: 'SELECT * FROM untitle_table_4 limit 5'
+        };
+
+        var payload = {
+            "query": {
+                "query": [
+                    {
+                        "query": "SELECT * FROM untitle_table_4 limit 1",
+                        "onerror": "SELECT * FROM untitle_table_4 limit 2"
+                    },
+                    {
+                        "query": "SELECT * FROM untitle_table_4 limit 3 failed",
+                        "onerror": "SELECT * FROM untitle_table_4 limit 4"
+                    }
+                ],
+                "onerror": "SELECT * FROM untitle_table_4 limit 5"
+            }
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.FAILED, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult.validateExpectedResponse(expectedQuery);
+                done();
+            });
+        });
+    });
+});

test/acceptance/batch/leader-multiple-users-query-order.test.js

@@ -0,0 +1,129 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+
+var TestClient = require('../../support/test-client');
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('multiple batch clients and users, job query order', function() {
+
+    before(function(done) {
+        this.batchTestClientA = new BatchTestClient({ name: 'consumerA' });
+        this.batchTestClientB = new BatchTestClient({ name: 'consumerB' });
+
+        this.testClient = new TestClient();
+        this.testClient.getResult(
+            [
+                'drop table if exists ordered_inserts_a',
+                'drop table if exists ordered_inserts_bbbbb',
+                'create table ordered_inserts_a (status numeric)',
+                'create table ordered_inserts_bbbbb (status numeric)'
+            ].join(';'),
+            done
+        );
+    });
+
+    after(function (done) {
+        this.batchTestClientA.drain(function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            this.batchTestClientB.drain(done);
+        }.bind(this));
+    });
+
+    function createJob(queries) {
+        return {
+            query: queries
+        };
+    }
+
+    it('should run job queries in order (multiple consumers)', function (done) {
+        var jobRequestA1 = createJob([
+            "insert into ordered_inserts_a values(1)",
+            "select pg_sleep(0.25)",
+            "insert into ordered_inserts_a values(2)"
+        ]);
+        var jobRequestA2 = createJob([
+            "insert into ordered_inserts_a values(3)"
+        ]);
+
+        var jobRequestB1 = createJob([
+            "insert into ordered_inserts_bbbbb values(1)"
+        ]);
+
+        var self = this;
+
+        this.batchTestClientA.createJob(jobRequestA1, function(err, jobResultA1) {
+            if (err) {
+                return done(err);
+            }
+
+            var override = { host: 'cartodb250user.cartodb.com' };
+            self.batchTestClientB.createJob(jobRequestB1, override, function(err, jobResultB1) {
+                if (err) {
+                    return done(err);
+                }
+
+                // we don't care about the producer
+                self.batchTestClientB.createJob(jobRequestA2, function(err, jobResultA2) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    jobResultA1.getStatus(function (err, jobA1) {
+                        if (err) {
+                            return done(err);
+                        }
+                        jobResultA2.getStatus(function(err, jobA2) {
+                            if (err) {
+                                return done(err);
+                            }
+                            jobResultB1.getStatus(function(err, jobB1) {
+                                assert.equal(jobA1.status, JobStatus.DONE);
+                                assert.equal(jobA2.status, JobStatus.DONE);
+                                assert.equal(jobB1.status, JobStatus.DONE);
+
+                                assert.ok(
+                                    new Date(jobA1.updated_at).getTime() < new Date(jobA2.updated_at).getTime(),
+                                    'A1 (' + jobA1.updated_at + ') ' +
+                                        'should finish before A2 (' + jobA2.updated_at + ')'
+                                );
+                                assert.ok(
+                                    new Date(jobB1.updated_at).getTime() < new Date(jobA1.updated_at).getTime(),
+                                    'B1 (' + jobA1.updated_at + ') ' +
+                                        'should finish before A1 (' + jobA1.updated_at + ')'
+                                );
+
+                                function statusMapper (status) { return { status: status }; }
+
+                                self.testClient.getResult('select * from ordered_inserts_a', function(err, rows) {
+                                    assert.ok(!err);
+
+                                    // cartodb250user and vizzuality test users share database
+                                    var expectedRows = [1, 2, 3].map(statusMapper);
+                                    assert.deepEqual(rows, expectedRows);
+
+                                    var query = 'select * from ordered_inserts_bbbbb';
+                                    self.testClient.getResult(query, override, function(err, rows) {
+                                        assert.ok(!err);
+
+                                        var expectedRows = [1].map(statusMapper);
+                                        assert.deepEqual(rows, expectedRows);
+
+                                        done();
+                                    });
+                                });
+                            });
+
+                        });
+                    });
+                });
+            });
+        });
+    });
+
+});

test/acceptance/batch/leader.job.query.order.test.js

@@ -0,0 +1,88 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+
+var TestClient = require('../../support/test-client');
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('multiple batch clients job query order', function() {
+
+    before(function(done) {
+        this.batchTestClient1 = new BatchTestClient({ name: 'consumerA' });
+        this.batchTestClient2 = new BatchTestClient({ name: 'consumerB' });
+
+        this.testClient = new TestClient();
+        this.testClient.getResult(
+            'drop table if exists ordered_inserts; create table ordered_inserts (status numeric)',
+            done
+        );
+    });
+
+    after(function (done) {
+        this.batchTestClient1.drain(function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            this.batchTestClient2.drain(done);
+        }.bind(this));
+    });
+
+    function createJob(queries) {
+        return {
+            query: queries
+        };
+    }
+
+    it('should run job queries in order (multiple consumers)', function (done) {
+        var jobRequest1 = createJob([
+            "insert into ordered_inserts values(1)",
+            "select pg_sleep(0.25)",
+            "insert into ordered_inserts values(2)"
+        ]);
+        var jobRequest2 = createJob([
+            "insert into ordered_inserts values(3)"
+        ]);
+
+        var self = this;
+
+        this.batchTestClient1.createJob(jobRequest1, function(err, jobResult1) {
+            if (err) {
+                return done(err);
+            }
+            this.batchTestClient2.createJob(jobRequest2, function(err, jobResult2) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult1.getStatus(function (err, job1) {
+                    if (err) {
+                        return done(err);
+                    }
+                    jobResult2.getStatus(function(err, job2) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.equal(job1.status, JobStatus.DONE);
+                        assert.equal(job2.status, JobStatus.DONE);
+
+                        self.testClient.getResult('select * from ordered_inserts', function(err, rows) {
+                            assert.ok(!err);
+
+                            assert.deepEqual(rows, [{ status: 1 }, { status: 2 }, { status: 3 }]);
+                            assert.ok(
+                                new Date(job1.updated_at).getTime() < new Date(job2.updated_at).getTime(),
+                                'job1 (' + job1.updated_at + ') should finish before job2 (' + job2.updated_at + ')'
+                            );
+                            done();
+                        });
+
+                    });
+                });
+            });
+        }.bind(this));
+    });
+
+});

test/acceptance/batch/queued-jobs-limit.test.js

@@ -0,0 +1,68 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+var TestClient = require('../../support/test-client');
+
+describe('max queued jobs', function() {
+
+    before(function(done) {
+        this.batch_max_queued_jobs = global.settings.batch_max_queued_jobs;
+        global.settings.batch_max_queued_jobs = 1;
+        this.server = require('../../../app/server')();
+        this.testClient = new TestClient();
+        this.testClient.getResult(
+            'drop table if exists max_queued_jobs_inserts; create table max_queued_jobs_inserts (status numeric)',
+            done
+        );
+    });
+
+    after(function (done) {
+        global.settings.batch_max_queued_jobs = this.batch_max_queued_jobs;
+        redisUtils.clean('batch:*', done);
+    });
+
+    function createJob(server, status, callback) {
+        assert.response(
+            server,
+            {
+                url: '/api/v2/sql/job?api_key=1234',
+                headers: {
+                    host: 'vizzuality.cartodb.com',
+                    'Content-Type': 'application/json'
+                },
+                method: 'POST',
+                data: JSON.stringify({
+                    query: "insert into max_queued_jobs_inserts values (1)"
+                })
+            },
+            {
+                status: status
+            },
+            function(err, res) {
+                if (err) {
+                    return callback(err);
+                }
+
+                return callback(null, JSON.parse(res.body));
+            }
+        );
+    }
+
+    it('POST /api/v2/sql/job should respond with 200 and the created job', function (done) {
+        var self = this;
+        createJob(this.server, 201, function(err) {
+            assert.ok(!err);
+
+            createJob(self.server, 400, function(err, res) {
+                assert.ok(!err);
+                assert.equal(res.error[0], "Failed to create job. Max number of jobs (" +
+                    global.settings.batch_max_queued_jobs + ") queued reached");
+                done();
+            });
+        });
+    });
+
+});

test/acceptance/batch/scheduler-basic.test.js

@@ -0,0 +1,99 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+
+var TestClient = require('../../support/test-client');
+var BatchTestClient = require('../../support/batch-test-client');
+var JobStatus = require('../../../batch/job_status');
+
+describe('basic scheduling', function() {
+
+    before(function(done) {
+        this.batchTestClientA = new BatchTestClient({ name: 'consumerA' });
+        this.batchTestClientB = new BatchTestClient({ name: 'consumerB' });
+
+        this.testClient = new TestClient();
+        this.testClient.getResult(
+            [
+                'drop table if exists ordered_inserts_a',
+                'create table ordered_inserts_a (status numeric)'
+            ].join(';'),
+            done
+        );
+    });
+
+    after(function (done) {
+        this.batchTestClientA.drain(function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            this.batchTestClientB.drain(done);
+        }.bind(this));
+    });
+
+    function createJob(queries) {
+        return {
+            query: queries
+        };
+    }
+
+    it('should run job queries in order (multiple consumers)', function (done) {
+        var jobRequestA1 = createJob([
+            "insert into ordered_inserts_a values(1)",
+            "select pg_sleep(0.25)",
+            "insert into ordered_inserts_a values(2)"
+        ]);
+        var jobRequestA2 = createJob([
+            "insert into ordered_inserts_a values(3)"
+        ]);
+
+        var self = this;
+
+        this.batchTestClientA.createJob(jobRequestA1, function(err, jobResultA1) {
+            if (err) {
+                return done(err);
+            }
+
+            // we don't care about the producer
+            self.batchTestClientB.createJob(jobRequestA2, function(err, jobResultA2) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResultA1.getStatus(function (err, jobA1) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    jobResultA2.getStatus(function(err, jobA2) {
+                        if (err) {
+                            return done(err);
+                        }
+                        assert.equal(jobA1.status, JobStatus.DONE);
+                        assert.equal(jobA2.status, JobStatus.DONE);
+
+                        assert.ok(
+                                new Date(jobA1.updated_at).getTime() < new Date(jobA2.updated_at).getTime(),
+                                'A1 (' + jobA1.updated_at + ') ' +
+                                'should finish before A2 (' + jobA2.updated_at + ')'
+                        );
+
+                        function statusMapper (status) { return { status: status }; }
+
+                        self.testClient.getResult('select * from ordered_inserts_a', function(err, rows) {
+                            assert.ok(!err);
+
+                            // cartodb250user and vizzuality test users share database
+                            var expectedRows = [1, 2, 3].map(statusMapper);
+                            assert.deepEqual(rows, expectedRows);
+
+                            return done();
+                        });
+                    });
+                });
+            });
+        });
+    });
+});

test/acceptance/batch/use-cases.test.js

@@ -0,0 +1,190 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var JobStatus = require('../../../batch/job_status');
+var BatchTestClient = require('../../support/batch-test-client');
+
+describe('Use cases', function () {
+    before(function() {
+        this.batchTestClient = new BatchTestClient();
+    });
+
+    after(function(done) {
+        this.batchTestClient.drain(done);
+    });
+
+    it('cancel a done job should return an error', function (done) {
+        var payload = {
+            query: "SELECT * FROM untitle_table_4"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.DONE, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.status, JobStatus.DONE);
+
+                jobResult.tryCancel(function (err, body) {
+                    assert.equal(body.error[0], "Cannot set status from done to cancelled");
+                    done();
+                });
+            });
+        });
+    });
+
+    it('cancel a running job', function (done) {
+        var payload = {
+            query: "SELECT * FROM untitle_table_4; select pg_sleep(3)"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.status, JobStatus.RUNNING);
+
+                jobResult.cancel(function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.CANCELLED);
+
+                    jobResult.tryCancel(function (err, body) {
+                        assert.equal(body.error[0], "Cannot set status from cancelled to cancelled");
+                        done();
+                    });
+                });
+            });
+        });
+    });
+
+    it('cancel a pending job', function (done) {
+        var self = this;
+        var payload1 = {
+            query: "SELECT * FROM untitle_table_4; select pg_sleep(3)"
+        };
+
+        this.batchTestClient.createJob(payload1, function(err, jobResult1) {
+            if (err) {
+                return done(err);
+            }
+
+            var payload2 = {
+                query: "SELECT * FROM untitle_table_4"
+            };
+
+            self.batchTestClient.createJob(payload2, function(err, jobResult2) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobResult2.getStatus(JobStatus.PENDING, function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.PENDING);
+
+                    jobResult2.cancel(function (err, job) {
+                        if (err) {
+                            return done(err);
+                        }
+
+                        assert.equal(job.status, JobStatus.CANCELLED);
+
+                        jobResult1.cancel(function (err, job) {
+                            if (err) {
+                                return done(err);
+                            }
+
+                            assert.equal(job.status, JobStatus.CANCELLED);
+                            done();
+                        });
+
+                    });
+                });
+            });
+        });
+    });
+
+    it('cancel a job with quotes', function (done) {
+        var payload = {
+            query: "SELECT name FROM untitle_table_4 WHERE name = 'Hawai'; select pg_sleep(3)"
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.status, JobStatus.RUNNING);
+
+                jobResult.cancel(function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.CANCELLED);
+                    done();
+                });
+            });
+        });
+    });
+
+    it('cancel a running multiquery job', function (done) {
+        var payload = {
+            query: [
+                "select pg_sleep(1)",
+                "select pg_sleep(1)",
+                "select pg_sleep(1)"
+            ]
+        };
+
+        this.batchTestClient.createJob(payload, function(err, jobResult) {
+            if (err) {
+                return done(err);
+            }
+
+            jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.status, JobStatus.RUNNING);
+
+                jobResult.cancel(function (err, job) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(job.status, JobStatus.CANCELLED);
+
+                    jobResult.tryCancel(function (err, body) {
+                        assert.equal(body.error[0], "Cannot set status from cancelled to cancelled");
+                        done();
+                    });
+                });
+            });
+        });
+    });
+});

test/acceptance/cache.js

@@ -0,0 +1,21 @@
+'use strict';
+
+var server = require('../../app/server')();
+const assert = require('../support/assert');
+
+describe('Cache', function () {
+    it('should return a Vary header', function (done) {
+        assert.response(server, {
+            url: '/api/v1/sql?api_key=1234&g=select%20*%20from%20untitle_table_4',
+            headers: {
+                host: 'vizzuality.cartodb.com'
+            },
+            method: 'GET'
+        },
+        {},
+        function(err, res) {
+            assert.equal(res.headers.vary, 'Authorization');
+            done();
+        });
+    });
+});

test/acceptance/copy-abort.js

@@ -0,0 +1,242 @@
+'use strict';
+
+const querystring = require('querystring');
+const StatsClient = require('../../app/stats/client');
+const statsClient = StatsClient.getInstance(global.settings.statsd);
+const server = require('../../app/server')(statsClient);
+const request = require('request');
+const assert = require('assert');
+
+const copyQuery = `COPY (
+    INSERT INTO copy_to_test
+    SELECT updated_at
+    FROM generate_series(
+        '1984-06-14 01:00:00'::timestamp,
+        '2018-06-14 01:00:00'::timestamp,
+        '1 hour'::interval
+    ) updated_at
+    RETURNING updated_at
+) TO STDOUT`;
+
+const createTableQuery = `CREATE TABLE copy_to_test AS
+    (SELECT '2018-06-15 14:49:05.126415+00'::timestamp AS updated_at)`;
+
+const dropTableQuery = `DROP TABLE copy_to_test`;
+
+const countQuery = `SELECT count(1) as count FROM copy_to_test`;
+
+function countInsertedRows (host, port, callback) {
+    setTimeout(function () {
+        const count = querystring.stringify({ q: countQuery, api_key: 1234 });
+
+        const options = {
+            url: `http://${host}:${port}/api/v1/sql?${count}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        request(options, function (err, res, body) {
+            if (err) {
+                return callback(err);
+            }
+
+            assert.equal(res.statusCode, 200);
+            const result = JSON.parse(body);
+            callback(null, result);
+        });
+    }, 100);
+}
+
+describe('Cancel "copy to" commands', function () {
+    before(function() {
+        this.db_pool_size = global.settings.db_pool_size;
+        global.settings.db_pool_size = 1;
+    });
+
+    after(function() {
+        global.settings.db_pool_size = this.db_pool_size;
+    });
+
+
+    beforeEach(function (done) {
+        this.listener = server.listen(0, '127.0.0.1');
+
+        this.listener.on('error', done);
+
+        this.listener.on('listening', () => {
+            const { address, port } = this.listener.address();
+
+            this.host = address;
+            this.port = port;
+
+            done();
+        });
+    });
+
+    beforeEach(function (done) {
+        const { host, port } = this;
+
+        const createTable = querystring.stringify({ q: createTableQuery, api_key: 1234});
+
+        const createTableOptions = {
+            url: `http://${host}:${port}/api/v1/sql?${createTable}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        request(createTableOptions, function (err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.equal(res.statusCode, 200);
+
+            done();
+        });
+    });
+
+    afterEach(function (done) {
+        const { host, port } = this;
+
+        const dropTable = querystring.stringify({ q: dropTableQuery, api_key: 1234 });
+
+        const dropTableOptions = {
+            url: `http://${host}:${port}/api/v1/sql?${dropTable}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        request(dropTableOptions, function (err, res) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.equal(res.statusCode, 200);
+
+            done();
+        });
+    });
+
+    afterEach(function (done) {
+        this.listener.close(done);
+    });
+
+    it('abort on response', function (done) {
+        const { host, port } = this;
+
+        const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
+
+        const options = {
+            url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        const req = request(options);
+
+        req.on('response', function () {
+            req.abort();
+
+            countInsertedRows(host, port, function (err, result) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(result.rows[0].count, 1);
+
+                done();
+            });
+        });
+    });
+
+    it('abort on data', function (done) {
+        const { host, port } = this;
+
+        const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
+
+        const options = {
+            url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        const req = request(options);
+
+        req.once('data', function () {
+            req.abort();
+
+            countInsertedRows(host, port, function (err, result) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(result.rows[0].count, 1);
+
+                done();
+            });
+        });
+    });
+
+
+    it('destroy on data', function (done) {
+        const { host, port } = this;
+
+        const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
+
+        const options = {
+            url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        const req = request(options);
+
+        let response;
+
+        req.on('response', function (res) {
+            response = res;
+        });
+
+        req.once('data', function () {
+            response.destroy();
+
+            countInsertedRows(host, port, function (err, result) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(result.rows[0].count, 1);
+
+                done();
+            });
+        });
+    });
+
+    it('destroy on response', function (done) {
+        const { host, port } = this;
+
+        const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
+
+        const options = {
+            url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        };
+
+        const req = request(options);
+
+        req.on('response', function (response) {
+            response.destroy();
+
+            countInsertedRows(host, port, function (err, result) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(result.rows[0].count, 1);
+
+                done();
+            });
+        });
+    });
+});

test/acceptance/copy-endpoints.js

@@ -0,0 +1,604 @@
+'use strict';
+
+require('../helper');
+
+const fs = require('fs');
+const querystring = require('querystring');
+const assert = require('../support/assert');
+const os = require('os');
+const { Client } = require('pg');
+const request = require('request');
+
+const StatsClient = require('../../app/stats/client');
+if (global.settings.statsd) {
+    // Perform keyword substitution in statsd
+    if (global.settings.statsd.prefix) {
+        const hostToken = os.hostname().split('.').reverse().join('.');
+        global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, hostToken);
+    }
+}
+const statsClient = StatsClient.getInstance(global.settings.statsd);
+const server = require('../../app/server')(statsClient);
+
+
+// Give it enough time to connect and issue the query
+// but not too much so as to disconnect in the middle of the query.
+const CLIENT_DISCONNECT_TIMEOUT = 100;
+const assertCanReuseCanceledConnection = function (done) {
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+            q: 'SELECT count(*) FROM copy_endpoints_test',
+        }),
+        headers: { host: 'vizzuality.cartodb.com' },
+        method: 'GET'
+    }, {}, function(err, res) {
+        assert.ifError(err);
+        assert.ok(res.statusCode === 200);
+        const result = JSON.parse(res.body);
+        assert.strictEqual(result.rows[0].count, 0);
+        done();
+    });
+};
+
+
+describe('copy-endpoints', function() {
+    before(function() {
+        this.client = new Client({
+            user: 'postgres',
+            host: 'localhost',
+            database: 'cartodb_test_user_1_db',
+            port: 5432,
+        });
+        this.client.connect();
+    });
+
+    after(function() {
+        this.client.end();
+    });
+
+    afterEach(function (done) {
+        this.client.query('TRUNCATE copy_endpoints_test', err => {
+            done(err);
+        });
+    });
+
+    describe('general', function() {
+        it('should work with copyfrom endpoint', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                const response = JSON.parse(res.body);
+                assert.equal(!!response.time, true);
+                assert.strictEqual(response.total_rows, 6);
+                done();
+            });
+        });
+
+        it('should fail with copyfrom endpoint and unexisting table', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY unexisting_table (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(
+                    JSON.parse(res.body),
+                    {
+                        error:['relation \"unexisting_table\" does not exist']
+                    }
+                );
+                done();
+            });
+        });
+
+        it('should fail with copyfrom endpoint and without csv', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(
+                    JSON.parse(res.body),
+                    {
+                        error:['No rows copied']
+                    }
+                );
+                done();
+            });
+        });
+
+        it('should fail with copyfrom endpoint and without q', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom",
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(
+                    JSON.parse(res.body),
+                    {
+                        error:["SQL is missing"]
+                    }
+                );
+                done();
+            });
+        });
+
+        it('should work with copyto endpoint', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },{}, function(err) {
+                assert.ifError(err);
+
+                assert.response(server, {
+                    url: "/api/v1/sql/copyto?" + querystring.stringify({
+                        q: 'COPY copy_endpoints_test TO STDOUT',
+                        filename: '/tmp/output.dmp'
+                    }),
+                    headers: {host: 'vizzuality.cartodb.com'},
+                    method: 'GET'
+                },{}, function(err, res) {
+                    assert.ifError(err);
+                    assert.strictEqual(
+                        res.body,
+                        '11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
+                    );
+
+                    assert.equal(res.headers['content-disposition'], 'attachment; filename=%2Ftmp%2Foutput.dmp');
+                    assert.equal(res.headers['content-type'], 'application/octet-stream');
+
+                    done();
+                });
+            });
+        });
+
+        it('should fail with copyto endpoint and without sql', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyto?" + querystring.stringify({
+                    filename: '/tmp/output.dmp'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(
+                    JSON.parse(res.body),
+                    {
+                        error:["SQL is missing"]
+                    }
+                );
+                done();
+            });
+        });
+
+        it('should work with copyfrom and gzip', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv.gz'),
+                headers: {
+                    host: 'vizzuality.cartodb.com',
+                    'content-encoding': 'gzip'
+                },
+                method: 'POST'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                const response = JSON.parse(res.body);
+                assert.equal(!!response.time, true);
+                assert.strictEqual(response.total_rows, 6);
+                done();
+            });
+        });
+
+        it('should return an error when gzip headers are not correct', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {
+                    host: 'vizzuality.cartodb.com',
+                    'content-encoding': 'gzip'
+                },
+                method: 'POST'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(
+                    JSON.parse(res.body),
+                    {
+                        error:["Error while gunzipping: incorrect header check"]
+                    }
+                );
+                done();
+            });
+        });
+    });
+
+
+    describe('timeout', function() {
+        before('set a 10 ms timeout', function() {
+            this.previous_timeout = global.settings.copy_timeout;
+            global.settings.copy_timeout = 10;
+        });
+
+        after('restore previous timeout', function() {
+            global.settings.copy_timeout = this.previous_timeout;
+        });
+
+        it('should fail with copyfrom and timeout', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: `COPY copy_endpoints_test (id, name)
+                    FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },
+            {
+                status: 429,
+                headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            },
+            function(err, res) {
+                assert.ifError(err);
+                assert.deepEqual(JSON.parse(res.body), {
+                    error: [
+                        'You are over platform\'s limits: SQL query timeout error.' +
+                            ' Refactor your query before running again or contact CARTO support for more details.',
+                    ],
+                    context: 'limit',
+                    detail: 'datasource'
+                });
+                done();
+            });
+        });
+
+        it('should fail with copyto and timeout', function(done){
+            assert.response(server, {
+                url: "/api/v1/sql/copyto?" + querystring.stringify({
+                    q: 'COPY populated_places_simple_reduced TO STDOUT',
+                    filename: '/tmp/output.dmp'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },{}, function(err, res) {
+                assert.ifError(err);
+                const error = {
+                    error: ['You are over platform\'s limits: SQL query timeout error.' +
+                            ' Refactor your query before running again or contact CARTO support for more details.',],
+                    context:"limit",
+                    detail:"datasource"
+                };
+                const expectedError = res.body.substring(res.body.length - JSON.stringify(error).length);
+                assert.deepEqual(JSON.parse(expectedError), error);
+                done();
+            });
+        });
+    });
+
+
+    describe('db connections', function() {
+        before(function() {
+            this.db_pool_size = global.settings.db_pool_size;
+            global.settings.db_pool_size = 1;
+        });
+
+        after(function() {
+            global.settings.db_pool_size = this.db_pool_size;
+        });
+
+        it('copyfrom', function(done) {
+            function doCopyFrom() {
+                return new Promise(resolve => {
+                    assert.response(server, {
+                        url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                            q: `COPY copy_endpoints_test (id, name)
+                                FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
+                        }),
+                        data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                        headers: {host: 'vizzuality.cartodb.com'},
+                        method: 'POST'
+                    },{}, function(err, res) {
+                        assert.ifError(err);
+                        const response = JSON.parse(res.body);
+                        assert.ok(response.time);
+                        resolve();
+                    });
+                });
+            }
+
+            Promise.all([doCopyFrom(), doCopyFrom(), doCopyFrom()]).then(function() {
+                done();
+            });
+        });
+
+        it('copyto', function(done) {
+            function doCopyTo() {
+                return new Promise(resolve => {
+                    assert.response(server, {
+                        url: "/api/v1/sql/copyto?" + querystring.stringify({
+                            q: `COPY (SELECT * FROM generate_series(1, 10000)) TO STDOUT`,
+                            filename: '/tmp/output.dmp'
+                        }),
+                        headers: {host: 'vizzuality.cartodb.com'},
+                        method: 'GET'
+                    },{}, function(err, res) {
+                        assert.ifError(err);
+                        assert.ok(res.body);
+                        resolve();
+                    });
+                });
+            }
+
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            },{}, function(err) {
+                assert.ifError(err);
+
+                Promise.all([doCopyTo(), doCopyTo(), doCopyTo()]).then(function() {
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('client disconnection', function() {
+        before(function() {
+            this.db_pool_size = global.settings.db_pool_size;
+            global.settings.db_pool_size = 1;
+        });
+
+        after(function() {
+            global.settings.db_pool_size = this.db_pool_size;
+        });
+
+        const assertCanReuseConnection = function (done) {
+            assert.response(server, {
+                url: '/api/v1/sql?' + querystring.stringify({
+                    q: 'SELECT 1',
+                }),
+                headers: { host: 'vizzuality.cartodb.com' },
+                method: 'GET'
+            }, {}, function(err, res) {
+                assert.ifError(err);
+                assert.ok(res.statusCode === 200);
+                done();
+            });
+        };
+
+        it('COPY TO returns the connection to the pool if the client disconnects', function(done) {
+            const listener = server.listen(0, '127.0.0.1');
+
+            listener.on('error', done);
+            listener.on('listening', function onServerListening () {
+
+                const { address, port } = listener.address();
+                const query = querystring.stringify({
+                    q: `COPY (SELECT * FROM generate_series(1, 1000)) TO STDOUT`
+                });
+
+                const options = {
+                    url: `http://${address}:${port}/api/v1/sql/copyto?${query}`,
+                    headers: { host: 'vizzuality.cartodb.com' },
+                    method: 'GET'
+                };
+
+                const req = request(options);
+
+                req.once('data', () => req.abort());
+                req.on('response', response => {
+                    response.on('end', () => {
+                        assertCanReuseConnection(done);
+                    });
+                });
+            });
+        });
+
+        it('COPY FROM returns the connection to the pool if the client disconnects', function(done) {
+            const listener = server.listen(0, '127.0.0.1');
+
+            listener.on('error', done);
+            listener.on('listening', function onServerListening () {
+
+                const { address, port } = listener.address();
+                const query = querystring.stringify({
+                    q: `COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
+                });
+
+                const options = {
+                    url: `http://${address}:${port}/api/v1/sql/copyfrom?${query}`,
+                    headers: { host: 'vizzuality.cartodb.com' },
+                    method: 'POST',
+                    data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv')
+                };
+
+                const req = request(options);
+
+                setTimeout(() => {
+                    req.abort();
+                    assertCanReuseCanceledConnection(done);
+                }, CLIENT_DISCONNECT_TIMEOUT);
+            });
+        });
+
+    });
+
+    describe('COPY timeouts: they can take longer than statement_timeout', function() {
+        before('set a very small statement_timeout for regular queries', function(done) {
+            assert.response(server, {
+                url: '/api/v1/sql?q=set statement_timeout = 10',
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            }, done);
+        });
+
+        after('restore normal statement_timeout for regular queries', function(done) {
+            assert.response(server, {
+                url: '/api/v1/sql?q=set statement_timeout = 2000',
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            }, done);
+        });
+
+        it('COPY FROM can take longer than regular statement_timeout', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: `COPY copy_endpoints_test (id, name)
+                    FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            }, {
+                status: 200,
+                headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            }, function(err, res) {
+                assert.ifError(err);
+                const response = JSON.parse(res.body);
+                assert.strictEqual(response.total_rows, 6);
+                done();
+            });
+        });
+
+        it('COPY TO can take longer than regular statement_timeout', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyto?" + querystring.stringify({
+                    q: 'COPY copy_endpoints_test TO STDOUT',
+                    filename: '/tmp/output.dmp'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            }, {}, function(err, res) {
+                assert.ifError(err);
+                assert.ok(res.statusCode === 200);
+                done();
+            });
+        });
+    });
+
+    describe('dbQuotaMiddleware', function() {
+        before('Set the remaining quota to 1 byte', function(done) {
+            // See the test/support/sql/quota_mock.sql
+            this.client.query(`CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
+                              RETURNS bigint AS
+                              $$
+                              BEGIN
+                                RETURN 250 * 1024 * 1024 - 1;
+                              END;
+                              $$ LANGUAGE 'plpgsql' VOLATILE;
+                              `, err => done(err));
+
+            this.db_pool_size = global.settings.db_pool_size;
+            global.settings.db_pool_size = 1;
+        });
+
+        after('Restore the old quota', function(done) {
+            // See the test/support/sql/quota_mock.sql
+            this.client.query(`CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
+                              RETURNS bigint AS
+                              $$
+                              BEGIN
+                                RETURN 200 * 1024 * 1024;
+                              END;
+                              $$ LANGUAGE 'plpgsql' VOLATILE;
+                              `, err => done(err));
+
+            global.settings.db_pool_size = this.db_pool_size;
+        });
+
+        it('COPY FROM fails with an error if DB quota is exhausted', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: `COPY copy_endpoints_test (id, name)
+                    FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            }, {
+                status: 400,
+                headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            }, function(err, res) {
+                const response = JSON.parse(res.body);
+                assert.deepEqual(response, { error: ["DB Quota exceeded"] });
+
+                setTimeout(() => assertCanReuseCanceledConnection(done), CLIENT_DISCONNECT_TIMEOUT);
+            });
+        });
+
+        it('COPY TO is not affected by remaining DB quota', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyto?" + querystring.stringify({
+                    q: 'COPY copy_endpoints_test TO STDOUT',
+                    filename: '/tmp/output.dmp'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            }, {}, function(err, res) {
+                assert.ifError(err);
+                assert.ok(res.statusCode === 200);
+                done();
+            });
+        });
+    });
+
+    describe('COPY FROM max POST size', function() {
+        before('Set a ridiculously small POST size limit', function() {
+            this.previous_max_post_size = global.settings.copy_from_max_post_size;
+            this.previous_max_post_size_pretty = global.settings.copy_from_max_post_size_pretty;
+            global.settings.copy_from_max_post_size = 10;
+            global.settings.copy_from_max_post_size_pretty = '10 bytes';
+            this.db_pool_size = global.settings.db_pool_size;
+            global.settings.db_pool_size = 1;
+        });
+        after('Restore the max POST size limit values', function() {
+            global.settings.copy_from_max_post_size = this.previous_max_post_size;
+            global.settings.copy_from_max_post_size_pretty = this.previous_max_post_size_pretty;
+            global.settings.db_pool_size = this.db_pool_size;
+        });
+
+        it('honors the max POST size limit', function(done) {
+            assert.response(server, {
+                url: "/api/v1/sql/copyfrom?" + querystring.stringify({
+                    q: `COPY copy_endpoints_test (id, name)
+                    FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
+                }),
+                data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'POST'
+            }, {
+                status: 400,
+                headers: { 'Content-Type': 'application/json; charset=utf-8' }
+            }, function(err, res) {
+                const response = JSON.parse(res.body);
+                assert.deepEqual(response, { error: ["COPY FROM maximum POST size of 10 bytes exceeded"] });
+
+                setTimeout(() => assertCanReuseCanceledConnection(done), CLIENT_DISCONNECT_TIMEOUT);
+            });
+        });
+    });
+});

test/acceptance/copy-statements.js

@@ -0,0 +1,80 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+
+
+describe('copy-statements', function() {
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    before(function(done) {
+        assert.response(server, {
+            url: "/api/v1/sql?" + querystring.stringify({
+                q: 'CREATE TABLE copy_test_table(a int)',
+                api_key: 1234
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, done);
+    });
+
+    after(function(done) {
+        assert.response(server, {
+            url: "/api/v1/sql?" + querystring.stringify({
+                q: 'DROP TABLE IF EXISTS copy_test_table',
+                api_key: 1234
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, done);
+    });
+
+    // Test effects of COPY
+    // See https://github.com/Vizzuality/cartodb-management/issues/1502
+    it('COPY TABLE with GET and auth', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql?" + querystring.stringify({
+                q: 'COPY copy_test_table FROM stdin;',
+                api_key: 1234
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{}, function(err, res) {
+        // We expect a problem, actually
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.deepEqual(JSON.parse(res.body), {"error":["COPY from stdin failed: No source stream defined"]});
+        done();
+        });
+    });
+
+    it('COPY TABLE with GET and auth', function(done){
+        assert.response(server, {
+            url: "/api/v1/sql?" + querystring.stringify({
+                q: "COPY copy_test_table to '/tmp/x';",
+                api_key: 1234
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{}, function(err, res) {
+        // We expect a problem, actually
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        const error_exp = /must be superuser.* to COPY.* a file/;
+        const hint_exp = /Anyone can COPY to stdout or from stdin. psql's \\copy command also works for anyone./;
+        assert.ok(JSON.parse(res.body).error[0].match(error_exp));
+        assert.ok(JSON.parse(res.body).hint.match(hint_exp));
+        done();
+        });
+    });
+
+
+});

test/acceptance/error-handler.js

@@ -0,0 +1,33 @@
+'use strict';
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+
+describe('error handler', function () {
+    it('should returns a errors header', function (done) {
+        const errorHeader = {
+            detail: undefined,
+            hint: undefined,
+            context: undefined,
+            statusCode: 400,
+            message: 'You must indicate a sql query'
+        };
+
+        assert.response(server, {
+            url: '/api/v1/sql',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        {
+            status: 400,
+            headers: {
+                'Content-Type': 'application/json; charset=utf-8',
+                'X-SQLAPI-Errors': JSON.stringify(errorHeader)
+            }
+        },
+        function(err){
+            assert.ifError(err);
+            done();
+        });
+    });
+});

test/acceptance/export/arraybuffer.js

@@ -0,0 +1,45 @@
+'use strict';
+
+require('../../helper');
+require('../../support/assert');
+
+
+var server = require('../../../app/server')();
+var assert = require('assert');
+var querystring = require('querystring');
+
+describe('export.arraybuffer', function() {
+
+it('GET /api/v1/sql as arraybuffer ', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT cartodb_id,name,1::integer,187.9 FROM untitle_table_4',
+          format: 'arraybuffer'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        assert.equal(res.headers['content-type'], "application/octet-stream");
+        done();
+    });
+});
+
+it('GET /api/v1/sql as arraybuffer does not support geometry types ', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT cartodb_id, the_geom FROM untitle_table_4',
+          format: 'arraybuffer'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 400, res.body);
+        var result = JSON.parse(res.body);
+        assert.equal(result.error[0], "geometry types are not supported");
+
+        done();
+    });
+});
+
+});

test/acceptance/export/csv.js

@@ -0,0 +1,209 @@
+'use strict';
+
+require('../../helper');
+require('../../support/assert');
+
+
+var server = require('../../../app/server')();
+var assert = require('assert');
+var querystring = require('querystring');
+
+describe('export.csv', function() {
+
+it('CSV format', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT * FROM untitle_table_4 WHERE cartodb_id = 1',
+          format: 'csv'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
+        var ct = res.headers['content-type'];
+        assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
+
+        var rows = res.body.split(/\r\n/);
+        var row0 = rows[0].split(',');
+        var row1 = rows[1].split(',');
+
+        assert.equal(row0[2], 'created_at');
+        assert.equal(row1[2], '2011-09-21 14:02:21.314252');
+
+        done();
+    });
+});
+
+it('CSV format, bigger than 81920 bytes', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({
+          q: 'SELECT 0 as fname FROM generate_series(0,81920)',
+          format: 'csv'
+        }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.ok(res.body.length > 81920, 'CSV smaller than expected: ' + res.body.length);
+        done();
+    });
+});
+
+
+it('CSV format from POST', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({q: "SELECT * FROM untitle_table_4 LIMIT 1", format: 'csv'}),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
+        var ct = res.headers['content-type'];
+        assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
+        done();
+    });
+});
+
+it('CSV format, custom filename', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&filename=mycsv.csv',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=mycsv.csv/gi.test(cd), cd);
+        var ct = res.headers['content-type'];
+        assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
+        var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
+        var checkFields = { name: true, cartodb_id: true, the_geom: true, the_geom_webmercator: true };
+        Object.keys(checkFields).forEach(function(f) {
+            var idx = row0.indexOf(f);
+            if ( checkFields[f] ) {
+                assert.ok(idx !== -1, "result does not include '" + f + "'");
+            } else {
+                assert.ok(idx === -1, "result includes '" + f + "' ("+idx+")");
+            }
+        });
+        done();
+    });
+});
+
+it('skipfields controls fields included in CSV output', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv' +
+            '&skipfields=unexistant,cartodb_id',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
+        var checkFields = { name: true, cartodb_id: false, the_geom: true, the_geom_webmercator: true };
+        Object.keys(checkFields).forEach(function(f) {
+            var idx = row0.indexOf(f);
+            if ( checkFields[f] ) {
+                assert.ok(idx !== -1, "result does not include '" + f + "'");
+            } else {
+                assert.ok(idx === -1, "result includes '" + f + "' ("+idx+")");
+            }
+        });
+        done();
+    });
+});
+
+it('GET /api/v1/sql as csv', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20cartodb_id,ST_AsEWKT(the_geom)%20as%20geom%20FROM%20untitle_table_4%20LIMIT%201' +
+            '&format=csv',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        assert.ok(res.body.match(/cartodb_id,geom\r\n.?1.?,"SRID=4326;POINT(.*)"\r\n/));
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/60
+it('GET /api/v1/sql as csv with no rows', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20true%20WHERE%20false&format=csv',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var obtained_lines = res.body.split('\r\n');
+        assert.ok(obtained_lines.length <= 2, // may or may not have an header
+          // See http://trac.osgeo.org/gdal/ticket/5234
+          'Too many lines in output (' + obtained_lines.length + '): ' + obtained_lines.join('\n'));
+        done();
+    });
+});
+
+it('GET /api/v1/sql as csv, properly escaped', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        assert.ok(res.body.match(/cartodb_id,address\r\n.?1.?,"Calle de Pérez Galdós 9, Madrid, Spain"\r\n/));
+        done();
+    });
+});
+
+it('GET /api/v1/sql as csv, concurrently', function(done){
+
+    var concurrency = 4;
+    var waiting = concurrency;
+    function validate(err, res){
+        assert.ok(res.body.match(/cartodb_id,address\r\n.?1.?,"Calle de Pérez Galdós 9, Madrid, Spain"\r\n/));
+        if ( ! --waiting ) {
+            done();
+        }
+    }
+    for (var i=0; i<concurrency; ++i) {
+        assert.response(server,
+            {
+                url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },
+            {
+                status: 200
+            },
+            validate
+        );
+    }
+});
+
+    it('expects 1200 rows in public table', function(done){
+        var limit = 1200;
+
+        assert.response(server, {
+                url: '/api/v1/sql?' + querystring.stringify({
+                    q: "SELECT * from populated_places_simple_reduced limit " + limit,
+                    format: 'csv'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },
+            {
+                status: 200
+            },
+            function(err, res) {
+                var headersPlusExtraLine = 2;
+                assert.equal(res.body.split('\n').length, limit + headersPlusExtraLine);
+                done();
+            }
+        );
+    });
+
+});

test/acceptance/export/folder.js

@@ -0,0 +1,35 @@
+'use strict';
+
+require('../../helper');
+require('../../support/assert');
+
+const fs = require('fs');
+let server = require('../../../app/server');
+const assert = require('assert');
+const querystring = require('querystring');
+
+describe('export folder', function() {
+    it('folder exists', function(done){
+        const currentTmpDir = global.settings.tmpDir;
+
+        const dynamicTmpDir = `/tmp/${new Date().getTime()}/a/b/c`;
+        global.settings.tmpDir = dynamicTmpDir;
+        server = server();
+
+        assert.response(server, {
+            url: '/api/v1/sql?' + querystring.stringify({
+                q: 'SELECT 1',
+            }),
+            headers: { host: 'vizzuality.cartodb.com' },
+            method: 'GET'
+        }, {}, function(err, res) {
+            assert.ifError(err);
+            assert.ok(res.statusCode === 200);
+            assert.ok(fs.existsSync(dynamicTmpDir));
+
+            global.settings.tmpDir = currentTmpDir;
+
+            done();
+        });
+    });
+});

test/acceptance/export/geojson.js

@@ -0,0 +1,212 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var querystring = require('querystring');
+
+// use dec_sep for internationalization
+var checkDecimals = function(x, dec_sep){
+    var tmp='' + x;
+    if (tmp.indexOf(dec_sep)>-1) {
+        return tmp.length - tmp.indexOf(dec_sep) - 1;
+    } else {
+        return 0;
+    }
+};
+
+describe('export.geojson', function() {
+
+// GEOJSON tests
+
+it('GET /api/v1/sql with SQL parameter, ensuring content-disposition set to geojson', function(done) {
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
+        done();
+    });
+});
+
+it('POST /api/v1/sql with SQL parameter, ensuring content-disposition set to geojson', function(done) {
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({q: "SELECT * FROM untitle_table_4", format: 'geojson' }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
+        done();
+    });
+});
+
+it('uses the last format parameter when multiple are used', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?format=csv&q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
+        done();
+    });
+});
+
+it('uses custom filename', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&filename=x',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /filename=x.geojson/gi.test(cd), cd);
+        done();
+    });
+});
+
+it('does not include the_geom and the_geom_webmercator properties by default', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var parsed_body = JSON.parse(res.body);
+        var row0 = parsed_body.features[0].properties;
+        var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':0, 'the_geom_webmercator':0};
+        for ( var f in checkfields ) {
+          if ( checkfields[f] ) {
+            assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+          } else {
+            assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+          }
+        }
+        done();
+    });
+});
+
+it('skipfields controls fields included in GeoJSON output', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&skipfields=unexistant,cartodb_id',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var parsed_body = JSON.parse(res.body);
+        var row0 = parsed_body.features[0].properties;
+        var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':0, 'the_geom_webmercator':0};
+        for ( var f in checkfields ) {
+          if ( checkfields[f] ) {
+            assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+          } else {
+            assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+          }
+        }
+        done();
+    });
+});
+
+
+it('GET /api/v1/sql as geojson limiting decimal places', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT ST_MakePoint(0.123,2.3456) as the_geom',
+          format: 'geojson',
+          dp: '1'}),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var result = JSON.parse(res.body);
+        assert.equal(1, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
+        done();
+    });
+});
+
+it('GET /api/v1/sql as geojson with default dp as 6', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT ST_MakePoint(0.12345678,2.3456787654) as the_geom',
+          format: 'geojson'}),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var result = JSON.parse(res.body);
+        assert.equal(6, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
+        done();
+    });
+});
+
+it('null geometries in geojson output', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: "SELECT 1 as gid, 'U' as name, null::geometry as the_geom ",
+          format: 'geojson'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
+        var gjson = JSON.parse(res.body);
+        var expected = {
+            type: 'FeatureCollection',
+            features: [ { type: 'Feature',
+                properties: { gid: 1, name: 'U' },
+                geometry: null } ]
+         };
+        assert.deepEqual(gjson, expected);
+        done();
+      });
+});
+
+it('stream response handle errors', function(done) {
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+            q: "SELECTT 1 as gid, null::geometry as the_geom ",
+            format: 'geojson'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 400, res.body);
+        var geoJson = JSON.parse(res.body);
+        assert.ok(geoJson.error);
+        assert.equal(geoJson.error.length, 1);
+        assert.ok(geoJson.error[0].match(/^syntax error at or near.*/));
+        done();
+    });
+});
+
+it('stream response with empty result set has valid output', function(done) {
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+            q: "SELECT 1 as gid, null::geometry as the_geom limit 0",
+            format: 'geojson'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var geoJson = JSON.parse(res.body);
+        var expectedGeoJson = {"type": "FeatureCollection", "features": []};
+        assert.deepEqual(geoJson, expectedGeoJson);
+        done();
+    });
+});
+
+});

test/acceptance/export/geopackage.js

@@ -0,0 +1,82 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var sqlite = require('sqlite3');
+var fs      = require('fs');
+
+describe('geopackage query', function(){
+    // Default name, cartodb-query, fails because of the hyphen.
+    var table_name = 'a_gpkg_table';
+    var base_url = '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=gpkg&filename=' + table_name;
+
+    it('returns a valid geopackage database', function(done){
+        assert.response(server, {
+            url: base_url,
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{ }, function(err, res) {
+            assert.equal(res.statusCode, 200, res.body);
+            assert.equal(res.headers["content-type"], "application/x-sqlite3; charset=utf-8");
+            assert.notEqual(res.headers["content-disposition"].indexOf(table_name + ".gpkg"), -1);
+            var db = new sqlite.Database(':memory:', res.body);
+            var qr = db.get("PRAGMA database_list", function(err) {
+                assert.equal(err, null);
+                done();
+            });
+            assert.notEqual(qr, undefined);
+        });
+
+    });
+
+    it('gets database and geopackage schema', function(done){
+        assert.response(server, {
+            url: base_url,
+            headers: {host: 'vizzuality.cartodb.com'},
+            encoding: 'binary',
+            method: 'GET'
+        },{ }, function(err, res) {
+            var tmpfile = '/tmp/a_geopackage_file.gpkg';
+            try {
+              fs.writeFileSync(tmpfile, res.body, 'binary');
+            } catch(err) {
+              return done(err);
+            }
+
+            var db = new sqlite.Database(tmpfile, function(err) {
+              if(!!err) {
+                return done(err);
+              }
+
+              db.serialize(function() {
+                var schemaQuery = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name";
+                var sqr = db.get(schemaQuery, function(err, row) {
+                  assert.equal(err, null);
+                  assert.equal(row.name, table_name);
+                });
+                assert.notEqual(sqr, undefined);
+
+                var gpkgQuery = "SELECT table_name FROM gpkg_contents";
+                var gqr = db.get(gpkgQuery, function(err, row) {
+                  assert.equal(row.table_name, table_name);
+                  assert.equal(err, null);
+                });
+                assert.notEqual(gqr, undefined);
+
+                var dataQuery = "SELECT * FROM " + table_name + " order by cartodb_id";
+                var dqr = db.get(dataQuery, function(err, row) {
+                  assert.equal(err, null);
+                  assert.equal(row.cartodb_id, 1);
+                  assert.equal(row.name, 'Hawai');
+                  assert.equal(row.fid, undefined);
+                  done();
+                });
+                assert.notEqual(dqr, undefined);
+              });
+            });
+        });
+    });
+
+});

test/acceptance/export/kml.js

@@ -0,0 +1,403 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var querystring = require('querystring');
+var libxmljs = require('libxmljs');
+
+describe('export.kml', function() {
+
+// Check if an attribute is in the KML output
+//
+// NOTE: "name" and "description" attributes are threated specially
+//       in that they are matched in case-insensitive way
+//
+var hasAttribute = function(kml, att) {
+
+  // Strip namespace:
+  //https://github.com/polotek/libxmljs/issues/212
+  kml = kml.replace(/ xmlns=[^>]*>/, '>');
+
+  var doc = libxmljs.parseXmlString(kml);
+  //console.log("doc: " + doc);
+  var xpath;
+
+  xpath = "//SimpleField[@name='" + att + "']";
+  if ( doc.get(xpath) ) {
+      return true;
+  }
+
+  xpath = "//Placemark/" + att;
+  if ( doc.get(xpath) ) {
+      return true;
+  }
+
+  var lcatt = att.toLowerCase();
+  if ( lcatt === 'name' || lcatt === 'description' ) {
+    xpath = "//Placemark/" + lcatt;
+    if ( doc.get(xpath) ) {
+        return true;
+    }
+  }
+
+  //if ( lowerkml.indexOf('simplefield name="'+ loweratt + '"') != -1 ) return true;
+  //if ( lowerkml.indexOf('<'+loweratt+'>') != -1 ) return true;
+  return false;
+};
+
+// Return the first coordinate array found in KML
+var extractCoordinates = function(kml) {
+
+  // Strip namespace:
+  //https://github.com/polotek/libxmljs/issues/212
+  kml = kml.replace(/ xmlns=[^>]*>/, '>');
+
+  var doc = libxmljs.parseXmlString(kml);
+  //console.log("doc: " + doc);
+  if ( ! doc ) {
+      return;
+  }
+  var coo = doc.get("//coordinates");
+  //console.log("coo: " + coo);
+  if ( ! coo ) {
+      return;
+  }
+  coo = coo.text();
+  //console.log("coo: " + coo);
+  if ( ! coo ) {
+      return;
+  }
+  coo = coo.split(' ');
+  //console.log("coo: " + coo);
+  for (var i=0; i<coo.length; ++i) {
+    coo[i] = coo[i].split(',');
+  }
+
+  return coo;
+};
+
+// Return the first folder name in KML
+var extractFolderName = function(kml) {
+
+  // Strip namespace:
+  //https://github.com/polotek/libxmljs/issues/212
+  kml = kml.replace(/ xmlns=[^>]*>/, '>');
+
+  var doc = libxmljs.parseXmlString(kml);
+  //console.log("doc: " + doc);
+  if ( ! doc ) {
+      return;
+  }
+  var coo = doc.get("//Document/Folder/name");
+  //console.log("coo: " + coo);
+  if ( ! coo ) {
+      return;
+  }
+  coo = coo.text();
+  //console.log("coo: " + coo);
+  if ( ! coo ) {
+      return;
+  }
+  return coo;
+};
+
+// KML tests
+
+it('KML format, unauthenticated', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        var row0 = res.body;
+        var checkfields = {'Name':1, 'address':1, 'cartodb_id':1, 'the_geom':0, 'the_geom_webmercator':0};
+        Object.keys(checkfields).forEach(function(f) {
+          if ( checkfields[f] ) {
+            assert.ok(hasAttribute(row0, f), "result does not include '" + f + "': " + row0);
+          } else {
+            assert.ok(!hasAttribute(row0, f), "result includes '" + f + "'");
+          }
+        });
+        done();
+    });
+});
+
+it('KML format, unauthenticated, POST', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml',
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        done();
+    });
+});
+
+it('KML format, bigger than 81920 bytes', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({
+          q: 'SELECT 0 as fname FROM generate_series(0,81920)',
+          format: 'kml'
+        }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        assert.ok(res.body.length > 81920, 'KML smaller than expected: ' + res.body.length);
+        done();
+    });
+});
+
+it('KML format, skipfields', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        var row0 = res.body;
+        var checkFields = {'Name':1, 'address':0, 'cartodb_id':0, 'the_geom':0, 'the_geom_webmercator':0};
+        Object.keys(checkFields).forEach(function(f) {
+          if ( checkFields[f] ) {
+            assert.ok(hasAttribute(row0, f), "result does not include '" + f + "': " + row0);
+          } else {
+            assert.ok(!hasAttribute(row0, f), "result includes '" + f + "'");
+          }
+        });
+        done();
+    });
+});
+
+it('KML format, unauthenticated, custom filename', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&filename=kmltest',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=kmltest.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        var name = extractFolderName(res.body);
+        assert.equal(name, "kmltest");
+        done();
+    });
+});
+
+it('KML format, authenticated', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&api_key=1234',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        done();
+    });
+});
+
+it('KML format, unauthenticated, concurrent requests', function(done){
+    var query = querystring.stringify({
+        q: "SELECT 'val', x, y, st_setsrid(st_makepoint(x,y),4326) as the_geom " +
+            "FROM generate_series(-180, 180) as x, generate_series(-90,90) y",
+        format: 'kml',
+        filename: 'multi'
+      });
+
+    var concurrency = 4;
+    var waiting = concurrency;
+
+    function validate(err, res) {
+        //console.log("Response ended");
+        assert.equal(res.statusCode, 200, res.body);
+        assert.ok(res.body);
+        var snippet = res.body.substr(0, 5);
+        assert.equal(snippet, "<?xml");
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=multi.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
+        if ( ! --waiting ) {
+            done();
+        }
+    }
+
+    var request = {
+        method: 'GET',
+        headers: { host: 'vizzuality.cartodb.com' },
+        url: '/api/v1/sql?' + query
+    };
+
+    for (var i=0; i<concurrency; ++i) {
+        //console.log("Sending request");
+        assert.response(server, request, { status: 200 }, validate);
+    }
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/60
+it('GET /api/v1/sql as kml with no rows', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20true%20WHERE%20false&format=kml',
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        // NOTE: GDAL-1.11+ added 'id="root_doc"' attribute to the output
+        var pat = new RegExp('^<\\?xml version="1.0" encoding="utf-8" \\?>' +
+            '<kml xmlns="http://www.opengis.net/kml/2.2">' +
+            '<Document( id="root_doc")?><Folder><name>cartodb_query</name></Folder></Document>' +
+            '</kml>$');
+        var body = res.body.replace(/\n/g,'');
+        assert.ok(body.match(pat),
+          "Response:\n" + body + '\ndoes not match pattern:\n' + pat);
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/90
+it('GET /api/v1/sql as kml with ending semicolon', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT true WHERE false;',
+          format: 'kml'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        // NOTE: GDAL-1.11+ added 'id="root_doc"' attribute to the output
+        var pat = new RegExp('^<\\?xml version="1.0" encoding="utf-8" \\?>' +
+            '<kml xmlns="http://www.opengis.net/kml/2.2">' +
+            '<Document( id="root_doc")?><Folder><name>cartodb_query</name></Folder></Document>' +
+            '</kml>$');
+        var body = res.body.replace(/\n/g,'');
+        assert.ok(body.match(pat),
+          "Response:\n" + body + '\ndoes not match pattern:\n' + pat);
+        done();
+    });
+});
+
+// See https://github.com/CartoDB/cartodb/issues/276
+it('check point coordinates, unauthenticated', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT * from untitle_table_4 WHERE cartodb_id = -1',
+          format: 'kml'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var coords = extractCoordinates(res.body);
+        assert(coords, 'No coordinates in ' + res.body);
+        assert.deepEqual(coords, [[33,16]]);
+        done();
+    });
+});
+
+// See https://github.com/CartoDB/cartodb/issues/276
+it('check point coordinates, authenticated', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT * from untitle_table_4 WHERE cartodb_id = -1',
+          api_key: 1234,
+          format: 'kml'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var coords = extractCoordinates(res.body);
+        assert(coords, 'No coordinates in ' + res.body);
+        assert.deepEqual(coords, [[33,16]]);
+        done();
+    });
+});
+
+
+    var limit = 1200;
+
+    it('expects ' + limit + ' placemarks in public table', function(done){
+
+        assert.response(server, {
+                url: '/api/v1/sql',
+                data: querystring.stringify({
+                    q: "SELECT * from populated_places_simple_reduced limit " + limit,
+                    format: 'kml'
+                }),
+                headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+                method: 'POST'
+            },
+            {
+                status: 200
+            },
+            function(err, res) {
+                assert.equal(res.body.match(/<Placemark>/g).length, limit);
+                done();
+            }
+        );
+    });
+
+    it('expects ' + limit + ' placemarks in private table using the API KEY', function(done){
+
+        assert.response(server, {
+                url: '/api/v1/sql?' + querystring.stringify({
+                    q: "SELECT * from populated_places_simple_reduced limit " + limit,
+                    api_key: 1234,
+                    format: 'kml'
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },
+            {
+                status: 200
+            },
+            function(err, res) {
+                assert.equal(res.body.match(/<Placemark>/g).length, limit);
+                done();
+            }
+        );
+    });
+
+    it('should work with queries returning no results', function(done) {
+        assert.response(
+            server,
+            {
+                url: "/api/v1/sql?" + querystring.stringify({
+                    q: "SELECT * FROM populated_places_simple_reduced LIMIT 0",
+                    format: 'kml'
+                }),
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                encoding: 'binary',
+                method: 'GET'
+            },
+            {
+                status: 200
+            },
+            function(err, res) {
+                assert.equal(res.body.match(/<Placemark>/g), null);
+                done();
+            }
+        );
+    });
+
+});

test/acceptance/export/shapefile.js

@@ -0,0 +1,405 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var querystring = require('querystring');
+var shapefile = require('shapefile');
+var _ = require('underscore');
+var zipfile = require('zipfile');
+var fs      = require('fs');
+
+describe('export.shapefile', function() {
+
+// SHP tests
+
+it('SHP format, unauthenticated', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+        // TODO: check DBF contents
+        fs.unlinkSync(tmpfile);
+        done();
+    });
+});
+
+it('SHP format, unauthenticated, POST', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
+        done();
+    });
+});
+
+it('SHP format, big size, POST', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: querystring.stringify({
+          q: 'SELECT 0 as fname, st_makepoint(i,i) FROM generate_series(0,81920) i',
+          format: 'shp'
+        }),
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
+        assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length);
+        done();
+    });
+});
+
+it('SHP format, unauthenticated, with custom filename', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=myshape.zip/gi.test(cd));
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        assert.ok(_.contains(zf.names, 'myshape.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'myshape.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'myshape.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'myshape.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+        fs.unlinkSync(tmpfile);
+        done();
+    });
+});
+
+it('SHP format, unauthenticated, with custom, dangerous filename', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=b;"%20()[]a',
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var fname = "b_______a";
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=b_______a.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        assert.ok(_.contains(zf.names, fname + '.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+        assert.ok(_.contains(zf.names, fname + '.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+        assert.ok(_.contains(zf.names, fname + '.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+        assert.ok(_.contains(zf.names, fname+ '.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+        fs.unlinkSync(tmpfile);
+        done();
+    });
+});
+
+it('SHP format, authenticated', function(done){
+    assert.response(server, {
+        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&api_key=1234',
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+        // TODO: check contents of the DBF
+        fs.unlinkSync(tmpfile);
+        done();
+    });
+});
+
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66
+it('SHP format, unauthenticated, with utf8 data', function(done){
+    var query = querystring.stringify({
+        q: "SELECT '♥♦♣♠' as f, st_makepoint(0,0,4326) as the_geom",
+        format: 'shp',
+        filename: 'myshape'
+      });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        var buffer = zf.readFileSync('myshape.dbf');
+        fs.unlinkSync(tmpfile);
+        var strings = buffer.toString();
+        assert.ok(/♥♦♣♠/.exec(strings), "Cannot find '♥♦♣♠' in here:\n" + strings);
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66
+it('mixed type geometry', function(done){
+    var query = querystring.stringify({
+        q: "SELECT 'POINT(0 0)'::geometry as g UNION ALL SELECT 'LINESTRING(0 0, 1 0)'::geometry",
+        format: 'shp'
+      });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body);
+        var parsedBody = JSON.parse(res.body);
+        var error = parsedBody.error[0];
+        var expectedError = /Attempt to write non-point \(LINESTRING\) geometry to point shapefile/g;
+        assert.ok(expectedError.test(error), error);
+        done();
+    });
+});
+
+// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/87
+it('errors are not confused with warnings', function(done){
+    var query = querystring.stringify({
+        q: [
+            "SELECT 'POINT(0 0)'::geometry as g, 1 as a_very_very_very_long_field_name",
+            "SELECT 'LINESTRING(0 0, 1 0)'::geometry, 2"
+        ].join(" UNION ALL "),
+        format: 'shp'
+      });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body);
+        var parsedBody = JSON.parse(res.body);
+        var error = parsedBody.error[0];
+        var expectedError = /Attempt to write non-point \(LINESTRING\) geometry to point shapefile/g;
+        assert.ok(expectedError.test(error), error);
+        done();
+    });
+});
+
+it('skipfields controls fields included in SHP output', function(done){
+    var query = querystring.stringify({
+        q: "SELECT 111 as skipme, 222 as keepme, 'POINT(0 0)'::geometry as g",
+        format: 'shp',
+        skipfields: 'skipme',
+        filename: 'myshape'
+      });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        var buffer = zf.readFileSync('myshape.dbf');
+        fs.unlinkSync(tmpfile);
+        var strings = buffer.toString();
+        assert.ok(!/skipme/.exec(strings), "Could not skip 'skipme' field:\n" + strings);
+        done();
+    });
+});
+
+it('SHP format, concurrently', function(done){
+    var concurrency = 1;
+    var waiting = concurrency;
+    function validate(err, res){
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+        // TODO: check DBF contents
+        fs.unlinkSync(tmpfile);
+        if ( ! --waiting ) {
+            done();
+        }
+    }
+    for (var i=0; i<concurrency; ++i) {
+        assert.response(
+            server,
+            {
+                url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
+                headers: {host: 'vizzuality.cartodb.com'},
+                encoding: 'binary',
+                method: 'GET'
+            },
+            {
+                status: 200
+            },
+            validate
+        );
+    }
+});
+
+// See https://github.com/CartoDB/CartoDB-SQL-API/issues/111
+it('point with null first', function(done){
+    var query = querystring.stringify({
+        q: "SELECT null::geometry as g UNION ALL SELECT 'SRID=4326;POINT(0 0)'::geometry",
+        format: 'shp'
+      });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        encoding: 'binary',
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
+        var tmpfile = '/tmp/myshape.zip';
+        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
+        if (writeErr) {
+            return done(writeErr);
+        }
+        var zf = new zipfile.ZipFile(tmpfile);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
+        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
+        // TODO: check contents of the DBF
+        fs.unlinkSync(tmpfile);
+        done();
+    });
+});
+
+    var limit = 1200;
+
+    it('expects ' + limit + ' rows in public table', function(done){
+
+        var filename = 'test_1200';
+
+        assert.response(server, {
+                url: '/api/v1/sql?' + querystring.stringify({
+                    q: "SELECT * from populated_places_simple_reduced limit " + limit,
+                    format: 'shp',
+                    filename: filename
+                }),
+                headers: { host: 'vizzuality.cartodb.com' },
+                encoding: 'binary',
+                method: 'GET'
+            },
+            {
+                status: 200
+            },
+            function(err, res) {
+                if (err) {
+                    return done(err);
+                }
+
+                var tmpShpPath = '/tmp/'+filename+'.zip';
+                err = fs.writeFileSync(tmpShpPath, res.body, 'binary');
+                if (err) {
+                    return done(err);
+                }
+
+                var zf = new zipfile.ZipFile(tmpShpPath);
+                zf.names.forEach(function(name) {
+                    var buffer = zf.readFileSync(name);
+                    var tmpDbfPath = '/tmp/' + name;
+                    err = fs.writeFileSync(tmpDbfPath, buffer);
+                    if (err) {
+                        return done(err);
+                    }
+                });
+                shapefile.read('/tmp/'+filename, function(err, collection) {
+                    if (err) {
+                        return done(err);
+                    }
+                    assert.equal(collection.features.length, limit);
+                    done();
+                });
+
+            }
+        );
+    });
+
+    it('SHP zip, wrong path for zip command should return error', function(done){
+        global.settings.zipCommand = '/wrong/path';
+        var query = querystring.stringify({
+            q: "SELECT st_makepoint(0,0,4326) as the_geom",
+            format: 'shp',
+            filename: 'myshape'
+        });
+        assert.response(server, {
+            url: '/api/v1/sql?' + query,
+            headers: {host: 'vizzuality.cartodb.com'},
+            encoding: 'binary',
+            method: 'GET'
+        },{ }, function(err, res){
+            assert.equal(res.statusCode, 400, res.body);
+            var parsedBody = JSON.parse(res.body);
+            var respBodyPattern = new RegExp('Error executing zip command,  Error: spawn(.*)ENOENT', 'i');
+            assert.equal(respBodyPattern.test(parsedBody.error[0]), true);
+            done();
+        });
+    });
+});

test/acceptance/export/spatialite.js

@@ -0,0 +1,56 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var sqlite = require('sqlite3');
+
+describe('spatialite query', function(){
+
+    it('returns a valid sqlite database', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=spatialite',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{ }, function(err, res) {
+            assert.equal(res.statusCode, 200, res.body);
+            assert.equal(res.headers["content-type"], "application/x-sqlite3; charset=utf-8");
+            var db = new sqlite.Database(':memory:', res.body);
+            var qr = db.get("PRAGMA database_list", function(err){
+                assert.equal(err, null);
+                done();
+            });
+            assert.notEqual(qr, undefined);
+        });
+
+    });
+
+    it('different file name', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=spatialite&filename=manolo',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{ }, function(err, res) {
+            assert.equal(res.headers["content-type"], "application/x-sqlite3; charset=utf-8");
+            assert.notEqual(res.headers["content-disposition"].indexOf("manolo.sqlite"), -1);
+            done();
+        });
+    });
+
+    it('gets database schema', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=spatialite',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{ }, function(err, res) {
+            var db = new sqlite.Database(':memory:', res.body);
+            var schemaQuery = "SELECT name, sql FROM sqlite_master WHERE type='table' ORDER BY name";
+            var qr = db.get(schemaQuery, function(err){
+                assert.equal(err, null);
+                done();
+            });
+            assert.notEqual(qr, undefined);
+        });
+    });
+});

test/acceptance/export/svg.js

@@ -0,0 +1,199 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var querystring = require('querystring');
+
+describe('export.svg', function() {
+
+it('GET /api/v1/sql with SVG format', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
+      format: "svg"
+    });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
+        // TODO: test viewBox
+        done();
+    });
+});
+
+it('POST /api/v1/sql with SVG format', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
+      format: "svg"
+    });
+    assert.response(server, {
+        url: '/api/v1/sql',
+        data: query,
+        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
+        method: 'POST'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd);
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
+        // TODO: test viewBox
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SVG format and custom filename', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
+      format: "svg",
+      filename: 'mysvg'
+    });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.ok(/filename=mysvg.svg/gi.test(cd), cd);
+        assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
+        // TODO: test viewBox
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SVG format and centered point', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
+      format: "svg"
+    });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body );
+        // TODO: test viewBox
+        // TODO: test radius
+        done();
+    });
+});
+
+it('GET /api/v1/sql with SVG format and trimmed decimals', function(done){
+    var queryobj = {
+      q: "SELECT 1 as cartodb_id, 'LINESTRING(0 0, 1024 768, 500.123456 600.98765432)'::geometry AS the_geom ",
+      format: "svg",
+      dp: 2
+    };
+    assert.response(server, {
+        url: '/api/v1/sql?' + querystring.stringify(queryobj),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 200, res.body);
+        var cd = res.headers['content-disposition'];
+        assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+        assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
+        assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.12 167.01" />') > 0, res.body );
+        // TODO: test viewBox
+
+        queryobj.dp = 3;
+        assert.response(server, {
+          url: '/api/v1/sql?' + querystring.stringify(queryobj),
+          headers: {host: 'vizzuality.cartodb.com'},
+          method: 'GET'
+        },{}, function(err, res) {
+          assert.equal(res.statusCode, 200, res.body);
+          var cd = res.headers['content-disposition'];
+          assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd);
+          assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
+          assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
+          assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.123 167.012" />') > 0, res.body );
+          // TODO: test viewBox
+          done();
+        });
+    });
+});
+
+// Test adding "the_geom" to skipfields
+// See http://github.com/Vizzuality/CartoDB-SQL-API/issues/73
+it('SVG format with "the_geom" in skipfields', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
+      format: "svg",
+      skipfields: "the_geom"
+    });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
+        assert.deepEqual(res.headers['content-disposition'], 'inline');
+        assert.deepEqual(JSON.parse(res.body), {
+          error:['column "the_geom" does not exist']
+        });
+        done();
+    });
+});
+
+it('SVG format with missing "the_geom" field', function(done){
+    var query = querystring.stringify({
+      q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS something_else ",
+      format: "svg"
+    });
+    assert.response(server, {
+        url: '/api/v1/sql?' + query,
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(err, res){
+        assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
+        assert.deepEqual(JSON.parse(res.body), {
+          error:['column "the_geom" does not exist']
+        });
+        done();
+    });
+});
+
+    it('should close on error and error must be the only key in the body', function(done) {
+        assert.response(
+            server,
+            {
+                url: "/api/v1/sql?" + querystring.stringify({
+                    q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4",
+                    format: 'svg'
+                }),
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                method: 'GET'
+            },
+            {
+                status: 400
+            },
+            function(err, res) {
+                var parsedBody = JSON.parse(res.body);
+                assert.deepEqual(Object.keys(parsedBody), ['error']);
+                assert.deepEqual(parsedBody.error, ["division by zero"]);
+                done();
+            }
+        );
+    });
+
+
+
+});

test/acceptance/export/timeout.js

@@ -0,0 +1,202 @@
+'use strict';
+
+const TestClient = require('../../support/test-client');
+
+require('../../support/assert');
+
+var assert = require('assert');
+var querystring = require('querystring');
+const db_utils = require('../../support/db_utils');
+
+describe('timeout', function () {
+    describe('export database', function () {
+        before(db_utils.resetPgBouncerConnections);
+        after(db_utils.resetPgBouncerConnections);
+
+        const databaseTimeoutQuery = `
+            select
+                ST_SetSRID(ST_Point(0, 0), 4326) as the_geom,
+                pg_sleep(0.2) as sleep,
+                1 as value
+        `;
+
+        const scenarios = [
+            {
+                desc: 'CSV',
+                format: 'csv',
+                contentType: 'application/x-www-form-urlencoded',
+                parser: querystring.stringify,
+                // only: true,
+                skip: true
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'Geopackage',
+                format: 'gpkg'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'KML',
+                format: 'kml'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'Shapefile',
+                format: 'shp'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'Spatialite',
+                format: 'spatialite'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'Array Buffer',
+                format: 'arraybuffer'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'GeoJSON',
+                format: 'geojson'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'JSON',
+                format: 'json'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'SVG',
+                format: 'svg'
+            },
+            {
+                query: databaseTimeoutQuery,
+                desc: 'TopoJSON',
+                format: 'topojson'
+            }
+        ];
+
+        beforeEach(function (done) {
+            this.testClient = new TestClient();
+            this.testClient.setUserDatabaseTimeoutLimit('localhost', 100, done);
+        });
+
+        afterEach(function (done) {
+            this.testClient.setUserDatabaseTimeoutLimit('localhost', 2000, done);
+        });
+
+        scenarios.forEach((scenario) => {
+            const test = scenario.only ? it.only : scenario.skip ? it.skip : it;
+
+            test(`${scenario.desc} export exceeding statement timeout responds 429 Over Limits`, function (done) {
+                const override = {
+                    'Content-Type': scenario.contentType,
+                    parser: scenario.parser,
+                    anonymous: true,
+                    format: scenario.format,
+                    response: {
+                        status: 429
+                    }
+                };
+
+                this.testClient.getResult(scenario.query, override, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepEqual(res, {
+                        error: [
+                            'You are over platform\'s limits: SQL query timeout error.' +
+                            ' Refactor your query before running again or contact CARTO support for more details.',
+                        ],
+                        context: 'limit',
+                        detail: 'datasource'
+                    });
+
+                    done();
+                });
+            });
+        });
+    });
+
+    describe('export ogr command timeout', function () {
+        const ogrCommandTimeoutQuery = `
+            select
+                ST_SetSRID(ST_Point(0, 0), 4326) as the_geom,
+                pg_sleep(0.2) as sleep,
+                1 as value
+            `;
+
+        const scenarios = [
+            {
+                query: ogrCommandTimeoutQuery,
+                desc: 'CSV',
+                format: 'csv',
+                contentType: 'application/x-www-form-urlencoded',
+                parser: querystring.stringify,
+                // only: true,
+                // skip: true
+            },
+            {
+                query: ogrCommandTimeoutQuery,
+                filename: 'wadus_gpkg_filename',
+                desc: 'Geopackage',
+                format: 'gpkg'
+            },
+            {
+                query: ogrCommandTimeoutQuery,
+                desc: 'KML',
+                format: 'kml'
+            },
+            {
+                query: ogrCommandTimeoutQuery,
+                desc: 'Shapefile',
+                format: 'shp'
+            },
+            {
+                query: ogrCommandTimeoutQuery,
+                desc: 'Spatialite',
+                format: 'spatialite'
+            }
+        ];
+
+        beforeEach(function (done) {
+            this.testClient = new TestClient();
+            this.testClient.setUserRenderTimeoutLimit('vizzuality', 100, done);
+        });
+
+        afterEach(function (done) {
+            this.testClient.setUserRenderTimeoutLimit('vizzuality', 0, done);
+        });
+
+        scenarios.forEach((scenario) => {
+            const test = scenario.only ? it.only : scenario.skip ? it.skip : it;
+
+            test(`${scenario.desc} export exceeding statement timeout responds 429 Over Limits`, function (done) {
+                const override = {
+                    'Content-Type': scenario.contentType,
+                    parser: scenario.parser,
+                    anonymous: true,
+                    format: scenario.format,
+                    filename: scenario.filename,
+                    response: {
+                        status: 429
+                    }
+                };
+
+                this.testClient.getResult(scenario.query, override, (err, res) => {
+                    assert.ifError(err);
+
+                    assert.deepEqual(res, {
+                        error: [
+                            'You are over platform\'s limits: SQL query timeout error.' +
+                            ' Refactor your query before running again or contact CARTO support for more details.',
+                        ],
+                        context: 'limit',
+                        detail: 'datasource'
+                    });
+
+                    done();
+                });
+            });
+        });
+    });
+});

test/acceptance/export/topojson.js

@@ -0,0 +1,267 @@
+'use strict';
+
+require('../../helper');
+
+var server = require('../../../app/server')();
+var assert = require('../../support/assert');
+var querystring = require('querystring');
+var _ = require('underscore');
+
+describe('export.topojson', function() {
+
+// TOPOJSON tests
+
+    function getRequest(query, extraParams) {
+        var params = {
+            q: query,
+            format: 'topojson'
+        };
+
+        params = _.extend(params, extraParams || {});
+
+        return {
+            url: '/api/v1/sql?' + querystring.stringify(params),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        };
+    }
+
+it('GET two polygons sharing an edge as topojson', function(done){
+    assert.response(server,
+        getRequest(
+            "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom " +
+            " UNION ALL " +
+            "SELECT 2, 'D', 'POLYGON((0 -5,0 5,-5 0,0 -5))'::geometry as the_geom "
+        ),
+        {
+            status: 200
+        },
+        function(err, res) {
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'TOPOJSON is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.topojson/gi.test(cd));
+        var topojson = JSON.parse(res.body);
+        assert.equal(topojson.type, 'Topology');
+
+        // Check transform
+        assert.ok(topojson.hasOwnProperty('transform'));
+        var trans = topojson.transform;
+        assert.equal(_.keys(trans).length, 2); // only scale and translate
+        assert.equal(trans.scale.length, 2); // scalex, scaley
+        assert.equal(Math.round(trans.scale[0]*1e6), 1000);
+        assert.equal(Math.round(trans.scale[1]*1e6), 1000);
+        assert.equal(trans.translate.length, 2); // translatex, translatey
+        assert.equal(trans.translate[0], -5);
+        assert.equal(trans.translate[1], -5);
+
+        // Check objects
+        assert.ok(topojson.hasOwnProperty('objects'));
+        assert.equal(_.keys(topojson.objects).length, 2);
+
+        var obj = topojson.objects[0];
+        //console.dir(obj);
+        // Expected:
+        // { type: 'Polygon',
+        //   arcs: [ [ 0, 1 ] ],
+        //   properties: { gid: 1, nam: 'U' } }
+        assert.equal(_.keys(obj).length, 3); // type, arcs, properties
+        assert.equal(obj.type, 'Polygon');
+        assert.equal(obj.arcs.length, 1); /* only shell, no holes */
+        var shell = obj.arcs[0];
+        assert.equal(shell.length, 2); /* one shared arc, one non-shared */
+        assert.equal(shell[0], 0); /* shared arc */
+        assert.equal(shell[1], 1); /* non-shared arc */
+        var props = obj.properties;
+        assert.equal(_.keys(props).length, 2); // gid, name
+        assert.equal(props.gid, 1);
+        assert.equal(props.name, 'U');
+
+        obj = topojson.objects[1];
+        //console.dir(obj);
+        // Expected:
+        // { type: 'Polygon',
+        //   arcs: [ [ 0, 2 ] ],
+        //   properties: { gid: 2, nam: 'D' } }
+        assert.equal(_.keys(obj).length, 3); // type, arcs, properties
+        assert.equal(obj.type, 'Polygon');
+        assert.equal(obj.arcs.length, 1); /* only shell, no holes */
+        shell = obj.arcs[0];
+        assert.equal(shell.length, 2); /* one shared arc, one non-shared */
+        assert.equal(shell[0], 0); /* shared arc */
+        assert.equal(shell[1], 2); /* non-shared arc */
+        props = obj.properties;
+        assert.equal(_.keys(props).length, 2); // gid, name
+        assert.equal(props.gid, 2);
+        assert.equal(props.name, 'D');
+
+        // Check arcs
+        assert.ok(topojson.hasOwnProperty('arcs'));
+        assert.equal(topojson.arcs.length, 3); // one shared, two non-shared
+        var arc = topojson.arcs[0]; // shared arc
+        assert.equal(arc.length, 2); // shared arc has two vertices
+        var p = arc[0];
+        assert.equal(Math.round(p[0]*trans.scale[0]), 0);
+        assert.equal(Math.round(p[1]*trans.scale[1]), 5);
+        p = arc[1];
+        assert.equal(Math.round(p[0]*trans.scale[0]), 5);
+        assert.equal(Math.round(p[1]*trans.scale[1]), 5);
+        arc = topojson.arcs[1]; // non shared arc
+        assert.equal(arc.length, 3); // non shared arcs have three vertices
+        p = arc[0];
+        assert.equal(Math.round(p[0]*trans.scale[0]), 5);
+        assert.equal(Math.round(p[1]*trans.scale[1]), 10);
+        p = arc[1];
+        assert.equal(Math.round(p[0]*trans.scale[0]), 5);
+        assert.equal(Math.round(p[1]*trans.scale[1]), -5);
+        p = arc[2];
+        assert.equal(Math.round(p[0]*trans.scale[0]), -10);
+        assert.equal(Math.round(p[1]*trans.scale[1]), 0);
+        arc = topojson.arcs[2]; // non shared arc
+        assert.equal(arc.length, 3); // non shared arcs have three vertices
+        p = arc[0];
+        assert.equal(Math.round(p[0]*trans.scale[0]), 5);
+        assert.equal(Math.round(p[1]*trans.scale[1]), 10);
+        p = arc[1];
+        assert.equal(Math.round(p[0]*trans.scale[0]), 0);
+        assert.equal(Math.round(p[1]*trans.scale[1]), -10);
+        p = arc[2];
+        assert.equal(Math.round(p[0]*trans.scale[0]), -5);
+        assert.equal(Math.round(p[1]*trans.scale[1]), 5);
+
+        done();
+    });
+});
+
+it('null geometries', function(done){
+    assert.response(server, getRequest(
+            "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom " +
+            " UNION ALL " +
+            "SELECT 2, 'D', null::geometry as the_geom "
+        ),
+        {
+            status: 200
+        },
+        function(err, res) {
+        var cd = res.headers['content-disposition'];
+        assert.equal(true, /^attachment/.test(cd), 'TOPOJSON is not disposed as attachment: ' + cd);
+        assert.equal(true, /filename=cartodb-query.topojson/gi.test(cd));
+        var topojson = JSON.parse(res.body);
+        assert.equal(topojson.type, 'Topology');
+
+        // Check transform
+        assert.ok(topojson.hasOwnProperty('transform'));
+        var trans = topojson.transform;
+        assert.equal(_.keys(trans).length, 2); // only scale and translate
+        assert.equal(trans.scale.length, 2); // scalex, scaley
+        assert.equal(Math.round(trans.scale[0]*1e6), 1000);
+        assert.equal(Math.round(trans.scale[1]*1e6), 500);
+        assert.equal(trans.translate.length, 2); // translatex, translatey
+        assert.equal(trans.translate[0], -5);
+        assert.equal(trans.translate[1], 0);
+
+        // Check objects
+        assert.ok(topojson.hasOwnProperty('objects'));
+        assert.equal(_.keys(topojson.objects).length, 1);
+
+        var obj = topojson.objects[0];
+        //console.dir(obj);
+        // Expected:
+        // { type: 'Polygon',
+        //   arcs: [ [ 0, 1 ] ],
+        //   properties: { gid: 1, nam: 'U' } }
+        assert.equal(_.keys(obj).length, 3); // type, arcs, properties
+        assert.equal(obj.type, 'Polygon');
+        assert.equal(obj.arcs.length, 1); /* only shell, no holes */
+        var shell = obj.arcs[0];
+        assert.equal(shell.length, 1); /* one non shared arc */
+        assert.equal(shell[0], 0); /* non-shared arc */
+        var props = obj.properties;
+        assert.equal(_.keys(props).length, 2); // gid, name
+        assert.equal(props.gid, 1);
+        assert.equal(props.name, 'U');
+
+        // Check arcs
+        assert.ok(topojson.hasOwnProperty('arcs'));
+        assert.equal(topojson.arcs.length, 1);
+        var arc = topojson.arcs[0];
+        assert.deepEqual(arc, [ [ 0, 0 ], [ 4999, 9999 ], [ 5000, -9999 ], [ -9999, 0 ] ]);
+
+        done();
+    });
+});
+
+    it('skipped fields are not returned', function(done) {
+        assert.response(server,
+            getRequest(
+                "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom",
+                {
+                    skipfields: 'name'
+                }
+            ),
+            {
+                status: 200
+            },
+            function(err, res) {
+                var parsedBody = JSON.parse(res.body);
+                assert.equal(parsedBody.objects[0].properties.gid, 1, 'gid was expected property');
+                assert.ok(!parsedBody.objects[0].properties.name);
+                done();
+            }
+        );
+    });
+
+    it('jsonp callback is invoked', function(done){
+        assert.response(
+            server,
+            getRequest(
+                "SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom",
+                {
+                    callback: 'foo_jsonp'
+                }
+            ),
+            {
+                status: 200
+            },
+            function(err, res) {
+                assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+                var didRunJsonCallback = false;
+                // jshint ignore:start
+                function foo_jsonp(body) {
+                    didRunJsonCallback = true;
+                }
+                eval(res.body);
+                // jshint ignore:end
+                assert.ok(didRunJsonCallback);
+                done();
+            }
+        );
+    });
+
+
+    it('should close on error and error must be the only key in the body', function(done) {
+        assert.response(
+            server,
+            {
+                url: "/api/v1/sql?" + querystring.stringify({
+                    q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4",
+                    format: 'topojson'
+                }),
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                method: 'GET'
+            },
+            {
+                status: 400
+            },
+            function(err, res) {
+                var parsedBody = JSON.parse(res.body);
+                assert.deepEqual(Object.keys(parsedBody), ['error']);
+                assert.deepEqual(parsedBody.error, ["division by zero"]);
+                done();
+            }
+        );
+    });
+
+
+});

test/acceptance/frontend_abort.js

@@ -0,0 +1,79 @@
+'use strict';
+
+require('../helper');
+
+var assert = require('../support/assert');
+var step = require('step');
+var net = require('net');
+
+var sql_server_data_handler;
+var sql_server_port = 5556;
+var sql_server = net.createServer(function(c) {
+  c.on('data', function(d) {
+    console.log("SQL Server got data: " + d);
+    if ( sql_server_data_handler ) {
+      console.log("Sending data to sql_server_data_handler");
+      sql_server_data_handler(null, d);
+    }
+    c.destroy();
+  });
+});
+
+describe('frontend abort', function() {
+
+before(function(done){
+  sql_server.listen(sql_server_port, done);
+});
+
+// See https://github.com/CartoDB/CartoDB-SQL-API/issues/129
+it('aborts request', function(done){
+//console.log("settings:"); console.dir(global.settings);
+  var db_host_backup = global.settings.db_host;
+  var db_port_backup = global.settings.db_port;
+  global.settings.db_host = 'localhost';
+  global.settings.db_port = sql_server_port;
+  var server = require('../../app/server')();
+  var timeout;
+  step(
+    function sendQuery() {
+      assert.response(server, {
+          url: '/api/v1/sql?q=SELECT+1',
+          method: 'GET',
+          timeout: 1,
+          headers: {host: 'vizzuality.localhost' }
+      },{}, this);
+    },
+    function checkResponse(err/*, res*/) {
+      assert(err); // expect timeout
+      assert.ok((''+err).match(/socket/), err);
+      sql_server_data_handler = this;
+      var next = this;
+      // If a call does not arrive to the sql server within
+      // the given timeout we're confident it means the request
+      // was successfully aborted
+      timeout = setTimeout(function() { next(null); }, 500);
+    },
+    function checkSqlServerData(err, data) {
+      clearTimeout(timeout);
+      assert.ok(!data, "SQL Server was contacted no matter client abort");
+      // TODO: intercept logs ?
+      return null;
+    },
+    function finish(err) {
+      global.settings.db_host = db_host_backup;
+      global.settings.db_port = db_port_backup;
+      done(err);
+    }
+  );
+});
+
+after(function(done) {
+  try {
+    sql_server.close(done);
+  } catch (er) {
+    console.log(er);
+    done(); // error expected as server is probably closed already
+  }
+});
+
+});

test/acceptance/health_check.js

@@ -0,0 +1,68 @@
+'use strict';
+
+require('../helper');
+require('../support/assert');
+
+var assert = require('assert');
+var server = require('../../app/server')();
+
+describe('health checks', function() {
+
+    beforeEach(function(done) {
+        global.settings.health = {
+            enabled: true
+            //username: 'vizzuality',
+            //query: 'select 1::text'
+        };
+        done();
+    });
+
+    var healthCheckRequest = {
+        url: '/api/v1/health',
+        method: 'GET',
+        headers: {
+            host: 'vizzuality.localhost'
+        }
+    };
+
+    it('returns 200 and ok=true with disabled configuration', function(done) {
+        global.settings.health.enabled = false;
+
+        assert.response(server,
+            healthCheckRequest,
+            {
+                status: 200
+            },
+            function(err, res) {
+                assert.ok(!err);
+
+                var parsed = JSON.parse(res.body);
+
+                assert.equal(parsed.enabled, false);
+                assert.ok(parsed.ok);
+
+                done();
+            }
+        );
+    });
+
+    it('returns 200 and ok=true with enabled configuration', function(done) {
+        assert.response(server,
+            healthCheckRequest,
+            {
+                status: 200
+            },
+            function(err, res) {
+                assert.ok(!err);
+
+                var parsed = JSON.parse(res.body);
+
+                assert.ok(parsed.enabled);
+                assert.ok(parsed.ok);
+
+                done();
+            }
+        );
+    });
+
+});

test/acceptance/last-modified-header.js

@@ -0,0 +1,114 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var qs = require('querystring');
+var MockDate = require('mockdate');
+
+describe('last modified header', function() {
+
+    var scenarios = [
+        {
+            tables: ['untitle_table_4'],
+            desc: 'should use last updated time from public table',
+            expectedLastModified: 'Wed, 01 Jan 2014 23:31:30 GMT'
+        },
+        {
+            tables: ['private_table'],
+            desc: 'should use last updated time from private table',
+            expectedLastModified: 'Thu, 01 Jan 2015 23:31:30 GMT'
+        },
+        {
+            tables: ['untitle_table_4', 'private_table'],
+            desc: 'should use most recent last updated time from private and public table',
+            expectedLastModified: 'Thu, 01 Jan 2015 23:31:30 GMT'
+        },
+        {
+            tables: ['populated_places_simple_reduced', 'private_table'],
+            desc: 'should use last updated time from table in cdb_tablemetadata instead of now() from unknown table',
+            expectedLastModified: 'Thu, 01 Jan 2015 23:31:30 GMT'
+        }
+    ];
+
+    scenarios.forEach(function(scenario) {
+        it(scenario.desc, function(done) {
+            var query = qs.stringify({
+                q: scenario.tables.map(function(table) {
+                    return 'select cartodb_id from ' + table;
+                }).join(' UNION ALL '),
+                api_key: 1234
+            });
+            assert.response(server,
+                {
+                    url: '/api/v1/sql?' + query,
+                    headers: {
+                        host: 'vizzuality.cartodb.com'
+                    },
+                    method: 'GET'
+                },
+                {
+                    statusCode: 200
+                },
+                function(err, res) {
+                    assert.equal(res.headers['last-modified'], scenario.expectedLastModified);
+                    done();
+                }
+            );
+        });
+    });
+
+    it('should use Date.now() for tables not present in cdb_tablemetadata', function(done) {
+        var query = qs.stringify({
+            q: 'select cartodb_id from populated_places_simple_reduced limit 1',
+            api_key: 1234
+        });
+        var fixedDateNow = Date.now();
+	MockDate.set(fixedDateNow);
+        assert.response(server,
+            {
+                url: '/api/v1/sql?' + query,
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                method: 'GET'
+            },
+            {
+                statusCode: 200
+            },
+            function(err, res) {
+                MockDate.reset();
+                assert.equal(res.headers['last-modified'], new Date(fixedDateNow).toUTCString());
+                done();
+            }
+        );
+    });
+
+    it('should use Date.now() for functions or results with no table associated', function(done) {
+        var query = qs.stringify({
+            q: 'select 1',
+            api_key: 1234
+        });
+        var fixedDateNow = Date.now();
+	MockDate.set(fixedDateNow);
+        assert.response(server,
+            {
+                url: '/api/v1/sql?' + query,
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                method: 'GET'
+            },
+            {
+                statusCode: 200
+            },
+            function(err, res) {
+                MockDate.reset();
+                assert.equal(res.headers['last-modified'], new Date(fixedDateNow).toUTCString());
+                done();
+            }
+        );
+    });
+
+});

test/acceptance/logging.js

@@ -0,0 +1,152 @@
+'use strict';
+
+require('../helper');
+
+var appServer = require('../../app/server');
+var assert = require('../support/assert');
+var qs = require('querystring');
+var log4js = require('log4js');
+
+describe('Logging SQL query on POST requests', function() {
+
+    var SQL_QUERY = "SELECT 'wadus'";
+    var API_KEY = 1234;
+    var BODY_PAYLOAD = {
+        q: SQL_QUERY,
+        api_key: API_KEY
+    };
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    var server;
+    before(function() {
+        global.settings.log_format = ':method :req[Host]:url :status :sql';
+        global.log4js = log4js;
+        global.log4js.configure({
+            appenders: [
+                {
+                    type: "console",
+                    layout: {
+                        type:'basic'
+                    }
+                }
+            ]
+        });
+        server = appServer();
+    });
+
+    after(function() {
+        global.log4js = null;
+        delete global.log4js;
+    });
+
+    function createPostRequest(body, contentType, getParams) {
+        var url = '/api/v1/sql';
+        if (getParams) {
+            url += '?' + qs.stringify(getParams);
+        }
+        return {
+            method: 'POST',
+            url: url,
+            data: body,
+            headers: {
+                host: 'vizzuality.cartodb.com',
+                'Content-Type': contentType
+            }
+        };
+    }
+
+    var LENGHTY_SUFFIX = ' [...]';
+
+    var postScenariosRequests = [
+        {
+            desc: 'should return json string for application/x-www-form-urlencoded',
+            request: createPostRequest(
+                qs.stringify(BODY_PAYLOAD), 'application/x-www-form-urlencoded'
+            )
+        },
+        {
+            desc: 'should return json string for application/x-www-form-urlencoded, with API key in GET param',
+            request: createPostRequest(
+                qs.stringify({q: SQL_QUERY}), 'application/x-www-form-urlencoded', {api_key: API_KEY}
+            )
+        },
+        {
+            desc: 'should return json string for application/json',
+            request: createPostRequest(
+                JSON.stringify(BODY_PAYLOAD), 'application/json'
+            )
+        },
+        {
+            desc: 'should return json string for application/json, with API key in GET param',
+            request: createPostRequest(
+                JSON.stringify({q: SQL_QUERY}), 'application/json', {api_key: API_KEY}
+            )
+        },
+        {
+            desc: 'should return a substring when sql query is very long',
+            request: createPostRequest(
+                JSON.stringify({q: "select '" + new Array(2500).join('a') +  "'"}), 'application/json'
+            ),
+            expectedSQLQueryToLog: "select '" + (new Array(2000 + 1 - "select '".length).join('a')) + LENGHTY_SUFFIX
+        }
+    ];
+
+    postScenariosRequests.forEach(function(scenario) {
+        it(scenario.desc, function(done) {
+            var called = 0;
+
+            var getSqlQueryFromRequestBodyFn = server.getSqlQueryFromRequestBody;
+
+            server.getSqlQueryFromRequestBody = function(req) {
+                called++;
+                var result = getSqlQueryFromRequestBodyFn(req);
+                assert.deepEqual(JSON.parse(result), {q: scenario.expectedSQLQueryToLog || SQL_QUERY});
+                return result;
+            };
+
+            assert.response(server, scenario.request, RESPONSE_OK, function(err) {
+                assert.ok(!err);
+                assert.equal(called, 1);
+
+                server.getSqlQueryFromRequestBody = getSqlQueryFromRequestBodyFn;
+
+                done();
+            });
+        });
+    });
+
+    it('should not log sql query in GET requests', function(done) {
+        var called = 0;
+
+        var getSqlQueryFromRequestBodyFn = server.getSqlQueryFromRequestBody;
+
+        server.getSqlQueryFromRequestBody = function(req) {
+            called++;
+            var result = getSqlQueryFromRequestBodyFn(req);
+            assert.equal(result, '');
+            return result;
+        };
+
+        assert.response(server,
+            {
+                method: 'GET',
+                url: '/api/v1/sql?' + qs.stringify(BODY_PAYLOAD),
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                }
+            },
+            RESPONSE_OK,
+            function(err) {
+                assert.ok(!err);
+                assert.equal(called, 1);
+
+                server.getSqlQueryFromRequestBody = getSqlQueryFromRequestBodyFn;
+
+                done();
+            }
+        );
+    });
+});

test/acceptance/oauth/oauth_test.py

@@ -0,0 +1,26 @@
+# TO RUN
+# > virtualenv env
+# > . env/bin/activate
+# > pip install oauth2
+# > pip install cartodb
+#
+# FILL IN THINGS BELOW
+# > python oauth_test.py
+
+from cartodb import CartoDB, CartoDBException
+
+import httplib2
+import oauth2 as oauth
+if __name__ == '__main__':
+
+    user = ''
+    password = ''
+    CONSUMER_KEY= ''
+    CONSUMER_SECRET= ''
+    cl = CartoDB(CONSUMER_KEY, CONSUMER_SECRET, user, password, 'simon')
+    try:
+        print cl.sql('select * from do_not_exist')
+    except CartoDBException as e:
+        print ("some error ocurred", e)
+    print cl.sql('select * from table');
+

test/acceptance/pagination.js

@@ -0,0 +1,175 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var step = require('step');
+
+
+describe('results-pagination', function() {
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    // Test for https://github.com/Vizzuality/CartoDB-SQL-API/issues/85
+    it("paging doesn't break x-cache-channel", function(done) {
+        assert.response(server, {
+            url: '/api/v1/sql?' + querystring.stringify({
+            // note: select casing intentionally mixed
+            q: 'selECT cartodb_id*3 FROM untitle_table_4',
+            api_key: '1234',
+            rows_per_page: 1,
+            page: 2
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            assert.equal(res.headers['x-cache-channel'], 'cartodb_test_user_1_db:public.untitle_table_4');
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.rows.length, 1);
+            done();
+        });
+    });
+
+    // Test page and rows_per_page params
+    it("paging", function(done){
+        var sql = 'SELECT * FROM (VALUES(1),(2),(3),(4),(5),(6),(7),(8),(9)) t(v)';
+        var pr = [ [2,3], [0,4] ]; // page and rows
+        var methods = [ 'GET', 'POST' ];
+        var authorized = 0;
+        var testing = 0;
+        var method = 0;
+        // jshint maxcomplexity:7
+        var testNext = function() {
+        if ( testing >= pr.length ) {
+            if ( method+1 >= methods.length ) {
+            if ( authorized ) {
+                done();
+                return;
+            } else {
+                authorized = 1;
+                method = 0;
+                testing = 0;
+            }
+            } else {
+            testing = 0;
+            ++method;
+            }
+        }
+        var prcur = pr[testing++];
+        console.log("Test " + testing + "/" + pr.length + " method " + methods[method] + " " +
+            ( authorized ? "authenticated" : "" ) );
+        var page = prcur[0];
+        var nrows = prcur[1];
+        var data_obj = {
+                q: sql,
+                rows_per_page: nrows,
+                page: page
+            };
+        if ( authorized ) {
+            data_obj.api_key = '1234';
+        }
+        var data = querystring.stringify(data_obj);
+        var req = {
+            url: '/api/v1/sql',
+            headers: {host: 'vizzuality.cartodb.com'}
+        };
+        if ( methods[method] === 'GET' ) {
+            req.method = 'GET';
+            req.url += '?' + data;
+        } else {
+            req.method = 'POST';
+            req.headers['Content-Type'] = 'application/x-www-form-urlencoded';
+            req.data = data;
+        }
+        assert.response(server, req, RESPONSE_OK, function(err, res) {
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.rows.length, nrows);
+            for (var i=0; i<nrows; ++i) {
+                var obt = parsed.rows[i].v;
+                var exp = page * nrows + i + 1;
+                assert.equal(obt, exp, "Value " + i + " in page " + page + " is " + obt + ", expected " + exp);
+            }
+            testNext();
+        });
+        };
+        testNext();
+    });
+
+    // Test paging with WITH queries
+    it("paging starting with comment", function(done){
+        var sql = "-- this is a comment\n" +
+                "SELECT * FROM (VALUES(1),(2),(3),(4),(5),(6),(7),(8),(9)) t(v)";
+        var nrows = 3;
+        var page = 2;
+        assert.response(server, {
+            url: '/api/v1/sql?' + querystring.stringify({
+            q: sql,
+            rows_per_page: nrows,
+            page: page
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var parsed = JSON.parse(res.body);
+            assert.equal(parsed.rows.length, 3);
+            for (var i=0; i<nrows; ++i) {
+                var obt = parsed.rows[i].v;
+                var exp = page * nrows + i + 1;
+                assert.equal(obt, exp, "Value " + i + " in page " + page + " is " + obt + ", expected " + exp);
+            }
+            done();
+        });
+    });
+
+    // See http://github.com/CartoDB/CartoDB-SQL-API/issues/127
+    it('SELECT INTO with paging', function(done){
+        var esc_tabname = 'test ""select into""'; // escaped ident
+        step(
+        function select_into() {
+            var next = this;
+            assert.response(server, {
+                url: "/api/v1/sql?" + querystring.stringify({
+                q: 'SELECT generate_series(1,10) InTO "' + esc_tabname + '"',
+                rows_per_page: 1, page: 1,
+                api_key: 1234
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            },RESPONSE_OK, function(err, res) { next(null, res); });
+        },
+        function check_res_test_fake_into_1(err) {
+            assert.ifError(err);
+            var next = this;
+            assert.response(server, {
+                url: "/api/v1/sql?" + querystring.stringify({
+                q: 'SELECT \' INTO "c"\' FROM "' + esc_tabname + '"',
+                rows_per_page: 1, page: 1,
+                api_key: 1234
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            }, RESPONSE_OK, function(err, res) { next(null, res); });
+        },
+        function check_res_drop_table(err, res) {
+            assert.ifError(err);
+            var out = JSON.parse(res.body);
+            assert.equal(out.total_rows, 1); // windowing works
+            var next = this;
+            assert.response(server, {
+                url: "/api/v1/sql?" + querystring.stringify({
+                q: 'DROP TABLE "' + esc_tabname + '"',
+                api_key: 1234
+                }),
+                headers: {host: 'vizzuality.cartodb.com'},
+                method: 'GET'
+            }, RESPONSE_OK, function(err, res) { next(null, res); });
+        },
+        done
+        );
+    });
+
+});

test/acceptance/pg-entities-access-validator.js

@@ -0,0 +1,66 @@
+'use strict';
+
+const assert = require('../support/assert');
+const TestClient = require('../support/test-client');
+
+describe('PG entities access validator', function () {
+    const forbiddenQueries = [
+        'select * from information_schema.tables',
+        'select * from pg_catalog.pg_auth_members'
+    ];
+
+    const testClientApiKey = new TestClient({ apiKey: 1234 });
+    const testClientAuthorized = new TestClient({ authorization: 'vizzuality:regular1' });
+
+    const expectedResponse = {
+        response: {
+            status: 403
+        }
+    };
+
+    function assertQuery(query, testClient, done) {
+        testClient.getResult(query, expectedResponse, (err, result) => {
+            assert.ifError(err);
+            assert.equal(result.error, 'system tables are forbidden');
+            done();
+        });
+    }
+
+    describe('validatePGEntitiesAccess enabled', function() {
+        before(function(){
+            global.settings.validatePGEntitiesAccess = true;
+        });
+
+        forbiddenQueries.forEach(query => {
+            it(`testClientApiKey: query: ${query}`, function(done) {
+                assertQuery(query, testClientApiKey, done);
+            });
+
+            it(`testClientAuthorized: query: ${query}`, function(done) {
+                assertQuery(query, testClientAuthorized, done);
+            });
+        });
+    });
+
+    describe('validatePGEntitiesAccess disabled', function() {
+        before(function(){
+            global.settings.validatePGEntitiesAccess = false;
+        });
+
+        forbiddenQueries.forEach(query => {
+            it(`testClientApiKey: query: ${query}`, function(done) {
+                testClientApiKey.getResult(query, err => {
+                    assert.ifError(err);
+                    done();
+                });
+            });
+
+            it(`testClientAuthorized: query: ${query}`, function(done) {
+                testClientAuthorized.getResult(query, err => {
+                    assert.ifError(err);
+                    done();
+                });
+            });
+        });
+    });
+});

test/acceptance/query-float-values.js

@@ -0,0 +1,72 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var step = require('step');
+
+describe('special numeric (float) values', function() {
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+    var HEADERS = {
+        host: 'vizzuality.localhost.lan:8080'
+    };
+    var METHOD = 'GET';
+    var URL = '/api/v1/sql?api_key=1234&';
+
+    it('should cast Infinity and NaN values properly', function (done) {
+        step(
+            function createTable () {
+                var next = this;
+                var opts = {
+                    url: URL + querystring.stringify({
+                        q: 'create table numbers_test(val float)'
+                    }),
+                    headers: HEADERS,
+                    method: METHOD
+                };
+                assert.response(server, opts, RESPONSE_OK, next);
+            },
+            function insertData (err) {
+                assert.ifError(err);
+                var next = this;
+                var opts = {
+                    url: URL + querystring.stringify({
+                        q: [
+                            'insert into numbers_test',
+                            ' values (\'NaN\'::float), (\'infinity\'::float), (\'-infinity\'::float), (1::float)'
+                        ].join('')
+                    }),
+                    headers: HEADERS,
+                    method: METHOD
+                };
+                assert.response(server, opts, RESPONSE_OK, next);
+            },
+            function queryData (err) {
+                assert.ifError(err);
+                var next = this;
+                var opts = {
+                    url: URL + querystring.stringify({
+                        q: 'select * from numbers_test'
+                    }),
+                    headers: HEADERS,
+                    method: METHOD
+                };
+                assert.response(server, opts, RESPONSE_OK, next);
+            },
+            function assertResult (err, res) {
+                assert.ifError(err);
+                var result = JSON.parse(res.body);
+                assert.ok(Array.isArray(result.rows));
+                assert.equal(result.rows[0].val, 'NaN');
+                assert.equal(result.rows[1].val, 'Infinity');
+                assert.equal(result.rows[2].val, '-Infinity');
+                assert.equal(result.rows[3].val, 1);
+                done();
+            }
+        );
+    });
+});

test/acceptance/query-multipart.js

@@ -0,0 +1,26 @@
+'use strict';
+
+require('../helper');
+
+const server = require('../../app/server')();
+const assert = require('../support/assert');
+
+describe('query-multipart', function() {
+    it('make query from a multipart form', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql',
+            formData: {
+                q: 'SELECT 2 as n'
+            },
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'POST'
+        },{}, function(err, res) {
+            assert.ifError(err);
+            const response = JSON.parse(res.body);
+            assert.equal(typeof(response.time) !== 'undefined', true);
+            assert.strictEqual(response.total_rows, 1);
+            assert.deepStrictEqual(response.rows, [{n:2}]);
+            done();
+        });
+    });
+});

test/acceptance/query-returning.js

@@ -0,0 +1,159 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var _ = require('underscore');
+
+
+describe('query-returning', function() {
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    var expected_rw_cache_control = 'no-cache,max-age=0,must-revalidate,public';
+
+    // Check results from INSERT
+    //
+    // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/13
+    it('INSERT returns affected rows', function(done){
+        assert.response(server, {
+            // view prepare_db.sh to see where to set api_key
+            url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+            "INSERT INTO private_table(name) VALUES('noret1') UNION VALUES('noret2')"
+            }),
+            headers: {host: 'vizzuality.localhost.lan:8080' },
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var out = JSON.parse(res.body);
+            assert.ok(out.hasOwnProperty('time'));
+            assert.equal(out.total_rows, 2);
+            assert.equal(out.rows.length, 0);
+            // Check cache headers
+            // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+            assert.ok(!res.hasOwnProperty('x-cache-channel'));
+            assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+            done();
+        });
+    });
+
+    // Check results from UPDATE
+    //
+    // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/13
+    it('UPDATE returns affected rows', function(done){
+        assert.response(server, {
+            // view prepare_db.sh to see where to set api_key
+            url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+            "UPDATE private_table SET name = upper(name) WHERE name in ('noret1', 'noret2')"
+            }),
+            headers: {host: 'vizzuality.localhost.lan:8080' },
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var out = JSON.parse(res.body);
+            assert.ok(out.hasOwnProperty('time'));
+            assert.equal(out.total_rows, 2);
+            assert.equal(out.rows.length, 0);
+            // Check cache headers
+            // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+            assert.ok(!res.hasOwnProperty('x-cache-channel'));
+            assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+            done();
+        });
+    });
+
+    // Check results from DELETE
+    //
+    // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/13
+    it('DELETE returns affected rows', function(done){
+        assert.response(server, {
+            // view prepare_db.sh to see where to set api_key
+            url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+            "DELETE FROM private_table WHERE name in ('NORET1', 'NORET2')"
+            }),
+            headers: {host: 'vizzuality.localhost.lan:8080' },
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var out = JSON.parse(res.body);
+            assert.ok(out.hasOwnProperty('time'));
+            assert.equal(out.total_rows, 2);
+            assert.equal(out.rows.length, 0);
+            // Check cache headers
+            // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/43
+            assert.ok(!res.hasOwnProperty('x-cache-channel'));
+            assert.equal(res.headers['cache-control'], expected_rw_cache_control);
+            done();
+        });
+    });
+
+    // Check results from INSERT .. RETURNING
+    //
+    // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/50
+    it('INSERT with RETURNING returns all results', function(done){
+        assert.response(server, {
+            // view prepare_db.sh to see where to set api_key
+            url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+            "INSERT INTO private_table(name) VALUES('test') RETURNING upper(name), reverse(name)"
+            }),
+            headers: {host: 'vizzuality.localhost.lan:8080' },
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var out = JSON.parse(res.body);
+            assert.ok(out.hasOwnProperty('time'));
+            assert.equal(out.total_rows, 1);
+            assert.equal(out.rows.length, 1);
+            assert.equal(_.keys(out.rows[0]).length, 2);
+            assert.equal(out.rows[0].upper, 'TEST');
+            assert.equal(out.rows[0].reverse, 'tset');
+            done();
+        });
+    });
+
+    // Check results from UPDATE .. RETURNING
+    //
+    // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/50
+    it('UPDATE with RETURNING returns all results', function(done){
+        assert.response(server, {
+            // view prepare_db.sh to see where to set api_key
+            url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+            "UPDATE private_table SET name = 'tost' WHERE name = 'test' RETURNING upper(name), reverse(name)"
+            }),
+            headers: {host: 'vizzuality.localhost.lan:8080' },
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var out = JSON.parse(res.body);
+            assert.ok(out.hasOwnProperty('time'));
+            assert.equal(out.total_rows, 1);
+            assert.equal(out.rows.length, 1);
+            assert.equal(_.keys(out.rows[0]).length, 2);
+            assert.equal(out.rows[0].upper, 'TOST');
+            assert.equal(out.rows[0].reverse, 'tsot');
+            done();
+        });
+    });
+
+    // Check results from DELETE .. RETURNING
+    //
+    // See https://github.com/Vizzuality/CartoDB-SQL-API/issues/50
+    it('DELETE with RETURNING returns all results', function(done){
+        assert.response(server, {
+            // view prepare_db.sh to see where to set api_key
+            url: "/api/v1/sql?api_key=1234&" + querystring.stringify({q:
+            "DELETE FROM private_table WHERE name = 'tost' RETURNING name"
+            }),
+            headers: {host: 'vizzuality.localhost.lan:8080' },
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var out = JSON.parse(res.body);
+            assert.ok(out.hasOwnProperty('time'));
+            assert.equal(out.total_rows, 1);
+            assert.equal(out.rows.length, 1);
+            assert.equal(_.keys(out.rows[0]).length, 1);
+            assert.equal(out.rows[0].name, 'tost');
+            done();
+        });
+    });
+
+});

test/acceptance/query-tables-api-cache.js

@@ -0,0 +1,101 @@
+'use strict';
+
+require('../helper');
+
+var qs = require('querystring');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+
+describe('query-tables-api', function() {
+
+    beforeEach(function(done) {
+        var tableCacheEnabled = global.settings.tableCacheEnabled || false;
+        if(!tableCacheEnabled) {
+            this.skip("tableCache is disabled");
+        }
+        done();
+    });
+
+    function getCacheStatus(callback) {
+        assert.response(
+            server,
+            {
+                method: 'GET',
+                url: '/api/v1/cachestatus'
+            },
+            {
+                status: 200
+            },
+            function(err, res) {
+                callback(null, JSON.parse(res.body));
+            }
+        );
+    }
+
+    var request = {
+        url: '/api/v1/sql?' + qs.stringify({
+            q: 'SELECT * FROM untitle_table_4'
+        }),
+        headers: {
+            host: 'vizzuality.cartodb.com'
+        },
+        method: 'GET'
+    };
+
+    var RESPONSE_OK = {
+        status: 200
+    };
+
+    it('should create a key in affected tables cache', function(done) {
+        assert.response(server, request, RESPONSE_OK, function(err) {
+            assert.ok(!err, err);
+
+            getCacheStatus(function(err, cacheStatus) {
+                assert.ok(!err, err);
+                assert.equal(cacheStatus.explain.keys, 1);
+                assert.equal(cacheStatus.explain.hits, 0);
+
+                done();
+            });
+        });
+    });
+
+    it('should use cache to retrieve affected tables', function(done) {
+        assert.response(server, request, RESPONSE_OK, function(err) {
+            assert.ok(!err, err);
+
+            getCacheStatus(function(err, cacheStatus) {
+                assert.ok(!err, err);
+                assert.equal(cacheStatus.explain.keys, 1);
+                assert.equal(cacheStatus.explain.hits, 1);
+
+                done();
+            });
+        });
+    });
+
+    it('should skip cache to retrieve affected tables', function(done) {
+        var masterRequest = {
+            url: '/api/v1/sql?' + qs.stringify({
+                q: 'SELECT * FROM untitle_table_4',
+                api_key: '1234'
+            }),
+            headers: {
+                host: 'vizzuality.cartodb.com'
+            },
+            method: 'GET'
+        };
+        assert.response(server, masterRequest, RESPONSE_OK, function(err) {
+            assert.ok(!err, err);
+
+            getCacheStatus(function(err, cacheStatus) {
+                assert.ok(!err, err);
+                assert.equal(cacheStatus.explain.keys, 1);
+                assert.equal(cacheStatus.explain.hits, 0);
+
+                done();
+            });
+        });
+    });
+});

test/acceptance/rate-limit.js

@@ -0,0 +1,107 @@
+'use strict';
+
+require('../helper');
+
+const qs = require('querystring');
+const assert = require('../support/assert');
+const redis = require('redis');
+const rateLimitMiddleware = require('../../app/middlewares/rate-limit');
+const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware;
+
+const app = require('../../app/server');
+let server;
+
+let redisClient;
+let keysToDelete = [];
+const user = 'vizzuality';
+
+var request = {
+    url: '/api/v1/sql?' + qs.stringify({
+        q: 'SELECT * FROM untitle_table_4'
+    }),
+    headers: {
+        host: 'vizzuality.cartodb.com'
+    },
+    method: 'GET'
+};
+
+
+function setLimit(count, period, burst) {
+    redisClient.SELECT(8, err => {
+        if (err) {
+            return;
+        }
+
+        const key = `limits:rate:store:${user}:sql:${RATE_LIMIT_ENDPOINTS_GROUPS.QUERY}`;
+        redisClient.rpush(key, burst);
+        redisClient.rpush(key, count);
+        redisClient.rpush(key, period);
+        keysToDelete.push(key);
+    });
+}
+
+function assertRequest (status, limit, remaining, reset, retry, done = null) {
+    assert.response(
+        server,
+        request,
+        { status },
+        function(err, res) {
+            assert.ifError(err);
+            assert.equal(res.headers['carto-rate-limit-limit'], limit);
+            assert.equal(res.headers['carto-rate-limit-remaining'], remaining);
+            assert.equal(res.headers['carto-rate-limit-reset'], reset);
+
+            if (retry) {
+                assert.equal(res.headers['retry-after'], retry);
+            }
+
+            if(status === 429) {
+                const expectedResponse = {
+                    error: ["You are over platform\'s limits. Please contact us to know more details"],
+                    context: "limit",
+                    detail: "rate-limit"
+                };
+
+                assert.deepEqual(JSON.parse(res.body), expectedResponse);
+            }
+
+            if (done) {
+                setTimeout(done, 1000);
+            }
+        }
+    );
+}
+
+describe('rate limit', function() {
+    before(function() {
+        global.settings.ratelimits.rateLimitsEnabled = true;
+        global.settings.ratelimits.endpoints.query = true;
+
+        server = app();
+        redisClient = redis.createClient(global.settings.redis_port);
+
+        const count = 1;
+        const period = 1;
+        const burst = 1;
+        setLimit(count, period, burst);
+    });
+
+    after(function() {
+        global.settings.ratelimits.rateLimitsEnabled = false;
+        global.settings.ratelimits.endpoints.query = false;
+
+        keysToDelete.forEach( key => {
+            redisClient.del(key);
+        });
+    });
+
+    it("1 req/sec: 2 req/seg should be limited", function(done) {
+        assertRequest(200, 2, 1, 1);
+        setTimeout( () => assertRequest(200, 2, 0, 1, null), 250 );
+        setTimeout( () => assertRequest(429, 2, 0, 1, 1),  500 );
+        setTimeout( () => assertRequest(429, 2, 0, 1, 1),  750 );
+        setTimeout( () => assertRequest(429, 2, 0, 1, 1),  950 );
+        setTimeout( () => assertRequest(200, 2, 0, 1, null, done), 1050 );
+    });
+
+});

test/acceptance/regressions.js

@@ -0,0 +1,66 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var qs = require('querystring');
+
+describe('regressions', function() {
+
+    it('issue #224: tables with . (dot) in name works and can be queried', function(done) {
+
+        function createRequest(sqlQuery) {
+            return {
+                url: '/api/v1/sql?' + qs.stringify({
+                    q: sqlQuery,
+                    api_key: 1234
+                }),
+                headers: {
+                    host: 'vizzuality.cartodb.com'
+                },
+                method: 'GET'
+            };
+        }
+
+        var responseOk = {
+            statusCode: 200
+        };
+
+        assert.response(server, createRequest('CREATE TABLE "foo.bar" (a int);'), responseOk,
+            function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.response(server, createRequest('INSERT INTO "foo.bar" (a) values (1), (2)'), responseOk,
+                    function(err, res) {
+                        if (err) {
+                            return done(err);
+                        }
+                        var parsedBody = JSON.parse(res.body);
+                        assert.equal(parsedBody.total_rows, 2);
+
+                        assert.response(server, createRequest('SELECT * FROM "foo.bar"'), responseOk,
+                            function(err, res) {
+                                if (err) {
+                                    return done(err);
+                                }
+
+                                // table should not get a cache channel as it won't get invalidated
+                                assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
+                                var parsedBody = JSON.parse(res.body);
+                                assert.equal(parsedBody.total_rows, 2);
+                                assert.deepEqual(parsedBody.rows, [{ a: 1 }, { a: 2 }]);
+
+                                // delete table
+                                assert.response(server, createRequest('DROP TABLE "foo.bar"'), responseOk, done);
+                            }
+                        );
+                    }
+                );
+            }
+        );
+    });
+
+});

test/acceptance/skipfields.js

@@ -0,0 +1,94 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var _ = require('underscore');
+
+
+describe('skipfields', function() {
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    it('skipfields controls included fields', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=' +
+                'SELECT%20*%20FROM%20untitle_table_4&skipfields=the_geom_webmercator,cartodb_id,unexistant',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res){
+            var row0 = JSON.parse(res.body).rows[0];
+            var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':0};
+            for ( var f in checkfields ) {
+            if ( checkfields[f] ) {
+                assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+            } else {
+                assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+            }
+            }
+            done();
+        });
+    });
+
+    it('multiple skipfields parameter do not kill the backend', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&skipfields=unexistent,the_geom_webmercator' +
+                '&skipfields=cartodb_id,unexistant',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res){
+            var row0 = JSON.parse(res.body).rows[0];
+            var checkfields = {'name':1, 'cartodb_id':0, 'the_geom':1, 'the_geom_webmercator':0};
+            for ( var f in checkfields ) {
+            if ( checkfields[f] ) {
+                assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+            } else {
+                assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+            }
+            }
+            done();
+        });
+    });
+
+    // See https://github.com/CartoDB/CartoDB-SQL-API/issues/109
+    it('schema response takes skipfields into account', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?' + querystring.stringify({
+            q: "SELECT 1 as a, 2 as b, 3 as c ",
+            skipfields: 'b'
+            }),
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        }, RESPONSE_OK, function(err, res) {
+            var parsedBody = JSON.parse(res.body);
+            assert.equal(_.keys(parsedBody.fields).length, 2);
+            assert.ok(parsedBody.fields.hasOwnProperty('a'));
+            assert.ok(!parsedBody.fields.hasOwnProperty('b'));
+            assert.ok(parsedBody.fields.hasOwnProperty('c'));
+            done();
+        });
+    });
+    it('field named "the_geom_webmercator" is not skipped by default', function(done){
+        assert.response(server, {
+            url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },{ }, function(err, res){
+            assert.equal(res.statusCode, 200, res.body);
+            var row0 = JSON.parse(res.body).rows[0];
+            var checkfields = {'name':1, 'cartodb_id':1, 'the_geom':1, 'the_geom_webmercator':1};
+            for ( var f in checkfields ) {
+            if ( checkfields[f] ) {
+                assert.ok(row0.hasOwnProperty(f), "result does not include '" + f + "'");
+            } else {
+                assert.ok(!row0.hasOwnProperty(f), "result includes '" + f + "'");
+            }
+            }
+            done();
+        });
+    });
+});

test/acceptance/stream-responses.js

@@ -0,0 +1,73 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+
+describe('stream-responses', function() {
+
+    function createFailingQueryRequest(format) {
+        var params = {
+            q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4"
+        };
+
+        if (format) {
+            params.format = format;
+        }
+
+        return {
+            url: "/api/v1/sql?" + querystring.stringify(params),
+            headers: {
+                host: 'vizzuality.cartodb.com'
+            },
+            method: 'GET'
+        };
+    }
+
+    var okResponse = {
+        status: 200
+    };
+
+    describe('format-json', function() {
+
+        it('should close on error and error message must be part of the response', function(done) {
+            assert.response(
+                server,
+                createFailingQueryRequest(),
+                okResponse,
+                function(err, res) {
+                    var parsedBody = JSON.parse(res.body);
+                    assert.equal(parsedBody.rows.length, 2);
+                    assert.deepEqual(parsedBody.fields, {
+                        the_geom: { type: "geometry" },
+                        cdb_ratio: { type: "number" }
+                    });
+                    assert.deepEqual(parsedBody.error, ["division by zero"]);
+                    done();
+                }
+            );
+        });
+
+    });
+
+    describe('format-geojson', function() {
+
+        it('should close on error and error message must be part of the response', function(done) {
+            assert.response(
+                server,
+                createFailingQueryRequest('geojson'),
+                okResponse,
+                function(err, res) {
+                    var parsedBody = JSON.parse(res.body);
+                    assert.equal(parsedBody.features.length, 2);
+                    assert.deepEqual(parsedBody.error, ["division by zero"]);
+                    done();
+                }
+            );
+        });
+
+    });
+
+});

test/acceptance/surrogate-key.js

@@ -0,0 +1,111 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var QueryTables = require('cartodb-query-tables');
+var _ = require('underscore');
+
+describe('Surrogate-Key header', function() {
+
+    function createGetRequest(sqlQuery) {
+        var query = querystring.stringify({
+            q: sqlQuery,
+            api_key: 1234
+        });
+        return {
+            url: '/api/v1/sql?' + query,
+            headers: {
+                host: 'vizzuality.cartodb.com'
+            },
+            method: 'GET'
+        };
+    }
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    function surrogateKeyHasTables(surrogateKey, expectedTables) {
+
+        var surrogateKeys = surrogateKey.split(" ");
+
+        var expectedSurrogateKeys = new QueryTables.DatabaseTablesEntry(expectedTables).key();
+
+        assert.equal(surrogateKeys.length, expectedSurrogateKeys.length);
+
+        var tablesDiff = _.difference(surrogateKeys, expectedSurrogateKeys);
+        assert.equal(tablesDiff.length, 0, 'Surrogate-Key missing tables: ' + tablesDiff.join(','));
+    }
+
+
+    function tableNamesInSurrogateKeyHeader(expectedTableNames, done) {
+        return function(err, res) {
+            surrogateKeyHasTables(res.headers['surrogate-key'], expectedTableNames);
+            done();
+        };
+    }
+
+    it('supports joins', function(done) {
+        var sql = "SELECT a.name as an, b.name as bn FROM untitle_table_4 a " +
+                "left join private_table b ON (a.cartodb_id = b.cartodb_id)";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'},
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
+        ], done));
+    });
+
+    it('supports multistatements', function(done) {
+        var sql = "SELECT * FROM untitle_table_4; SELECT * FROM private_table";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'},
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
+        ], done));
+    });
+
+    it('supports explicit transactions', function(done) {
+        var sql =  "BEGIN; SELECT * FROM untitle_table_4; COMMIT; BEGIN; SELECT * FROM private_table; COMMIT;";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'},
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
+        ], done));
+    });
+
+    it('survives partial transactions', function(done) {
+        var sql = "BEGIN; SELECT * FROM untitle_table_4";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
+            {dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
+        ], done));
+    });
+
+    it('should not add header for functions', function(done) {
+        var sql = "SELECT format('%s', 'wadus')";
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
+            assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']);
+            done();
+        });
+    });
+
+    it('should not add header for CDB_QueryTables', function(done) {
+        var sql = "SELECT CDB_QueryTablesText('select * from untitle_table_4')";
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
+            assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']);
+            done();
+        });
+    });
+
+    it('should not add header for non table results', function(done) {
+        var sql = "SELECT 'wadus'::text";
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
+            assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']);
+            done();
+        });
+    });
+
+});

test/acceptance/system-queries.js

@@ -0,0 +1,64 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+
+
+describe('system-queries', function() {
+
+    var systemQueriesSuitesToTest = [
+        {
+            desc: 'pg_ queries work with api_key and fail otherwise',
+            queries: [
+                'SELECT * FROM pg_attribute',
+                'SELECT * FROM PG_attribute',
+                'SELECT * FROM "pg_attribute"',
+                'SELECT a.* FROM untitle_table_4 a,pg_attribute',
+                'SELECT * FROM geometry_columns'
+            ],
+            api_key_works: true,
+            no_api_key_works: false
+        },
+        {
+            desc: 'Possible false positive queries will work with api_key and without it',
+            queries: [
+                "SELECT 'pg_'",
+                'SELECT pg_attribute FROM ( select 1 as pg_attribute ) as f',
+                'SELECT * FROM cpg_test'
+            ],
+            api_key_works: true,
+            no_api_key_works: true
+        }
+    ];
+
+    systemQueriesSuitesToTest.forEach(function(suiteToTest) {
+        var apiKeyStatusErrorCode = !!suiteToTest.api_key_works ? 200 : 403;
+        testSystemQueries(suiteToTest.desc + ' with api_key', suiteToTest.queries, apiKeyStatusErrorCode, '1234');
+        var noApiKeyStatusErrorCode = !!suiteToTest.no_api_key_works ? 200 : 403;
+        testSystemQueries(suiteToTest.desc, suiteToTest.queries, noApiKeyStatusErrorCode);
+    });
+
+    function testSystemQueries(description, queries, statusErrorCode, apiKey) {
+        queries.forEach(function(query) {
+            it('[' + description + ']  query: ' + query, function(done) {
+                var queryStringParams = {q: query};
+                if (!!apiKey) {
+                    queryStringParams.api_key = apiKey;
+                }
+                var request = {
+                    headers: {host: 'vizzuality.cartodb.com'},
+                    method: 'GET',
+                    url: '/api/v1/sql?' + querystring.stringify(queryStringParams)
+                };
+                assert.response(server, request, function(err, response) {
+                    assert.equal(response.statusCode, statusErrorCode);
+                    done();
+                });
+            });
+        });
+    }
+
+});

test/acceptance/timeout.js

@@ -0,0 +1,55 @@
+'use strict';
+
+/**
+ *
+ * Requires the database and tables setup in config/environments/test.js to exist
+ * Ensure the user is present in the pgbouncer auth file too
+ * TODO: Add OAuth tests.
+ *
+ * To run this test, ensure that cartodb_test_user_1_db metadata exists
+ * in Redis for the vizzuality.cartodb.com domain
+ *
+ * SELECT 5
+ * HSET rails:users:vizzuality id 1
+ * HSET rails:users:vizzuality database_name cartodb_test_user_1_db
+ *
+ */
+require('../helper');
+
+
+var assert = require('../support/assert');
+var step = require('step');
+
+describe('timeout', function() {
+
+// See https://github.com/CartoDB/CartoDB-SQL-API/issues/128
+it('after configured milliseconds', function(done){
+  var testTimeout = 10;
+//console.log("settings:"); console.dir(global.settings);
+  var timeoutBackup = global.settings.node_socket_timeout;
+  global.settings.node_socket_timeout = testTimeout;
+  var server = require('../../app/server')();
+  step(
+    function sendLongQuery() {
+      assert.response(server, {
+          url: '/api/v1/sql?q=SELECT+count(*)+FROM+generate_series(1,100000)',
+          method: 'GET',
+          headers: {host: 'vizzuality.localhost' }
+      },{}, this);
+    },
+    function checkResponse(err/*, res*/) {
+      assert.ok(err);
+      assert.ok(err.message.match(/hang up/), err);
+      return null;
+    },
+    function finish(err) {
+      global.settings.node_socket_timeout = timeoutBackup;
+      done(err);
+    }
+  );
+});
+
+// TODO: check that the query is interrupted on timeout!
+//See #129
+
+});

test/acceptance/transaction.js

@@ -0,0 +1,56 @@
+'use strict';
+
+require('../helper');
+
+var assert = require('../support/assert');
+var qs = require('querystring');
+var request = require('request');
+
+describe('transaction', function() {
+
+    var SERVER_PORT = 5554;
+
+    var server;
+    before(function(done) {
+        server = require('../../app/server')();
+        this.listener = server.listen(SERVER_PORT, '127.0.0.1');
+        this.listener.on('listening', done);
+    });
+
+    after(function(done) {
+        this.listener.close(done);
+    });
+
+    var sqlRequest = request.defaults({
+        headers: { host: 'vizzuality.localhost' }
+    });
+
+    function requestUrl(query) {
+        return 'http://127.0.0.1:' + SERVER_PORT + '/api/v1/sql?' + qs.stringify({ q: query });
+    }
+
+    var errorQuery = 'BEGIN; PREPARE _pstm AS select error; EXECUTE _pstm; COMMIT;';
+
+    it('should NOT fail to second request after error in transaction', function(done) {
+        sqlRequest(requestUrl(errorQuery), function(err, response, body) {
+            assert.ok(!err);
+            assert.equal(response.statusCode, 400);
+
+            var parsedBody = JSON.parse(body);
+            assert.ok(parsedBody);
+            assert.deepEqual(parsedBody, { error: ['column "error" does not exist'] });
+
+            sqlRequest(requestUrl('select 1 as foo'), function (err, response, body) {
+                assert.ok(!err);
+                assert.equal(response.statusCode, 200);
+
+                var parsedBody = JSON.parse(body);
+                assert.ok(parsedBody);
+                assert.deepEqual(parsedBody.rows, [{ foo: 1 }]);
+
+                done();
+            });
+        });
+    });
+
+});

test/acceptance/x-cache-channel.js

@@ -0,0 +1,112 @@
+'use strict';
+
+require('../helper');
+
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+var querystring = require('querystring');
+var _ = require('underscore');
+
+describe('X-Cache-Channel header', function() {
+
+    function createGetRequest(sqlQuery) {
+        var query = querystring.stringify({
+            q: sqlQuery,
+            api_key: 1234
+        });
+        return {
+            url: '/api/v1/sql?' + query,
+            headers: {
+                host: 'vizzuality.cartodb.com'
+            },
+            method: 'GET'
+        };
+    }
+
+    var RESPONSE_OK = {
+        statusCode: 200
+    };
+
+    function xCacheChannelHeaderHasTables(xCacheChannel, expectedTablesNames) {
+        var databaseAndTables = xCacheChannel.split(':');
+        var databaseName = databaseAndTables[0];
+
+        assert.equal(databaseName, 'cartodb_test_user_1_db');
+
+        var headerTableNames = databaseAndTables[1].split(',');
+        assert.equal(headerTableNames.length, expectedTablesNames.length);
+
+        var tablesDiff = _.difference(expectedTablesNames, headerTableNames);
+        assert.equal(tablesDiff.length, 0, 'X-Cache-Channel header missing tables: ' + tablesDiff.join(','));
+    }
+
+    function tableNamesInCacheChannelHeader(expectedTableNames, done) {
+        return function(err, res) {
+            xCacheChannelHeaderHasTables(res.headers['x-cache-channel'], expectedTableNames);
+            done();
+        };
+    }
+
+    it('supports joins', function(done) {
+        var sql = "SELECT a.name as an, b.name as bn FROM untitle_table_4 a " +
+            "left join private_table b ON (a.cartodb_id = b.cartodb_id)";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
+            'public.private_table',
+            'public.untitle_table_4'
+        ], done));
+    });
+
+    it('supports multistatements', function(done) {
+        var sql = "SELECT * FROM untitle_table_4; SELECT * FROM private_table";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
+            'public.private_table',
+            'public.untitle_table_4'
+        ], done));
+    });
+
+    it('supports explicit transactions', function(done) {
+        var sql =  "BEGIN; SELECT * FROM untitle_table_4; COMMIT; BEGIN; SELECT * FROM private_table; COMMIT;";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
+            'public.private_table',
+            'public.untitle_table_4'
+        ], done));
+    });
+
+    it('survives partial transactions', function(done) {
+        var sql = "BEGIN; SELECT * FROM untitle_table_4";
+
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
+            'public.untitle_table_4'
+        ], done));
+    });
+
+    it('should not add header for functions', function(done) {
+        var sql = "SELECT format('%s', 'wadus')";
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
+            assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']);
+            done();
+        });
+    });
+
+    it('should not add header for CDB_QueryTables', function(done) {
+        var sql = "SELECT CDB_QueryTablesText('select * from untitle_table_4')";
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
+            assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']);
+            done();
+        });
+    });
+
+    it('should not add header for non table results', function(done) {
+        var sql = "SELECT 'wadus'::text";
+        assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
+            assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']);
+            done();
+        });
+    });
+
+
+
+});

test/helper.js

@@ -0,0 +1,4 @@
+'use strict';
+
+global.settings = require('../config/environments/test');
+process.env.NODE_ENV = 'test';

test/integration/batch/job-queue.test.js

@@ -0,0 +1,179 @@
+'use strict';
+
+require('../../helper');
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+var JobPublisher = require('../../../batch/pubsub/job-publisher');
+var JobQueue = require('../../../batch/job_queue');
+
+var JobBackend = require('../../../batch/job_backend');
+var JobService = require('../../../batch/job_service');
+var JobCanceller = require('../../../batch/job_canceller');
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+
+describe('job queue', function () {
+    var pool = redisUtils.getPool();
+    var jobPublisher = new JobPublisher(pool);
+    var jobQueue =  new JobQueue(metadataBackend, jobPublisher);
+    var jobBackend = new JobBackend(metadataBackend, jobQueue);
+    var jobCanceller = new JobCanceller();
+    var jobService = new JobService(jobBackend, jobCanceller);
+
+    var userA = 'userA';
+    var userB = 'userB';
+
+    beforeEach(function () {
+        this.jobQueue = new JobQueue(metadataBackend, jobPublisher);
+    });
+
+    afterEach(function (done) {
+        redisUtils.clean('batch:*', done);
+    });
+
+    it('should find queues for one user', function (done) {
+        var self = this;
+
+        this.jobQueue.enqueue(userA, 'wadus-wadus-wadus-wadus', function(err) {
+            if (err) {
+                return done(err);
+            }
+
+            self.jobQueue.scanQueues(function (err, queues) {
+                assert.ifError(err);
+                assert.equal(queues.length, 1);
+                assert.equal(queues[0], userA);
+                return done();
+            });
+        });
+    });
+
+    it('should find queues for more than one user', function (done) {
+        var self = this;
+
+        this.jobQueue.enqueue(userA, 'wadus-wadus-wadus-wadus', function(err) {
+            if (err) {
+                return done(err);
+            }
+            self.jobQueue.enqueue(userB, 'wadus-wadus-wadus-wadus', function(err) {
+                if (err) {
+                    return done(err);
+                }
+
+                self.jobQueue.scanQueues(function (err, queues) {
+                    assert.ifError(err);
+                    assert.equal(queues.length, 2);
+                    assert.ok(queues[0] === userA || queues[0] === userB);
+                    assert.ok(queues[1] === userA || queues[1] === userB);
+
+                    return done();
+                });
+            });
+        });
+    });
+
+    it('should find queues from jobs not using new Redis SETs for users', function(done) {
+        var self = this;
+        var redisArgs = [JobQueue.QUEUE.PREFIX + userA, 'wadus-id'];
+        metadataBackend.redisCmd(JobQueue.QUEUE.DB, 'LPUSH', redisArgs, function (err) {
+            assert.ok(!err, err);
+            self.jobQueue.scanQueues(function (err, queues) {
+                assert.ok(!err, err);
+
+                assert.equal(queues.length, 1);
+                assert.equal(queues[0], userA);
+
+                return done();
+            });
+        });
+    });
+
+    it('.scanQueues() should feed queue index', function (done) {
+        var self = this;
+
+        var data = {
+            user: 'vizzuality',
+            query: 'select 1 as cartodb_id',
+            host: 'localhost'
+        };
+
+        jobService.create(data, function (err) {
+            if (err) {
+                return done(err);
+            }
+
+            self.jobQueue.scanQueues(function (err, queuesFromScan) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(queuesFromScan.length, 1);
+                assert.ok(queuesFromScan.indexOf(data.user) >= 0);
+
+                self.jobQueue.getQueues(function (err, queuesFromIndex) {
+                    if (err) {
+                        done(err);
+                    }
+
+                    assert.equal(queuesFromIndex.length, 1);
+                    assert.ok(queuesFromIndex.indexOf(data.user) >= 0);
+
+                    redisUtils.clean('batch:*', done);
+                });
+
+            });
+        });
+    });
+
+    it('.scanQueues() should feed queue index with two users', function (done) {
+        var self = this;
+
+        var jobVizzuality = {
+            user: 'vizzuality',
+            query: 'select 1 as cartodb_id',
+            host: 'localhost'
+        };
+
+        var jobWadus = {
+            user: 'wadus',
+            query: 'select 1 as cartodb_id',
+            host: 'localhost'
+        };
+
+        jobService.create(jobVizzuality, function (err) {
+            if (err) {
+                return done(err);
+            }
+
+            jobService.create(jobWadus, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                self.jobQueue.scanQueues(function (err, queuesFromScan) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(queuesFromScan.length, 2);
+                    assert.ok(queuesFromScan.indexOf(jobVizzuality.user) >= 0);
+                    assert.ok(queuesFromScan.indexOf(jobWadus.user) >= 0);
+
+                    self.jobQueue.getQueues(function (err, queuesFromIndex) {
+                        if (err) {
+                            done(err);
+                        }
+
+                        assert.equal(queuesFromIndex.length, 2);
+                        assert.ok(queuesFromIndex.indexOf(jobVizzuality.user) >= 0);
+                        assert.ok(queuesFromIndex.indexOf(jobWadus.user) >= 0);
+
+                        redisUtils.clean('batch:*', done);
+                    });
+
+                });
+            });
+        });
+    });
+});

test/integration/batch/job_backend.test.js

@@ -0,0 +1,219 @@
+'use strict';
+
+require('../../helper');
+
+var BATCH_SOURCE = '../../../batch/';
+
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+
+var JobQueue = require(BATCH_SOURCE + 'job_queue');
+var JobBackend = require(BATCH_SOURCE + 'job_backend');
+var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher');
+var JobFactory = require(BATCH_SOURCE + 'models/job_factory');
+var jobStatus = require(BATCH_SOURCE + 'job_status');
+
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+var jobPublisher = new JobPublisher(redisUtils.getPool());
+var jobQueue =  new JobQueue(metadataBackend, jobPublisher);
+
+var queue = require('queue-async');
+
+var USER = 'vizzuality';
+var QUERY = 'select pg_sleep(0)';
+var HOST = 'localhost';
+var JOB = {
+    user: USER,
+    query: QUERY,
+    host: HOST
+};
+
+function createWadusJob() {
+    return JobFactory.create(JSON.parse(JSON.stringify(JOB)));
+}
+
+describe('job backend', function() {
+    var jobBackend = new JobBackend(metadataBackend, jobQueue);
+
+    after(function (done) {
+        redisUtils.clean('batch:*', done);
+    });
+
+    it('.create() should persist a job', function (done) {
+        var job = createWadusJob();
+
+        jobBackend.create(job.data, function (err, jobCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(jobCreated.job_id);
+            assert.equal(jobCreated.status, jobStatus.PENDING);
+            done();
+        });
+    });
+
+    it('.create() should return error', function (done) {
+        var job = createWadusJob();
+
+        delete job.data.job_id;
+
+        jobBackend.create(job.data, function (err) {
+            assert.ok(err);
+            assert.equal(err.name, 'NotFoundError');
+            assert.equal(err.message, 'Job with id undefined not found');
+            done();
+        });
+    });
+
+    it('.get() should return a job with the given id', function (done) {
+        var jobData = createWadusJob();
+
+        jobBackend.create(jobData.data, function (err, jobCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(jobCreated.job_id);
+
+            jobBackend.get(jobCreated.job_id, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.job_id, jobCreated.job_id);
+                assert.equal(job.user, jobData.data.user);
+                assert.equal(job.query, jobData.data.query);
+                assert.equal(job.host, jobData.data.host);
+                assert.equal(job.status, jobStatus.PENDING);
+                done();
+            });
+        });
+    });
+
+    it('.update() should update an existent job', function (done) {
+        var job = createWadusJob();
+
+        jobBackend.create(job.data, function (err, jobCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            jobCreated.query = 'select pg_sleep(1)';
+
+            var job = JobFactory.create(jobCreated);
+
+            jobBackend.update(job.data, function (err, jobUpdated) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(jobUpdated.query, 'select pg_sleep(1)');
+                done();
+            });
+        });
+    });
+
+    it('.update() should return error when updates a nonexistent job', function (done) {
+        var job = createWadusJob();
+
+        jobBackend.update(job.data, function (err) {
+            assert.ok(err, err);
+            assert.equal(err.name, 'NotFoundError');
+            assert.equal(err.message, 'Job with id ' + job.data.job_id + ' not found');
+            done();
+        });
+    });
+
+    it('.save() should save a job', function (done) {
+        var job = createWadusJob();
+
+        jobBackend.save(job.data, function (err, jobSaved) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(jobSaved.job_id);
+
+            assert.equal(jobSaved.user, job.data.user);
+            assert.equal(jobSaved.query, job.data.query);
+            assert.equal(jobSaved.host, job.data.host);
+            assert.equal(jobSaved.status, jobStatus.PENDING);
+            done();
+        });
+    });
+
+    it('.addWorkInProgressJob() should add current job to user and host lists', function (done) {
+        var job = createWadusJob();
+
+        jobBackend.addWorkInProgressJob(job.data.user, job.data.job_id, function (err) {
+            if (err) {
+                return done(err);
+            }
+            done();
+        });
+    });
+
+    it('.listWorkInProgressJobByUser() should retrieve WIP jobs of given user', function (done) {
+        var testStepsQueue = queue(1);
+
+        testStepsQueue.defer(redisUtils.clean, 'batch:wip:user:*');
+        testStepsQueue.defer(jobBackend.addWorkInProgressJob.bind(jobBackend), 'vizzuality', 'wadus');
+        testStepsQueue.defer(jobBackend.listWorkInProgressJobByUser.bind(jobBackend), 'vizzuality');
+
+        testStepsQueue.awaitAll(function (err, results) {
+            if (err) {
+                return done(err);
+            }
+            assert.deepEqual(results[2], ['wadus']);
+            done();
+        });
+    });
+
+    it('.listWorkInProgressJobs() should retrieve WIP users', function (done) {
+        var jobs = [{ user: 'userA', id: 'jobId1' }, { user: 'userA', id: 'jobId2' }, { user: 'userB', id: 'jobId3' }];
+
+        var testStepsQueue = queue(1);
+
+        jobs.forEach(function (job) {
+            testStepsQueue.defer(jobBackend.addWorkInProgressJob.bind(jobBackend), job.user, job.id);
+        });
+
+        testStepsQueue.awaitAll(function (err) {
+            if (err) {
+                done(err);
+            }
+
+            jobBackend.listWorkInProgressJobs(function (err, users) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.ok(users.userA);
+                assert.deepEqual(users.userA, [ 'jobId1', 'jobId2' ]);
+                assert.ok(users.userB);
+                assert.deepEqual(users.userB, [ 'jobId3' ]);
+                done();
+            });
+
+        });
+    });
+
+    it('.clearWorkInProgressJob() should remove job from work in progress list', function (done) {
+        var job = createWadusJob();
+
+        jobBackend.addWorkInProgressJob(job.data.user, job.data.job_id, function (err) {
+            if (err) {
+                return done(err);
+            }
+
+            jobBackend.clearWorkInProgressJob(job.data.user, job.data.job_id, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                done();
+            });
+        });
+    });
+});

test/integration/batch/job_canceller.test.js

@@ -0,0 +1,119 @@
+'use strict';
+
+require('../../helper');
+
+var BATCH_SOURCE = '../../../batch/';
+
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+
+var JobQueue = require(BATCH_SOURCE + 'job_queue');
+var JobBackend = require(BATCH_SOURCE + 'job_backend');
+var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher');
+var jobStatus = require(BATCH_SOURCE + 'job_status');
+var JobCanceller = require(BATCH_SOURCE + 'job_canceller');
+var PSQL = require('cartodb-psql');
+
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+var jobPublisher = new JobPublisher(redisUtils.getPool());
+var jobQueue =  new JobQueue(metadataBackend, jobPublisher);
+var jobBackend = new JobBackend(metadataBackend, jobQueue);
+var JobFactory = require(BATCH_SOURCE + 'models/job_factory');
+
+var USER = 'vizzuality';
+var QUERY = 'select pg_sleep(0)';
+var HOST = 'localhost';
+
+// sets job to running, run its query and returns inmediatly (don't wait for query finishes)
+// in order to test query cancelation/draining
+function runQueryHelper(job, callback) {
+    var job_id = job.job_id;
+    var sql = job.query;
+
+    job.status = jobStatus.RUNNING;
+
+    jobBackend.update(job, function (err) {
+        if (err) {
+            return callback(err);
+        }
+
+        const dbConfiguration = {
+            host: job.host,
+            port: job.port,
+            dbname: job.dbname,
+            user: job.dbuser,
+            pass: job.pass,
+        };
+
+        const pg = new PSQL(dbConfiguration);
+
+        sql = '/* ' + job_id + ' */ ' + sql;
+
+        pg.eventedQuery(sql, function (err, query) {
+            if (err) {
+                return callback(err);
+            }
+
+            callback(null, query);
+        });
+    });
+}
+
+function createWadusJob(query) {
+    query = query || QUERY;
+    return JobFactory.create(JSON.parse(JSON.stringify({
+        user: USER,
+        query: query,
+        host: HOST,
+        dbname: 'cartodb_test_user_1_db',
+        dbuser: 'test_cartodb_user_1',
+        port: 5432,
+        pass: 'test_cartodb_user_1_pass',
+    })));
+}
+
+describe('job canceller', function() {
+    var jobCanceller = new JobCanceller();
+
+    after(function (done) {
+        redisUtils.clean('batch:*', done);
+    });
+
+    it('.cancel() should cancel a job', function (done) {
+        var job = createWadusJob('select pg_sleep(1)');
+
+        jobBackend.create(job.data, function (err, jobCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.equal(job.data.job_id, jobCreated.job_id);
+
+            runQueryHelper(job.data, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobCanceller.cancel(job, function (err) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    done();
+                });
+            });
+        });
+    });
+
+    it('.cancel() a non running job should not return an error', function (done) {
+        var job = createWadusJob();
+
+        jobCanceller.cancel(job, function (err) {
+            if (err) {
+                return done(err);
+            }
+
+            done();
+        });
+    });
+});

test/integration/batch/job_publisher.test.js

@@ -0,0 +1,39 @@
+'use strict';
+
+require('../../helper');
+
+var BATCH_SOURCE = '../../../batch/';
+
+var assert = require('../../support/assert');
+
+var redisUtils = require('../../support/redis_utils');
+
+
+var Channel = require(BATCH_SOURCE + 'pubsub/channel');
+var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher');
+
+var HOST = 'wadus';
+
+describe('job publisher', function() {
+    var jobPublisher = new JobPublisher(redisUtils.getPool());
+
+    it('.publish() should publish in job channel', function (done) {
+        redisUtils.getPool().acquire(Channel.DB, function (err, client) {
+            if (err) {
+                return done(err);
+            }
+
+            client.subscribe(Channel.NAME);
+
+            client.on('message', function (channel, host) {
+                assert.equal(host, HOST);
+                assert.equal(channel, Channel.NAME);
+                client.unsubscribe(Channel.NAME);
+                done();
+            });
+
+            jobPublisher.publish(HOST);
+        });
+    });
+
+});

test/integration/batch/job_runner.test.js

@@ -0,0 +1,80 @@
+'use strict';
+
+require('../../helper');
+
+var BATCH_SOURCE = '../../../batch/';
+
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+
+var JobQueue = require(BATCH_SOURCE + 'job_queue');
+var JobBackend = require(BATCH_SOURCE + 'job_backend');
+var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher');
+var jobStatus = require(BATCH_SOURCE + 'job_status');
+var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user_database_metadata_service');
+var JobCanceller = require(BATCH_SOURCE + 'job_canceller');
+var JobService = require(BATCH_SOURCE + 'job_service');
+var JobRunner = require(BATCH_SOURCE + 'job_runner');
+var QueryRunner = require(BATCH_SOURCE + 'query_runner');
+
+
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+var jobPublisher = new JobPublisher(redisUtils.getPool());
+var jobQueue =  new JobQueue(metadataBackend, jobPublisher);
+var jobBackend = new JobBackend(metadataBackend, jobQueue);
+var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend);
+var jobCanceller = new JobCanceller();
+var jobService = new JobService(jobBackend, jobCanceller);
+var queryRunner = new QueryRunner(userDatabaseMetadataService);
+var StatsD = require('node-statsd').StatsD;
+var statsdClient = new StatsD(global.settings.statsd);
+
+var USER = 'vizzuality';
+var QUERY = 'select pg_sleep(0)';
+var HOST = 'localhost';
+var JOB = {
+    user: USER,
+    query: QUERY,
+    host: HOST,
+    dbname: 'cartodb_test_user_1_db',
+    dbuser: 'test_cartodb_user_1',
+    port: 5432,
+    pass: 'test_cartodb_user_1_pass',
+};
+
+describe('job runner', function() {
+    var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient);
+
+    after(function (done) {
+        redisUtils.clean('batch:*', function() {
+            redisUtils.clean('limits:batch:*', done);
+        });
+    });
+
+    it('.run() should run a job', function (done) {
+        jobService.create(JOB, function (err, job) {
+            if (err) {
+                return done(err);
+            }
+
+            jobRunner.run(job.data.job_id, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.data.status, jobStatus.DONE);
+                done();
+            });
+        });
+    });
+
+    it('.run() should return a job not found error', function (done) {
+        jobRunner.run('wadus_job_id', function (err) {
+            assert.ok(err, err);
+            assert.equal(err.name, 'NotFoundError');
+            assert.equal(err.message, 'Job with id wadus_job_id not found');
+            done();
+        });
+    });
+
+});

test/integration/batch/job_service.test.js

@@ -0,0 +1,205 @@
+'use strict';
+
+require('../../helper');
+
+var BATCH_SOURCE = '../../../batch/';
+
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+
+var JobQueue = require(BATCH_SOURCE + 'job_queue');
+var JobBackend = require(BATCH_SOURCE + 'job_backend');
+var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher');
+var jobStatus = require(BATCH_SOURCE + 'job_status');
+var JobCanceller = require(BATCH_SOURCE + 'job_canceller');
+var JobService = require(BATCH_SOURCE + 'job_service');
+var PSQL = require('cartodb-psql');
+
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+var jobPublisher = new JobPublisher(redisUtils.getPool());
+var jobQueue =  new JobQueue(metadataBackend, jobPublisher);
+var jobBackend = new JobBackend(metadataBackend, jobQueue);
+var jobCanceller = new JobCanceller();
+
+var USER = 'vizzuality';
+var QUERY = 'select pg_sleep(0)';
+var HOST = 'localhost';
+var JOB = {
+    user: USER,
+    query: QUERY,
+    host: HOST,
+    dbname: 'cartodb_test_user_1_db',
+    dbuser: 'test_cartodb_user_1',
+    port: 5432,
+    pass: 'test_cartodb_user_1_pass',
+
+};
+
+function createWadusDataJob() {
+    return JSON.parse(JSON.stringify(JOB));
+}
+
+// sets job to running, run its query and returns inmediatly (don't wait for query finishes)
+// in order to test query cancelation/draining
+function runQueryHelper(job, callback) {
+    var job_id = job.job_id;
+    var sql = job.query;
+
+    job.status = jobStatus.RUNNING;
+
+    jobBackend.update(job, function (err) {
+        if (err) {
+            return callback(err);
+        }
+
+        const dbConfiguration = {
+            host: job.host,
+            port: job.port,
+            dbname: job.dbname,
+            user: job.dbuser,
+            pass: job.pass,
+        };
+
+        var pg = new PSQL(dbConfiguration);
+
+        sql = '/* ' + job_id + ' */ ' + sql;
+
+        pg.eventedQuery(sql, function (err, query) {
+            if (err) {
+                return callback(err);
+            }
+
+            callback(null, query);
+        });
+    });
+}
+
+describe('job service', function() {
+    var jobService = new JobService(jobBackend, jobCanceller);
+
+    after(function (done) {
+        redisUtils.clean('batch:*', done);
+    });
+
+    it('.get() should return a job', function (done) {
+        jobService.create(createWadusDataJob(), function (err, jobCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            jobService.get(jobCreated.data.job_id, function (err, job) {
+                if (err) {
+                    return done(err);
+                }
+
+                assert.equal(job.data.job_id, jobCreated.data.job_id);
+                done();
+            });
+        });
+    });
+
+    it('.get() should return a not found error', function (done) {
+        jobService.get('wadus_job_id', function (err) {
+            assert.ok(err);
+            assert.equal(err.message, 'Job with id wadus_job_id not found');
+            done();
+        });
+    });
+
+    it('.create() should persist a job', function (done) {
+        jobService.create(createWadusDataJob(), function (err, jobCreated) {
+            if (err) {
+                return done(err);
+            }
+
+            assert.ok(jobCreated.data.job_id);
+            assert.equal(jobCreated.data.status, jobStatus.PENDING);
+            done();
+        });
+    });
+
+    it('.create() should return error with invalid job data', function (done) {
+        var job = createWadusDataJob();
+
+        delete job.query;
+
+        jobService.create(job, function (err) {
+            assert.ok(err);
+            assert.equal(err.message, 'You must indicate a valid SQL');
+            done();
+        });
+    });
+
+    it('.cancel() should cancel a running job', function (done) {
+        var job = createWadusDataJob();
+        job.query = 'select pg_sleep(3)';
+
+        jobService.create(job, function (err, job) {
+            if (err) {
+                return done(err);
+            }
+
+            runQueryHelper(job.data, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobService.cancel(job.data.job_id, function (err, jobCancelled) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(jobCancelled.data.job_id, job.data.job_id);
+                    assert.equal(jobCancelled.data.status, jobStatus.CANCELLED);
+                    done();
+                });
+            });
+        });
+    });
+
+    it('.cancel() should return a job not found error', function (done) {
+        jobService.cancel('wadus_job_id', function (err) {
+            assert.ok(err, err);
+            assert.equal(err.name, 'NotFoundError');
+            assert.equal(err.message, 'Job with id wadus_job_id not found');
+            done();
+        });
+    });
+
+    it('.drain() should draing a running job', function (done) {
+        var job = createWadusDataJob();
+        job.query = 'select pg_sleep(3)';
+
+        jobService.create(job, function (err, job) {
+            if (err) {
+                return done(err);
+            }
+
+            runQueryHelper(job.data, function (err) {
+                if (err) {
+                    return done(err);
+                }
+
+                jobService.drain(job.data.job_id, function (err, jobDrained) {
+                    if (err) {
+                        return done(err);
+                    }
+
+                    assert.equal(jobDrained.job_id, job.data.job_id);
+                    assert.equal(jobDrained.status, jobStatus.PENDING);
+                    done();
+                });
+            });
+        });
+    });
+
+    it('.drain() should return a job not found error', function (done) {
+        jobService.drain('wadus_job_id', function (err) {
+            assert.ok(err, err);
+            assert.equal(err.name, 'NotFoundError');
+            assert.equal(err.message, 'Job with id wadus_job_id not found');
+            done();
+        });
+    });
+
+});

test/integration/batch/locker.js

@@ -0,0 +1,60 @@
+'use strict';
+
+require('../../helper');
+
+var assert = require('../../support/assert');
+var redisUtils = require('../../support/redis_utils');
+var Locker = require('../../../batch/leader/locker');
+
+describe('locker', function() {
+    var host = 'localhost';
+
+    var TTL = 500;
+
+    var config = { ttl: TTL, pool: redisUtils.getPool() };
+
+    it('should lock and unlock', function (done) {
+        var lockerA = Locker.create('redis-distlock', config);
+        var lockerB = Locker.create('redis-distlock', config);
+        lockerA.lock(host, function(err, lock) {
+            if (err) {
+                return done(err);
+            }
+            assert.ok(lock);
+
+            // others can't lock on same host
+            lockerB.lock(host, function(err) {
+                assert.ok(err);
+                assert.equal(err.name, 'LockError');
+
+                lockerA.unlock(host, function(err) {
+                    assert.ok(!err);
+                    // others can lock after unlock
+                    lockerB.lock(host, function(err, lock2) {
+                        assert.ok(!err);
+                        assert.ok(lock2);
+                        lockerB.unlock(host, done);
+                    });
+                });
+            });
+        });
+    });
+
+    it('should lock and keep locking until unlock', function (done) {
+        var lockerA = Locker.create('redis-distlock', config);
+        var lockerB = Locker.create('redis-distlock', config);
+        lockerA.lock(host, function(err, lock) {
+            if (err) {
+                return done(err);
+            }
+            setTimeout(function() {
+                lockerB.lock(host, function(err) {
+                    assert.ok(err);
+
+                    assert.ok(lock);
+                    lockerA.unlock(host, done);
+                });
+            }, 2 * TTL);
+        });
+    });
+});

test/integration/batch/scheduler.js

@@ -0,0 +1,204 @@
+'use strict';
+
+require('../../helper');
+var debug = require('../../../batch/util/debug')('scheduler-test');
+var assert = require('../../support/assert');
+var Scheduler = require('../../../batch/scheduler/scheduler');
+var FixedCapacity = require('../../../batch/scheduler/capacity/fixed');
+
+describe('scheduler', function() {
+
+    var USER_FINISHED = true;
+
+    var USER_A = 'userA';
+    var USER_B = 'userB';
+    var USER_C = 'userC';
+
+    function TaskRunner(userTasks) {
+        this.results = [];
+        this.userTasks = userTasks;
+    }
+
+    TaskRunner.prototype.run = function(user, callback) {
+        this.results.push(user);
+        this.userTasks[user]--;
+        setTimeout(function() {
+            return callback(null, this.userTasks[user] === 0);
+        }.bind(this), 50);
+    };
+
+    function ManualTaskRunner() {
+        this.userTasks = {};
+    }
+
+    ManualTaskRunner.prototype.run = function(user, callback) {
+        if (!this.userTasks.hasOwnProperty(user)) {
+            this.userTasks[user] = [];
+        }
+        this.userTasks[user].push(callback);
+    };
+
+    ManualTaskRunner.prototype.dispatch = function(user, isDone) {
+        if (this.userTasks.hasOwnProperty(user)) {
+            var cb = this.userTasks[user].shift();
+            if (cb) {
+                return cb(null, isDone);
+            }
+        }
+    };
+
+
+    // simulate one by one or infinity capacity
+    var capacities = [new FixedCapacity(1), new FixedCapacity(2), new FixedCapacity(Infinity)];
+
+    capacities.forEach(function(capacity) {
+
+        it('regression #1', function (done) {
+            var taskRunner = new TaskRunner({
+                userA: 2,
+                userB: 2
+            });
+            var scheduler = new Scheduler(capacity, taskRunner);
+            scheduler.add(USER_A);
+            scheduler.add(USER_B);
+
+            scheduler.on('done', function() {
+                var results = taskRunner.results;
+
+                assert.equal(results.length, 4);
+
+                assert.equal(results[0], USER_A);
+                assert.equal(results[1], USER_B);
+                assert.equal(results[2], USER_A);
+                assert.equal(results[3], USER_B);
+
+                return done();
+            });
+
+            scheduler.schedule();
+        });
+
+        it('regression #2: it should restart task after it was done but got re-scheduled', function (done) {
+            var taskRunner = new ManualTaskRunner();
+            var scheduler = new Scheduler(capacity, taskRunner);
+            debug('Adding users A and B');
+            scheduler.add(USER_A);
+            scheduler.add(USER_B);
+
+            var acquiredUsers = [];
+
+            scheduler.on('done', function() {
+                debug('Users %j', acquiredUsers);
+                assert.equal(acquiredUsers[0], USER_A);
+                assert.equal(acquiredUsers[1], USER_B);
+                assert.equal(acquiredUsers[2], USER_A);
+                assert.equal(acquiredUsers[3], USER_B);
+
+                assert.equal(acquiredUsers.length, 4);
+
+                return done();
+            });
+
+            scheduler.on('acquired', function(user) {
+                debug('Acquired user %s', user);
+                acquiredUsers.push(user);
+            });
+
+            scheduler.schedule();
+
+            debug('User A will be mark as DONE');
+            taskRunner.dispatch(USER_A, USER_FINISHED);
+
+            debug('User B should be running');
+            debug('User A submit a new task');
+            scheduler.add(USER_A);
+
+            debug('User B will get another task to run');
+            taskRunner.dispatch(USER_B);
+
+            debug('User A should start working on this new task');
+            taskRunner.dispatch(USER_A, USER_FINISHED);
+            taskRunner.dispatch(USER_B, USER_FINISHED);
+        });
+
+        it('should run tasks', function (done) {
+            var taskRunner = new TaskRunner({
+                userA: 1
+            });
+            var scheduler = new Scheduler(capacity, taskRunner);
+            scheduler.add(USER_A);
+
+            scheduler.on('done', function() {
+                var results = taskRunner.results;
+
+                assert.equal(results.length, 1);
+
+                assert.equal(results[0], USER_A);
+
+                return done();
+            });
+
+            scheduler.schedule();
+        });
+
+
+        it('should run tasks for different users', function (done) {
+            var taskRunner = new TaskRunner({
+                userA: 1,
+                userB: 1,
+                userC: 1
+            });
+            var scheduler = new Scheduler(capacity, taskRunner);
+            scheduler.add(USER_A);
+            scheduler.add(USER_B);
+            scheduler.add(USER_C);
+
+            scheduler.on('done', function() {
+                var results = taskRunner.results;
+
+                assert.equal(results.length, 3);
+
+                assert.equal(results[0], USER_A);
+                assert.equal(results[1], USER_B);
+                assert.equal(results[2], USER_C);
+
+                return done();
+            });
+
+            scheduler.schedule();
+        });
+
+        it('should be fair when scheduling tasks', function (done) {
+            var taskRunner = new TaskRunner({
+                userA: 3,
+                userB: 2,
+                userC: 1
+            });
+
+            var scheduler = new Scheduler(capacity, taskRunner);
+            scheduler.add(USER_A);
+            scheduler.add(USER_A);
+            scheduler.add(USER_A);
+            scheduler.add(USER_B);
+            scheduler.add(USER_B);
+            scheduler.add(USER_C);
+
+            scheduler.on('done', function() {
+                var results = taskRunner.results;
+
+                assert.equal(results.length, 6);
+
+                assert.equal(results[0], USER_A);
+                assert.equal(results[1], USER_B);
+                assert.equal(results[2], USER_C);
+                assert.equal(results[3], USER_A);
+                assert.equal(results[4], USER_B);
+                assert.equal(results[5], USER_A);
+
+                return done();
+            });
+
+            scheduler.schedule();
+        });
+    });
+});

test/integration/stream_copy.test.js

@@ -0,0 +1,21 @@
+'use strict';
+
+require('../helper');
+const assert = require('assert');
+
+const StreamCopy = require('../../app/services/stream_copy');
+
+describe('stream copy', function() {
+    it('uses batch api port', function(done) {
+        const userDbParams = {
+            dbname: 'cartodb_test_user_1_db',
+            dbuser: 'test_cartodb_user_1',
+            pass: 'test_cartodb_user_1_pass',
+            port: 'invalid_port'
+        };
+        const sql = 'COPY dummy_table FROM STDIN';
+        const streamCopy = new StreamCopy(sql, userDbParams);
+        assert.equal(streamCopy.pg.dbopts.port, global.settings.db_batch_port);
+        done();
+    });
+});

test/integration/utils/table_cache_factory.test.js

@@ -0,0 +1,41 @@
+'use strict';
+
+require('../../helper');
+var assert = require('assert');
+var LRU = require('lru-cache');
+var NoCache = require('../../../app/utils/no_cache');
+
+var TableCacheFactory = require('../../../app/utils/table_cache_factory');
+var factory = new TableCacheFactory();
+
+describe('TableCacheFactory', function() {
+
+    it('returns a NoCache by default', function() {
+        var tableCache = factory.build({});
+        assert(tableCache instanceof NoCache);
+    });
+
+    it('returns a NoCache if it is disabled in settings', function() {
+        var tableCache = factory.build({tableCacheEnabled: false});
+        assert(tableCache instanceof NoCache);
+    });
+
+    it('returns an LRU if enabled in settings, with its default settings', function() {
+        var tableCache = factory.build({tableCacheEnabled: true});
+        assert(tableCache instanceof LRU);
+        assert.equal(tableCache._max, 8192);
+        assert.equal(tableCache._maxAge, 1000*60*10);
+    });
+
+    it('returns an LRU if enabled in settings, with the passed settings', function() {
+        var tableCache = factory.build({
+            tableCacheEnabled: true,
+            tableCacheMax: 42,
+            tableCacheMaxAge: 1000
+        });
+        assert(tableCache instanceof LRU);
+        assert.equal(tableCache._max, 42);
+        assert.equal(tableCache._maxAge, 1000);
+    });
+
+});

test/prepare_db.sh

@@ -0,0 +1,250 @@
+#!/bin/sh
+
+# this script prepare database and redis instance to run acceptance test
+#
+# NOTE: assumes existance of a "template_postgis" loaded with
+#       compatible version of postgis (legacy.sql included)
+
+PREPARE_REDIS=yes
+PREPARE_PGSQL=yes
+OFFLINE=no
+
+while [ -n "$1" ]; do
+  if test "$1" = "--skip-pg"; then
+    PREPARE_PGSQL=no
+    shift; continue
+  elif test "$1" = "--skip-redis"; then
+    PREPARE_REDIS=no
+    shift; continue
+  elif test "$1" = "--offline"; then
+    OFFLINE=yes
+    shift; continue
+  fi
+done
+
+die() {
+        msg=$1
+        echo "${msg}" >&2
+        exit 1
+}
+
+# This is where postgresql connection parameters are read from
+TESTENV=../config/environments/test.js
+
+# Extract postgres configuration
+PGHOST=`node -e "console.log(require('${TESTENV}').db_host || '')"`
+echo "PGHOST: [$PGHOST]"
+PGPORT=`node -e "console.log(require('${TESTENV}').db_port || '')"`
+echo "PGPORT: [$PGPORT]"
+
+PUBLICUSER=`node -e "console.log(require('${TESTENV}').db_pubuser || 'xxx')"`
+PUBLICPASS=`node -e "console.log(require('${TESTENV}').db_pubuser_pass || 'xxx')"`
+echo "PUBLICUSER: [${PUBLICUSER}]"
+echo "PUBLICPASS: [${PUBLICPASS}]"
+
+
+TESTUSERID=1
+
+TESTUSER=`node -e "console.log(require('${TESTENV}').db_user || '')"`
+if test -z "$TESTUSER"; then
+  echo "Missing db_user from ${TESTENV}" >&2
+  exit 1
+fi
+TESTUSER=`echo ${TESTUSER} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+echo "TESTUSER: [${TESTUSER}]"
+
+TESTPASS=`node -e "console.log(require('${TESTENV}').db_user_pass || '')"`
+TESTPASS=`echo ${TESTPASS} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+echo "TESTPASS: [${TESTPASS}]"
+
+TEST_DB=`node -e "console.log(require('${TESTENV}').db_base_name || '')"`
+if test -z "$TEST_DB"; then
+  echo "Missing db_base_name from ${TESTENV}" >&2
+  exit 1
+fi
+TEST_DB=`echo ${TEST_DB} | sed "s/<%= user_id %>/${TESTUSERID}/"`
+
+export PGHOST PGPORT
+
+if test x"$PREPARE_PGSQL" = xyes; then
+
+  echo "preparing postgres..."
+  echo "PostgreSQL server version: `psql -A -t -c 'select version()'`"
+  echo "PAUSE; RESUME;" | psql -p 6432 pgbouncer # make sure there are no connections pgbouncer -> test_db
+  dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise
+  createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database"
+  psql -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";' ${TEST_DB}
+  psql -c "CREATE EXTENSION IF NOT EXISTS plpythonu;" ${TEST_DB}
+
+  LOCAL_SQL_SCRIPTS='test populated_places_simple_reduced py_sleep quota_mock'
+  REMOTE_SQL_SCRIPTS='CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_OverviewsSupport CDB_Overviews'
+
+  if test x"$OFFLINE" = xno; then
+      CURL_ARGS=""
+      for i in ${REMOTE_SQL_SCRIPTS}
+      do
+        CURL_ARGS="${CURL_ARGS}\"https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql\" -o support/sql/$i.sql "
+      done
+      echo "Downloading and updating: ${REMOTE_SQL_SCRIPTS}"
+      echo ${CURL_ARGS} | xargs curl -L -s
+  fi
+
+  PG_PARALLEL=$(pg_config --version | (awk '{$2*=1000; if ($2 >= 9600) print 1; else print 0;}' 2> /dev/null || echo 0))
+
+  psql -c "CREATE EXTENSION IF NOT EXISTS plpythonu;" ${TEST_DB}
+  ALL_SQL_SCRIPTS="${REMOTE_SQL_SCRIPTS} ${LOCAL_SQL_SCRIPTS}"
+  for i in ${ALL_SQL_SCRIPTS}
+  do
+    # Strip PARALLEL labels for PostgreSQL releases before 9.6
+    if [ $PG_PARALLEL -eq 0 ]; then
+      TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXXXX)
+      sed -e 's/PARALLEL \= [A-Z]*,/''/g' \
+          -e 's/PARALLEL [A-Z]*/''/g' support/sql/${i}.sql > $TMPFILE
+      mv $TMPFILE support/sql/${i}.sql
+    fi
+    cat support/sql/${i}.sql |
+      sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" |
+      sed "s/:PUBLICUSER/${PUBLICUSER}/" |
+      sed "s/:PUBLICPASS/${PUBLICPASS}/" |
+      sed "s/:TESTUSER/${TESTUSER}/" |
+      sed "s/:TESTPASS/${TESTPASS}/" |
+      psql -q -v ON_ERROR_STOP=1 ${TEST_DB} > /dev/null || exit 1
+  done
+
+fi
+
+if test x"$PREPARE_REDIS" = xyes; then
+
+  REDIS_HOST=`node -e "console.log(require('${TESTENV}').redis_host || '127.0.0.1')"`
+  REDIS_PORT=`node -e "console.log(require('${TESTENV}').redis_port || '6336')"`
+
+  echo "preparing redis..."
+
+  # delete previous publicuser
+  cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HDEL rails:users:vizzuality database_host
+HDEL rails:users:vizzuality database_publicuser
+EOF
+
+  cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET rails:users:vizzuality \
+ id 1 \
+ database_name ${TEST_DB} \
+ database_host ${PGHOST} \
+ map_key 1234
+SADD rails:users:vizzuality:map_key 1235
+EOF
+
+  # A user configured as with cartodb-2.5.0+
+  cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET rails:users:cartodb250user \
+ id ${TESTUSERID} \
+ database_name ${TEST_DB} \
+ database_host ${PGHOST} \
+ database_password ${TESTPASS} \
+ map_key 1234
+SADD rails:users:cartodb250user:map_key 1234
+EOF
+
+  cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 3
+HMSET rails:oauth_access_tokens:l0lPbtP68ao8NfStCiA3V3neqfM03JKhToxhUQTR \
+ consumer_key fZeNGv5iYayvItgDYHUbot1Ukb5rVyX6QAg8GaY2 \
+ consumer_secret IBLCvPEefxbIiGZhGlakYV4eM8AbVSwsHxwEYpzx \
+ access_token_token l0lPbtP68ao8NfStCiA3V3neqfM03JKhToxhUQTR \
+ access_token_secret 22zBIek567fMDEebzfnSdGe8peMFVFqAreOENaDK \
+ user_id 1 \
+ time sometime
+EOF
+
+  cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET rails:users:cartofante \
+ id 2 \
+ database_name ${TEST_DB} \
+ database_host ${PGHOST} \
+ database_password test_cartodb_user_2_pass \
+ map_key 4321
+SADD rails:users:fallback_1:map_key 4321
+EOF
+
+# delete previous jobs
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+EVAL "return redis.call('del', unpack(redis.call('keys', ARGV[1])))" 0 batch:jobs:*
+EOF
+
+# delete job queue
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+DEL batch:queues:localhost
+EOF
+
+# delete user index
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+DEL batch:users:vizzuality
+EOF
+
+# User: vizzuality
+
+# API Key Default public
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET api_keys:vizzuality:default_public \
+  user "vizzuality" \
+  type "default" \
+  grants_sql "true" \
+  database_role "testpublicuser" \
+  database_password "public"
+EOF
+
+# API Key Master
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET api_keys:vizzuality:1234 \
+  user "vizzuality" \
+  type "master" \
+  grants_sql "true" \
+  database_role "${TESTUSER}" \
+  database_password "${TESTPASS}"
+EOF
+
+# API Key Regular1
+cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+  HMSET api_keys:vizzuality:regular1 \
+    user "vizzuality" \
+    type "regular" \
+    grants_sql "true" \
+    database_role "regular_1" \
+    database_password "regular1"
+EOF
+
+# API Key Regular1
+cat <<EOF | redis-cli -p ${REDIS_PORT} -n 5
+  HMSET api_keys:vizzuality:regular2 \
+    user "vizzuality" \
+    type "regular" \
+    grants_sql "true" \
+    database_role "regular_2" \
+    database_password "regular2"
+EOF
+
+# User: cartodb250user
+
+# API Key Default public
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET api_keys:cartodb250user:default_public \
+  user "cartodb250user" \
+  type "default" \
+  grants_sql "true" \
+  database_role "testpublicuser" \
+  database_password "public"
+EOF
+
+# API Key Master
+cat <<EOF | redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT} -n 5
+HMSET api_keys:cartodb250user:1234 \
+  user "cartodb250user" \
+  type "master" \
+  grants_sql "true" \
+  database_role "${TESTUSER}" \
+  database_password "${TESTPASS}"
+EOF
+
+fi
+
+echo "ok, you can run test now"

test/run_tests.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+
+# To make output dates deterministic
+export TZ='Europe/Rome'
+export PGAPPNAME='cartodb_sqlapi_tester'
+
+# In case PGUSER env variable does not exist we attempt to use `postgres`
+if test x"${PGUSER}" = x; then
+    echo "PGUSER not found"
+    PGUSER=postgres
+else
+    echo "PGUSER found = ${PGUSER}"
+fi
+
+OPT_CREATE_PGSQL=yes # create/prepare the postgresql test database
+OPT_CREATE_REDIS=yes # create/prepare the redis test databases
+OPT_DROP_PGSQL=yes   # drop the postgreql test environment
+OPT_DROP_REDIS=yes   # drop the redis test environment
+OPT_COVERAGE=no      # run tests with coverage
+OPT_OFFLINE=no       # do not donwload scripts
+
+
+cd $(dirname $0)
+BASEDIR=$(pwd)
+cd -
+
+REDIS_PORT=`node -e "console.log(require('${BASEDIR}/../config/environments/test.js').redis_port)"`
+export REDIS_PORT
+echo "REDIS_PORT: [${REDIS_PORT}]"
+
+cleanup() {
+  if test x"$OPT_DROP" = xyes; then
+    if test x"$PID_REDIS" = x; then
+      PID_REDIS=$(cat ${BASEDIR}/redis.pid)
+      if test x"$PID_REDIS" = x; then
+        echo "Could not find a test redis pid to kill it"
+        return;
+      fi
+    fi
+    echo "Cleaning up"
+    kill ${PID_REDIS}
+  fi
+}
+
+cleanup_and_exit() {
+	cleanup
+	exit
+}
+
+die() {
+	msg=$1
+	echo "${msg}" >&2
+	cleanup
+	exit 1
+}
+
+trap 'cleanup_and_exit' 1 2 3 5 9 13
+
+while [ -n "$1" ]; do
+        if test "$1" = "--nodrop"; then
+                OPT_DROP_REDIS=no
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-pg"; then
+                OPT_DROP_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nodrop-redis"; then
+                OPT_DROP_REDIS=no
+                shift
+                continue
+        elif test "$1" = "--nocreate"; then
+                OPT_CREATE_REDIS=no
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-pg"; then
+                OPT_CREATE_PGSQL=no
+                shift
+                continue
+        elif test "$1" = "--nocreate-redis"; then
+                OPT_CREATE_REDIS=no
+                shift
+                continue
+        elif test "$1" = "--with-coverage"; then
+                OPT_COVERAGE=yes
+                shift
+                continue
+        elif test "$1" = "--offline"; then
+                OPT_OFFLINE=yes
+                shift
+                continue
+        else
+                break
+        fi
+done
+
+if [ -z "$1" ]; then
+        echo "Usage: $0 [<options>] <test> [<test>]" >&2
+        echo "Options:" >&2
+        echo " --nocreate         do not create the test environment on start" >&2
+        echo " --nocreate-pg      do not create the pgsql test environment" >&2
+        echo " --nocreate-redis   do not create the redis test environment" >&2
+        echo " --nodrop           do not drop the test environment on exit" >&2
+        echo " --nodrop-pg        do not drop the pgsql test environment" >&2
+        echo " --nodrop-redis     do not drop the redis test environment" >&2
+        echo " --with-coverage    use istanbul to determine code coverage" >&2
+        exit 1
+fi
+
+TESTS=$@
+
+if test x"$OPT_CREATE_REDIS" = xyes; then
+  echo "Starting redis on port ${REDIS_PORT}"
+  REDIS_CELL_PATH="${BASEDIR}/support/libredis_cell.so"
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    REDIS_CELL_PATH="${BASEDIR}/support/libredis_cell.dylib"
+  fi
+  echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${REDIS_CELL_PATH} > ${BASEDIR}/test.log &
+  PID_REDIS=$!
+  echo ${PID_REDIS} > ${BASEDIR}/redis.pid
+fi
+
+PREPARE_DB_OPTS=
+if test x"$OPT_CREATE_PGSQL" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-pg"
+fi
+if test x"$OPT_CREATE_REDIS" != xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --skip-redis"
+fi
+if test x"$OPT_OFFLINE" == xyes; then
+  PREPARE_DB_OPTS="$PREPARE_DB_OPTS --offline"
+fi
+
+echo "Preparing the environment"
+cd ${BASEDIR}
+sh prepare_db.sh ${PREPARE_DB_OPTS} || die "database preparation failure"
+cd -
+
+PATH=node_modules/.bin/:node_modules/mocha/bin:$PATH
+
+echo
+echo "Environment:"
+echo
+echo "  ogr2ogr version: "`ogr2ogr --version`
+echo
+
+if test x"$OPT_COVERAGE" = xyes; then
+  echo "Running tests with coverage"
+  ./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd --trace -t 5000 ${TESTS}
+else
+  echo "Running tests"
+  mocha -u tdd -t 5000 ${TESTS}
+fi
+ret=$?
+
+cleanup || exit 1
+
+exit $ret

test/run_tests_docker.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+usage() {
+    /etc/init.d/postgresql stop
+    echo "Usage: $0 [nodejs10|nodejs6]"
+    exit 1
+}
+
+echo "$0 $1"
+
+# start PostgreSQL
+/etc/init.d/postgresql start
+
+# Configure
+./configure
+
+echo "Node.js version:"
+node -v
+
+# install dependencies
+NODEJS_VERSION=${1-nodejs10}
+
+if [ "$NODEJS_VERSION" = "nodejs10" ];
+then
+    echo "npm version on install:"
+    npm -v
+    mv npm-shrinkwrap.json npm-shrinkwrap.json.backup
+    npm ci
+    npm ls
+    mv npm-shrinkwrap.json.backup npm-shrinkwrap.json
+elif [ "$NODEJS_VERSION" = "nodejs6" ];
+then
+    echo "npm version on install:"
+    npm -v
+    mv package-lock.json package-lock.json.backup
+    npm i
+    npm ls
+    mv package-lock.json.backup package-lock.json
+else
+    usage
+fi
+
+# run tests
+echo "npm version on tests:"
+npm -v
+
+npm test

test/support/.gitignore

@@ -0,0 +1 @@
+CDB_*.sql

test/support/assert.js

@@ -0,0 +1,127 @@
+'use strict';
+
+var assert = module.exports = exports = require('assert');
+var request = require('request');
+var debug = require('debug')('assert-response');
+
+assert.response = function(server, req, res, callback) {
+    if (!callback) {
+        callback = res;
+        res = {};
+    }
+
+    var port = 5555,
+        host = '127.0.0.1';
+
+    var listeningAttempts = 0;
+    var listener;
+    function listen() {
+        if (listeningAttempts > 25) {
+            var message = 'Tried too many ports';
+            debug(message);
+            return callback(new Error(message));
+        }
+        listener = server.listen(port, host);
+        listener.on('error', function() {
+            port++;
+            listeningAttempts++;
+            listen();
+        });
+        listener.on('listening', onServerListening);
+    }
+
+    listen();
+
+    debug('Request definition', req);
+
+    // jshint maxcomplexity:10
+    function onServerListening() {
+        debug('Server listening on port = %d', port);
+        var status = res.status || res.statusCode;
+        var requestParams = {
+            url: 'http://' + host + ':' + port + req.url,
+            method: req.method || 'GET',
+            headers: req.headers || {},
+            timeout: req.timeout || 5000,
+            encoding: req.encoding || 'utf8'
+        };
+
+        if (req.body || req.data) {
+            requestParams.body = req.body || req.data;
+        }
+
+        if (req.formData) {
+            requestParams.formData = req.formData;
+        }
+
+        debug('Request params', requestParams);
+        request(requestParams, function assert$response$requestHandler(error, response, body) {
+            debug('Request response', error);
+            listener.close(function() {
+                debug('Server closed');
+                if (error) {
+                    return callback(error);
+                }
+
+                response = response || {};
+                response.body = response.body || body;
+                debug('Response status', response.statusCode)
+
+                // Assert response body
+                if (res.body) {
+                    var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
+                    assert.ok(
+                        eql,
+                        colorize('[red]{Invalid response body.}\n' +
+                            '     Expected: [green]{' + res.body + '}\n' +
+                            '     Got: [red]{' + response.body + '}')
+                    );
+                }
+
+                // Assert response status
+                if (typeof status === 'number') {
+                    assert.equal(response.statusCode, status,
+                        colorize('[red]{Invalid response status code.}\n' +
+                            '     Expected: [green]{' + status + '}\n' +
+                            '     Got: [red]{' + response.statusCode + '}\n' +
+                            '     Body: ' + response.body)
+                    );
+                }
+
+                // Assert response headers
+                if (res.headers) {
+                    var keys = Object.keys(res.headers);
+                    for (var i = 0, len = keys.length; i < len; ++i) {
+                        var name = keys[i],
+                            actual = response.headers[name.toLowerCase()],
+                            expected = res.headers[name],
+                            headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
+                        assert.ok(headerEql,
+                            colorize('Invalid response header [bold]{' + name + '}.\n' +
+                                '     Expected: [green]{' + expected + '}\n' +
+                                '     Got: [red]{' + actual + '}')
+                        );
+                    }
+                }
+
+                // Callback
+                return callback(null, response);
+            });
+        });
+
+    }
+};
+
+/**
+ * Colorize the given string using ansi-escape sequences.
+ * Disabled when --boring is set.
+ *
+ * @param {String} str
+ * @return {String}
+ */
+function colorize(str) {
+    var colors = { bold: 1, red: 31, green: 32, yellow: 33 };
+    return str.replace(/\[(\w+)\]\{([^]*?)\}/g, function(_, color, str) {
+        return '\x1B[' + colors[color] + 'm' + str + '\x1B[0m';
+    });
+}

test/support/batch-test-client.js

@@ -0,0 +1,300 @@
+'use strict';
+
+require('../helper');
+var assert = require('assert');
+var appServer = require('../../app/server');
+var redisUtils = require('./redis_utils');
+var debug = require('debug')('batch-test-client');
+
+var JobStatus = require('../../batch/job_status');
+var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
+var batchFactory = require('../../batch/index');
+
+function response(code) {
+    return {
+        status: code
+    };
+}
+
+var RESPONSE = {
+    OK: response(200),
+    CREATED: response(201),
+    BAD_REQUEST: response(400)
+};
+
+
+function BatchTestClient(config) {
+    this.config = config || {};
+    this.server = appServer();
+
+    this.batch = batchFactory(metadataBackend, redisUtils.getPool(), this.config.name);
+    this.batch.start();
+
+    this.pendingJobs = [];
+    this.ready = false;
+    this.batch.on('ready', function() {
+        this.ready = true;
+        this.pendingJobs.forEach(function(pendingJob) {
+            this.createJob(pendingJob.job, pendingJob.override, pendingJob.callback);
+        }.bind(this));
+    }.bind(this));
+}
+
+module.exports = BatchTestClient;
+
+BatchTestClient.prototype.isReady = function() {
+    return this.ready;
+};
+
+BatchTestClient.prototype.getExpectedResponse = function (override) {
+    return override.response || this.config.response || RESPONSE.CREATED;
+};
+
+BatchTestClient.prototype.createJob = function(job, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+    if (!this.isReady()) {
+        this.pendingJobs.push({
+            job: job,
+            override: override || {},
+            callback: callback
+        });
+        return debug('Waiting for Batch service to be ready');
+    }
+    assert.response(
+        this.server,
+        {
+            url: this.getUrl(override),
+            headers: {
+                host: this.getHost(override),
+                'Content-Type': 'application/json',
+                authorization: this.getAuthorization(override)
+            },
+            method: 'POST',
+            data: JSON.stringify(job)
+        },
+        this.getExpectedResponse(override),
+        function (err, res) {
+            if (err) {
+                return callback(err);
+            }
+
+            if (res.statusCode < 400) {
+                return callback(null, new JobResult(JSON.parse(res.body), this, override));
+            } else {
+                return callback(null, res);
+            }
+        }.bind(this)
+    );
+};
+
+BatchTestClient.prototype.getJobStatus = function(jobId, override, callback) {
+    assert.response(
+        this.server,
+        {
+            url: this.getUrl(override, jobId),
+            headers: {
+                host: this.getHost(override),
+                authorization: this.getAuthorization(override)
+            },
+            method: 'GET',
+            timeout: override.timeout
+        },
+        RESPONSE.OK,
+        function (err, res) {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, JSON.parse(res.body));
+        }
+    );
+};
+
+BatchTestClient.prototype.getWorkInProgressJobs = function(override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    assert.response(
+        this.server,
+        {
+            url: '/api/v1/jobs-wip',
+            headers: {
+                host: this.getHost(override)
+            },
+            method: 'GET'
+        },
+        RESPONSE.OK,
+        function (err, res) {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, JSON.parse(res.body));
+        }
+    );
+};
+
+BatchTestClient.prototype.cancelJob = function(jobId, override, callback) {
+    assert.response(
+        this.server,
+        {
+            url: this.getUrl(override, jobId),
+            headers: {
+                host: this.getHost(override)
+            },
+            method: 'DELETE'
+        },
+        override.statusCode,
+        function (err, res) {
+            if (err) {
+                return callback(err);
+            }
+            return callback(null, JSON.parse(res.body));
+        }
+    );
+};
+
+BatchTestClient.prototype.drain = function(callback) {
+    this.batch.stop(function() {
+        return redisUtils.clean('batch:*', callback);
+    });
+};
+
+BatchTestClient.prototype.getHost = function(override) {
+    return override.host || this.config.host || 'vizzuality.cartodb.com';
+};
+
+BatchTestClient.prototype.getAuthorization = function (override) {
+    const auth = override.authorization || this.config.authorization;
+
+    if (auth) {
+        return `Basic ${new Buffer(auth).toString('base64')}`;
+    }
+};
+
+BatchTestClient.prototype.getUrl = function(override, jobId) {
+    var urlParts = ['/api/v2/sql/job'];
+    if (jobId) {
+        urlParts.push(jobId);
+    }
+    return `${urlParts.join('/')}${override.anonymous ? '' : '?api_key=' + this.getApiKey(override)}`;
+};
+
+BatchTestClient.prototype.getApiKey = function(override) {
+    return override.apiKey || this.config.apiKey || '1234';
+};
+
+/****************** JobResult ******************/
+
+
+function JobResult(job, batchTestClient, override) {
+    this.job = job;
+    this.batchTestClient = batchTestClient;
+    this.override = override;
+}
+
+JobResult.prototype.getStatus = function(requiredStatus, callback) {
+    if (!callback) {
+        callback = requiredStatus;
+        requiredStatus = undefined;
+    }
+
+    var self = this;
+    var attempts = 1;
+    self.override.timeout = 1000;
+
+    var interval = setInterval(function () {
+        self.batchTestClient.getJobStatus(self.job.job_id, self.override, function (err, job) {
+            if (err) {
+                clearInterval(interval);
+                return callback(err);
+            }
+            attempts += 1;
+
+            if (attempts > 20) {
+                clearInterval(interval);
+                return callback(new Error('Reached maximum number of request (20) to check job status'));
+            }
+
+            if (hasRequiredStatus(job, requiredStatus)) {
+                clearInterval(interval);
+                self.job = job;
+                return callback(null, job);
+            } else {
+                debug('Job %s [status=%s] waiting to be done', self.job.job_id, job.status);
+            }
+        });
+    }, 100);
+};
+
+function hasRequiredStatus(job, requiredStatus) {
+    if (requiredStatus) {
+        return job.status === requiredStatus;
+    }
+
+    if (JobStatus.isFinal(job.status)) {
+        if (job.fallback_status !== undefined) {
+            if (JobStatus.isFinal(job.fallback_status) || job.fallback_status === JobStatus.SKIPPED) {
+                return true;
+            }
+        } else {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+JobResult.prototype.cancel = function (callback) {
+    var self = this;
+    this.override.statusCode = response(RESPONSE.OK);
+    this.batchTestClient.cancelJob(this.job.job_id, this.override, function (err, job) {
+        if (err) {
+            return callback(err);
+        }
+        self.job = job;
+        callback(null, job);
+    });
+};
+
+JobResult.prototype.tryCancel = function (callback) {
+    var self = this;
+    this.override.statusCode = response();
+    this.batchTestClient.cancelJob(this.job.job_id, this.override, function (err, job) {
+        if (err) {
+            return callback(err);
+        }
+        self.job = job;
+        callback(null, job);
+    });
+};
+
+JobResult.prototype.validateExpectedResponse = function (expected) {
+    var actual = this.job.query;
+
+    actual.query.forEach(function(actualQuery, index) {
+        var expectedQuery = expected.query[index];
+        assert.ok(expectedQuery);
+        Object.keys(expectedQuery).forEach(function(expectedKey) {
+            assert.equal(
+                actualQuery[expectedKey],
+                expectedQuery[expectedKey],
+                'Expected value for key "' + expectedKey + '" does not match: ' + actualQuery[expectedKey] + ' ==' +
+                expectedQuery[expectedKey] + ' at query index=' + index + '. Full response: ' +
+                JSON.stringify(actual, null, 4)
+            );
+        });
+        var propsToCheckDate = ['started_at', 'ended_at'];
+        propsToCheckDate.forEach(function(propToCheckDate) {
+            if (actualQuery.hasOwnProperty(propToCheckDate)) {
+                assert.ok(new Date(actualQuery[propToCheckDate]));
+            }
+        });
+    });
+
+    assert.equal(actual.onsuccess, expected.onsuccess);
+    assert.equal(actual.onerror, expected.onerror);
+};

test/support/csv/copy_test_table.csv

@@ -0,0 +1,7 @@
+id,name
+11,Paul
+12,Peter
+13,Matthew
+14,
+15,James
+16,John

test/support/db_utils.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const { Client } = require('pg');
+
+const dbConfig = {
+    db_user: process.env.PGUSER || 'postgres',
+    db_host: global.settings.db_host,
+    db_port: global.settings.db_port,
+    db_batch_port: global.settings.db_batch_port
+};
+
+module.exports.resetPgBouncerConnections = function (callback) {
+    // We assume there's no pgbouncer if db_port === db_batch_port
+    if (dbConfig.db_port === dbConfig.db_batch_port) {
+        return callback();
+    }
+
+    const client = new Client({
+        database: 'pgbouncer',
+        user: dbConfig.db_user,
+        host: dbConfig.db_host,
+        port: dbConfig.db_port
+    });
+
+    // We just chain a PAUSE followed by a RESUME to reset internal pool connections of PgBouncer
+    client.connect();
+    client.query('PAUSE', err => {
+        if (err) {
+            return callback(err);
+        }
+        client.query('RESUME', err => {
+            client.end();
+            return callback(err);
+        });
+    });
+};

test/support/redis_utils.js

@@ -0,0 +1,39 @@
+'use strict';
+
+var RedisPool = require('redis-mpool');
+
+var redisConfig = {
+    host: global.settings.redis_host,
+    port: global.settings.redis_port,
+    max: global.settings.redisPool,
+    idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
+    reapIntervalMillis: global.settings.redisReapIntervalMillis
+};
+var metadataBackend = require('cartodb-redis')(redisConfig);
+
+module.exports.clean = function clean(pattern, callback) {
+    metadataBackend.redisCmd(5, 'KEYS', [ pattern ], function (err, keys) {
+        if (err) {
+            return callback(err);
+        }
+
+        if (!keys || !keys.length) {
+            return callback();
+        }
+
+        metadataBackend.redisCmd(5, 'DEL', keys, callback);
+    });
+};
+
+module.exports.getConfig = function getConfig() {
+    return redisConfig;
+};
+
+var pool = new RedisPool(redisConfig);
+module.exports.getPool = function getPool() {
+    return pool;
+};
+
+module.exports.configureUserMetadata = function configureUserMetadata(action, params, callback) {
+    metadataBackend.redisCmd(5, action, params, callback);
+}

test/support/sql/py_sleep.sql

@@ -0,0 +1,5 @@
+CREATE OR REPLACE FUNCTION py_sleep(t FLOAT8)
+RETURNS void AS $$
+  import time
+  time.sleep(t)
+$$ LANGUAGE plpythonu;

test/support/sql/quota_mock.sql

@@ -0,0 +1,17 @@
+-- See https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_Quota.sql
+
+CREATE OR REPLACE FUNCTION _CDB_UserQuotaInBytes()
+RETURNS int8 AS
+$$
+    -- 250 MB
+    SELECT (250 * 1024 * 1024)::int8;
+$$ LANGUAGE sql IMMUTABLE;
+
+CREATE OR REPLACE FUNCTION CDB_UserDataSize(schema_name TEXT)
+RETURNS bigint AS
+$$
+BEGIN
+    -- 100 MB
+    RETURN 100 * 1024 * 1024;
+END;
+$$ LANGUAGE 'plpgsql' VOLATILE;

test/support/sql/test.sql

@@ -0,0 +1,228 @@
+--
+-- sql-api test database
+--
+-- To use:
+--
+-- > dropdb -Upostgres -hlocalhost  cartodb_test_user_1_db
+-- > createdb -Upostgres -hlocalhost -Ttemplate_postgis -Opostgres -EUTF8 cartodb_test_user_1_db
+-- > psql -Upostgres -hlocalhost cartodb_test_user_1_db < test.sql
+--
+-- NOTE: requires a postgis template called template_postgis with CDB functions included
+--
+
+SET statement_timeout = 0;
+SET client_encoding = 'UTF8';
+SET standard_conforming_strings = off;
+SET check_function_bodies = false;
+SET client_min_messages = warning;
+SET escape_string_warning = off;
+SET search_path = public, pg_catalog;
+SET default_tablespace = '';
+SET default_with_oids = false;
+
+-- first table
+DROP TABLE IF EXISTS untitle_table_4;
+CREATE TABLE untitle_table_4 (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    -- NOTE: the_geom_webmercator is intentionally listed _before_ the_geom
+    --       see https://github.com/CartoDB/CartoDB-SQL-API/issues/116
+    the_geom_webmercator geometry,
+    the_geom geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+CREATE SEQUENCE untitle_table_4_cartodb_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE untitle_table_4_cartodb_id_seq OWNED BY untitle_table_4.cartodb_id;
+
+SELECT pg_catalog.setval('untitle_table_4_cartodb_id_seq', 60, true);
+
+ALTER TABLE untitle_table_4 ALTER COLUMN cartodb_id SET DEFAULT nextval('untitle_table_4_cartodb_id_seq'::regclass);
+
+INSERT INTO untitle_table_4
+(updated_at, created_at, cartodb_id, name, address, the_geom, the_geom_webmercator)
+VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21', -1, 'Test', 'Fake for testing', 'SRID=4326;POINT(33 16)', 'SRID=3857;POINT(3673543.19617803 1804722.76625729)');
+
+ALTER TABLE ONLY untitle_table_4 ADD CONSTRAINT untitle_table_4_pkey PRIMARY KEY (cartodb_id);
+
+CREATE INDEX untitle_table_4_the_geom_idx ON untitle_table_4 USING gist (the_geom);
+CREATE INDEX untitle_table_4_the_geom_webmercator_idx ON untitle_table_4 USING gist (the_geom_webmercator);
+
+-- second table
+DROP TABLE IF EXISTS scoped_table_1;
+CREATE TABLE scoped_table_1 (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom_webmercator geometry,
+    the_geom geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+CREATE SEQUENCE scoped_table_1_cartodb_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE scoped_table_1_cartodb_id_seq OWNED BY scoped_table_1.cartodb_id;
+
+SELECT pg_catalog.setval('scoped_table_1_cartodb_id_seq', 60, true);
+
+ALTER TABLE scoped_table_1 ALTER COLUMN cartodb_id SET DEFAULT nextval('scoped_table_1_cartodb_id_seq'::regclass);
+
+INSERT INTO scoped_table_1
+(updated_at, created_at, cartodb_id, name, address, the_geom, the_geom_webmercator)
+VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241');
+
+ALTER TABLE ONLY scoped_table_1 ADD CONSTRAINT scoped_table_1_pkey PRIMARY KEY (cartodb_id);
+
+CREATE INDEX scoped_table_1_the_geom_idx ON scoped_table_1 USING gist (the_geom);
+CREATE INDEX scoped_table_1_the_geom_webmercator_idx ON scoped_table_1 USING gist (the_geom_webmercator);
+
+-- private table
+DROP TABLE IF EXISTS private_table;
+CREATE TABLE private_table (
+    updated_at timestamp without time zone DEFAULT now(),
+    created_at timestamp without time zone DEFAULT now(),
+    cartodb_id integer NOT NULL,
+    name character varying,
+    address character varying,
+    the_geom geometry,
+    the_geom_webmercator geometry,
+    CONSTRAINT enforce_dims_the_geom CHECK ((st_ndims(the_geom) = 2)),
+    CONSTRAINT enforce_dims_the_geom_webmercator CHECK ((st_ndims(the_geom_webmercator) = 2)),
+    CONSTRAINT enforce_geotype_the_geom CHECK (((geometrytype(the_geom) = 'POINT'::text) OR (the_geom IS NULL))),
+    CONSTRAINT enforce_geotype_the_geom_webmercator CHECK (((geometrytype(the_geom_webmercator) = 'POINT'::text) OR (the_geom_webmercator IS NULL))),
+    CONSTRAINT enforce_srid_the_geom CHECK ((st_srid(the_geom) = 4326)),
+    CONSTRAINT enforce_srid_the_geom_webmercator CHECK ((st_srid(the_geom_webmercator) = 3857))
+);
+
+CREATE SEQUENCE untitle_table_4_cartodb_id_seq_p
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE untitle_table_4_cartodb_id_seq_p OWNED BY private_table.cartodb_id;
+
+SELECT pg_catalog.setval('untitle_table_4_cartodb_id_seq_p', 60, true);
+
+ALTER TABLE private_table ALTER COLUMN cartodb_id SET DEFAULT nextval('untitle_table_4_cartodb_id_seq_p'::regclass);
+
+INSERT INTO private_table VALUES
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.314252', 1, 'Hawai', 'Calle de Pérez Galdós 9, Madrid, Spain', '0101000020E6100000A6B73F170D990DC064E8D84125364440', '0101000020110F000076491621312319C122D4663F1DCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.319101', 2, 'El Estocolmo', 'Calle de la Palma 72, Madrid, Spain', '0101000020E6100000C90567F0F7AB0DC0AB07CC43A6364440', '0101000020110F0000C4356B29423319C15DD1092DADCC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.324', 3, 'El Rey del Tallarín', 'Plaza Conde de Toreno 2, Madrid, Spain', '0101000020E610000021C8410933AD0DC0CB0EF10F5B364440', '0101000020110F000053E71AC64D3419C10F664E4659CC5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.329509', 4, 'El Lacón', 'Manuel Fernández y González 8, Madrid, Spain', '0101000020E6100000BC5983F755990DC07D923B6C22354440', '0101000020110F00005DACDB056F2319C1EC41A980FCCA5241'),
+('2011-09-21 14:02:21.358706', '2011-09-21 14:02:21.334931', 5, 'El Pico', 'Calle Divino Pastor 12, Madrid, Spain', '0101000020E61000003B6D8D08C6A10DC0371B2B31CF364440', '0101000020110F00005F716E91992A19C17DAAA4D6DACC5241');
+
+ALTER TABLE ONLY private_table ADD CONSTRAINT untitle_table_4_pkey_p PRIMARY KEY (cartodb_id);
+
+CREATE INDEX untitle_table_4_the_geom_idx_p ON private_table USING gist (the_geom);
+CREATE INDEX untitle_table_4_the_geom_webmercator_idx_p ON private_table USING gist (the_geom_webmercator);
+
+-- public user role
+DROP USER IF EXISTS :PUBLICUSER;
+CREATE USER :PUBLICUSER WITH PASSWORD ':PUBLICPASS';
+ALTER ROLE :PUBLICUSER SET statement_timeout = 2000;
+GRANT SELECT ON TABLE scoped_table_1 TO :PUBLICUSER;
+
+-- regular user role 1
+DROP USER IF EXISTS regular_1;
+CREATE USER regular_1 WITH PASSWORD 'regular1';
+ALTER ROLE regular_1 SET statement_timeout = 2000;
+
+GRANT ALL ON TABLE scoped_table_1 TO regular_1;
+GRANT ALL ON SEQUENCE scoped_table_1_cartodb_id_seq TO regular_1;
+
+-- regular user role 2
+DROP USER IF EXISTS regular_2;
+CREATE USER regular_2 WITH PASSWORD 'regular2';
+ALTER ROLE regular_2 SET statement_timeout = 2000;
+
+-- fallback user role
+DROP USER IF EXISTS test_cartodb_user_2;
+CREATE USER test_cartodb_user_2 WITH PASSWORD 'test_cartodb_user_2_pass';
+GRANT ALL ON TABLE scoped_table_1 TO test_cartodb_user_2;
+GRANT ALL ON SEQUENCE scoped_table_1_cartodb_id_seq TO test_cartodb_user_2;
+
+-- db owner role
+DROP USER IF EXISTS :TESTUSER;
+CREATE USER :TESTUSER WITH PASSWORD ':TESTPASS';
+
+GRANT ALL ON TABLE untitle_table_4 TO :TESTUSER;
+GRANT SELECT ON TABLE untitle_table_4 TO :PUBLICUSER;
+GRANT ALL ON TABLE private_table TO :TESTUSER;
+GRANT ALL ON TABLE scoped_table_1 TO :TESTUSER;
+GRANT ALL ON SEQUENCE untitle_table_4_cartodb_id_seq_p TO :TESTUSER;
+
+GRANT ALL ON TABLE spatial_ref_sys TO :TESTUSER, :PUBLICUSER;
+
+REVOKE ALL ON geometry_columns FROM public;
+GRANT ALL ON geometry_columns TO :TESTUSER;
+GRANT ALL ON geography_columns TO :TESTUSER;
+GRANT SELECT ON geometry_columns TO :PUBLICUSER;
+GRANT SELECT ON geography_columns TO :PUBLICUSER;
+
+-- For https://github.com/CartoDB/CartoDB-SQL-API/issues/118
+DROP TABLE IF EXISTS cpg_test;
+CREATE TABLE cpg_test (a int);
+GRANT ALL ON TABLE cpg_test TO :TESTUSER;
+GRANT SELECT ON TABLE cpg_test TO :PUBLICUSER;
+
+
+CREATE TABLE IF NOT EXISTS
+  CDB_TableMetadata (
+    tabname regclass not null primary key,
+    updated_at timestamp with time zone not null default now()
+  );
+
+INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('untitle_table_4'::regclass, '2014-01-01T23:31:30.123Z');
+INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('private_table'::regclass, '2015-01-01T23:31:30.123Z');
+INSERT INTO CDB_TableMetadata (tabname, updated_at) VALUES ('scoped_table_1'::regclass, '2015-01-01T23:31:30.123Z');
+
+GRANT SELECT ON CDB_TableMetadata TO :TESTUSER;
+GRANT SELECT ON CDB_TableMetadata TO test_cartodb_user_2;
+
+DROP TABLE IF EXISTS copy_endpoints_test;
+CREATE TABLE copy_endpoints_test (
+    id integer,
+    name text,
+    age integer default 10
+);
+GRANT ALL ON TABLE copy_endpoints_test TO :TESTUSER;
+GRANT ALL ON TABLE copy_endpoints_test TO :PUBLICUSER;

test/support/test-client.js

@@ -0,0 +1,150 @@
+'use strict';
+
+require('../helper');
+var assert = require('assert');
+var appServer = require('../../app/server');
+var redisUtils = require('./redis_utils');
+const step = require('step');
+const PSQL = require('cartodb-psql');
+const _ = require('underscore');
+
+function response(code) {
+    return {
+        status: code
+    };
+}
+
+var RESPONSE = {
+    OK: response(200),
+    CREATED: response(201)
+};
+
+
+function TestClient(config) {
+    this.config = config || {};
+    this.server = appServer();
+}
+
+module.exports = TestClient;
+
+
+TestClient.prototype.getResult = function(query, override, callback) {
+    if (!callback) {
+        callback = override;
+        override = {};
+    }
+
+    assert.response(
+        this.server,
+        {
+            url: this.getUrl(override),
+            headers: {
+                host: this.getHost(override),
+                'Content-Type': this.getContentType(override),
+                authorization: this.getAuthorization(override)
+            },
+            method: 'POST',
+            data: this.getParser(override)({
+                q: query,
+                format: this.getFormat(override),
+                filename: this.getFilename(override)
+            })
+        },
+        this.getExpectedResponse(override),
+        function (err, res) {
+            if (err) {
+                return callback(err);
+            }
+            var result = JSON.parse(res.body);
+
+            if (res.statusCode > 299) {
+                return callback(null, result);
+            }
+
+            return callback(null, result.rows || [], result);
+        }
+    );
+};
+
+TestClient.prototype.getHost = function(override) {
+    return override.host || this.config.host || 'vizzuality.cartodb.com';
+};
+
+TestClient.prototype.getAuthorization = function (override) {
+    const auth = override.authorization || this.config.authorization;
+
+    if (auth) {
+        return `Basic ${new Buffer(auth).toString('base64')}`;
+    }
+};
+
+TestClient.prototype.getContentType = function(override) {
+    return override['Content-Type'] || this.config['Content-Type'] || 'application/json';
+};
+
+TestClient.prototype.getParser = function (override) {
+    return override.parser || this.config.parser || JSON.stringify
+}
+
+TestClient.prototype.getUrl = function(override) {
+    if (override.anonymous) {
+        return '/api/v1/sql?';
+    }
+
+    return '/api/v2/sql?api_key=' + (override.apiKey || this.config.apiKey || '1234');
+};
+
+TestClient.prototype.getExpectedResponse = function (override) {
+    return override.response || this.config.response || RESPONSE.OK;
+};
+
+TestClient.prototype.getFormat = function (override) {
+    return override.format || this.config.format || undefined;
+};
+
+TestClient.prototype.getFilename = function (override) {
+    return override.filename || this.config.filename || undefined;
+};
+
+TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) {
+    const userTimeoutLimitsKey = `limits:timeout:${user}`;
+    const params = [
+        userTimeoutLimitsKey,
+        'render', userTimeoutLimit,
+        'render_public', userTimeoutLimit
+    ];
+
+    redisUtils.configureUserMetadata('hmset', params, callback);
+};
+
+TestClient.prototype.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) {
+    const dbname = _.template(global.settings.db_base_name, { user_id: 1 });
+    const dbuser = _.template(global.settings.db_user, { user_id: 1 })
+    const pass = _.template(global.settings.db_user_pass, { user_id: 1 })
+    const publicuser = global.settings.db_pubuser;
+
+    const psql = new PSQL({
+        user: 'postgres',
+        dbname: dbname,
+        host: global.settings.db_host,
+        port: global.settings.db_port
+    });
+
+    // we need to guarantee all new connections have the new settings
+    psql.end();
+
+    step(
+        function configureTimeouts () {
+            const timeoutSQLs = [
+                `ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
+                `ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
+                `ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`
+            ];
+
+            const group = this.group();
+
+            timeoutSQLs.forEach(sql => psql.query(sql, group()));
+        },
+        callback
+    );
+};

test/unit/apikeyauth.test.js

@@ -0,0 +1,93 @@
+'use strict';
+
+require('../helper');
+
+var ApikeyAuth = require('../../app/auth/apikey');
+var assert = require('assert');
+
+describe.skip('has credentials', function() {
+
+    var noCredentialsRequests = [
+        {
+            des: 'there is not api_key/map_key in the request query',
+            req: {query:{}}
+        },
+        {
+            des: 'api_key is undefined`ish in the request query',
+            req: {query:{api_key:null}}
+        },
+        {
+            des: 'map_key is undefined`ish in the request query',
+            req: {query:{map_key:null}}
+        },
+        {
+            des: 'there is not api_key/map_key in the request body',
+            req: {query:{}, body:{}}
+        },
+        {
+            des: 'api_key is undefined`ish in the request body',
+            req: {query:{}, body:{api_key:null}}
+        },
+        {
+            des: 'map_key is undefined`ish in the request body',
+            req: {query:{}, body:{map_key:null}}
+        }
+    ];
+
+    noCredentialsRequests.forEach(function(request) {
+        it('has no credentials if ' + request.des, function() {
+            testCredentials(request.req, false);
+        });
+    });
+
+    var credentialsRequests = [
+        {
+            des: 'there is api_key in the request query',
+            req: {query:{api_key: 'foo'}}
+        },
+        {
+            des: 'there is api_key in the request query',
+            req: {query:{map_key: 'foo'}}
+        },
+        {
+            des: 'there is api_key in the request body',
+            req: {query:{}, body:{api_key:'foo'}}
+        },
+        {
+            des: 'there is map_key in the request body',
+            req: {query:{}, body:{map_key:'foo'}}
+        }
+    ];
+
+    credentialsRequests.forEach(function(request) {
+        it('has credentials if ' + request.des, function() {
+            testCredentials(request.req, true);
+        });
+    });
+
+    function testCredentials(req, hasCredentials) {
+        var apiKeyAuth = new ApikeyAuth(req);
+        assert.equal(apiKeyAuth.hasCredentials(), hasCredentials);
+    }
+
+});
+
+describe.skip('verifyCredentials', function() {
+
+    it('callbacks with true value when request api_key is the same', function(done) {
+        testVerifyCredentials({query:{api_key: 'foo'}}, {apiKey: 'foo'}, true, done);
+    });
+
+    it('callbacks with false value when request api_key is different', function(done) {
+        testVerifyCredentials({query:{api_key: 'foo'}}, {apiKey: 'bar'}, false, done);
+    });
+
+    function testVerifyCredentials(req, options, shouldBeValid, done) {
+        var apiKeyAuth = new ApikeyAuth(req);
+        apiKeyAuth.verifyCredentials(options, function(err, validCredentials) {
+            assert.equal(validCredentials, shouldBeValid);
+            done();
+        });
+    }
+
+});

test/unit/batch/job_publisher.js

@@ -0,0 +1,39 @@
+'use strict';
+
+var Channel = require('../../../batch/pubsub/channel');
+var JobPublisher = require('../../../batch/pubsub/job-publisher');
+var assert = require('assert');
+
+describe('batch API job publisher', function () {
+    beforeEach(function () {
+        var self = this;
+        this.host = 'irrelevantHost';
+        this.redis = {
+            createClient: function () {
+                return this;
+            },
+            publish: function () {
+                var isValidFirstArg = arguments[0] === Channel.NAME;
+                var isValidSecondArg = arguments[1] === self.host;
+                self.redis.publishIsCalledWithValidArgs = isValidFirstArg && isValidSecondArg;
+            },
+            on: function () {},
+            ping: function (cb) {
+                cb();
+            }
+        };
+        this.pool = {
+            acquire: function (db, cb) {
+                cb(null, self.redis);
+            }
+        };
+
+        this.jobPublisher = new JobPublisher(this.pool);
+    });
+
+    it('.publish() should publish new messages', function () {
+        this.jobPublisher.publish(this.host);
+        assert.ok(this.redis.publishIsCalledWithValidArgs);
+    });
+
+});

test/unit/batch/job_queue.js

@@ -0,0 +1,49 @@
+'use strict';
+
+var JobQueue = require('../../../batch/job_queue');
+var assert = require('assert');
+
+describe('batch API job queue', function () {
+    beforeEach(function () {
+        this.metadataBackend = {
+            redisCmd: function () {
+                var callback = arguments[arguments.length -1];
+                process.nextTick(function () {
+                    callback(null, 'irrelevantJob');
+                });
+            },
+            redisMultiCmd: function () {
+                var callback = arguments[arguments.length -1];
+                process.nextTick(function () {
+                    callback(null, 'irrelevantJob');
+                });
+            }
+        };
+        this.jobPublisher = {
+            publish: function () {}
+        };
+        this.jobQueue = new JobQueue(this.metadataBackend, this.jobPublisher);
+    });
+
+    it('.enqueue() should enqueue the provided job', function (done) {
+        this.jobQueue.enqueue('irrelevantJob', 'irrelevantHost', function (err) {
+            assert.ok(!err);
+            done();
+        });
+    });
+
+    it('.dequeue() should dequeue the next job', function (done) {
+        this.jobQueue.dequeue('irrelevantHost', function (err) {
+            assert.ok(!err);
+            done();
+        });
+    });
+
+    it('.enqueueFirst() should dequeue the next job', function (done) {
+        this.jobQueue.enqueueFirst('irrelevantJob', 'irrelevantHost', function (err) {
+            assert.ok(!err);
+            done();
+        });
+    });
+
+});

test/unit/batch/job_subscriber.js

@@ -0,0 +1,71 @@
+'use strict';
+
+var Channel = require('../../../batch/pubsub/channel');
+var JobSubscriber = require('../../../batch/pubsub/job-subscriber');
+var assert = require('assert');
+
+describe('batch API job subscriber', function () {
+    beforeEach(function () {
+        var self = this;
+
+        this.onMessageListener = function () {};
+        this.redis = {
+            createClient: function () {
+                return this;
+            },
+            subscribe: function () {
+                var isValidFirstArg = arguments[0] === Channel.NAME;
+                self.redis.subscribeIsCalledWithValidArgs = isValidFirstArg;
+            },
+            on: function () {
+                if (arguments[0] === 'message') {
+                    self.redis.onIsCalledWithValidArgs = true;
+                }
+            },
+            unsubscribe: function () {
+                var isValidFirstArg = arguments[0] === Channel.NAME;
+                self.redis.unsubscribeIsCalledWithValidArgs = isValidFirstArg;
+            },
+            scan: function(params, callback) {
+                return callback(null, ['0']);
+            },
+            removeAllListeners: function () {
+                return this;
+            },
+            smembers: function (key, callback) {
+                callback(null, []);
+            },
+            connected: true,
+        };
+        this.pool = {
+            acquire: function (db, cb) {
+                cb(null, self.redis);
+            },
+            release: function(/*db, client*/) {
+
+            }
+        };
+        this.queueSeeker = {
+            seek: function () {
+                var callback = arguments[1];
+
+                callback(null, []);
+            }
+        };
+
+        this.jobSubscriber = new JobSubscriber(this.pool, this.queueSeeker);
+    });
+
+    it('.subscribe() should listen for incoming messages', function () {
+        this.jobSubscriber.subscribe(this.onMessageListener);
+        assert.ok(this.redis.onIsCalledWithValidArgs);
+        assert.ok(this.redis.subscribeIsCalledWithValidArgs);
+    });
+
+    it('.unsubscribe() should stop listening for incoming messages', function () {
+        this.jobSubscriber.subscribe(this.onMessageListener);
+        this.jobSubscriber.unsubscribe();
+        assert.ok(this.redis.unsubscribeIsCalledWithValidArgs);
+    });
+
+});

test/unit/error_handler.test.js

@@ -0,0 +1,143 @@
+'use strict';
+
+var assert = require('assert');
+var errorMiddleware = require('../../app/middlewares/error');
+require('../helper');
+
+const req = { query: { callback: true } };
+
+const getRes = () => {
+    return {
+        headers: {},
+        set (key, value) {
+            this.headers[key] = value;
+        },
+        header (key, value) {
+            this.set(key, value);
+        },
+        statusCode: 0,
+        status (status) {
+            this.statusCode = status;
+        },
+        json () {},
+        jsonp () {}
+    };
+};
+
+const getErrorHeader = (context, detail, hint, message) => {
+    return {
+        context,
+        detail,
+        hint,
+        statusCode: 400,
+        message
+    };
+};
+
+describe('error-handler', function() {
+    it('should return a header with errors', function (done) {
+
+        let error = new Error('error test');
+        error.detail = 'test detail';
+        error.hint = 'test hint';
+        error.context = 'test context';
+
+        const errorHeader = getErrorHeader(
+            error.context,
+            error.detail,
+            error.hint,
+            error.message
+        );
+
+        const res = getRes();
+
+        errorMiddleware()(error, req, res, function next () {
+            assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+            assert.deepEqual(
+                res.headers['X-SQLAPI-Errors'],
+                JSON.stringify(errorHeader)
+            );
+
+            done();
+        });
+    });
+
+    it('JSONP should return a header with error statuscode', function (done) {
+        let error = new Error('error test');
+        error.detail = 'test detail';
+        error.hint = 'test hint';
+        error.context = 'test context';
+
+        const errorHeader = getErrorHeader(
+            error.context,
+            error.detail,
+            error.hint,
+            error.message
+        );
+
+        const res = getRes();        
+
+        errorMiddleware()(error, req, res, function next () {
+            assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+            assert.deepEqual(
+                res.headers['X-SQLAPI-Errors'],
+                JSON.stringify(errorHeader)
+            );
+
+            done();
+        });
+    });
+
+    it('should escape chars that broke logs regex', function (done) {
+        const badString = 'error: ( ) = " \" \' * $ & |';
+        const escapedString = 'error                     ';
+
+        let error = new Error(badString);
+        error.detail = badString;
+        error.hint = badString;
+        error.context = badString;
+
+        const errorHeader = getErrorHeader(
+            escapedString,
+            escapedString,
+            escapedString,
+            escapedString
+        );
+
+        const res = getRes();
+        
+        errorMiddleware()(error, req, res, function () {
+            assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+            assert.deepEqual(
+                res.headers['X-SQLAPI-Errors'],
+                JSON.stringify(errorHeader)
+            );
+
+            done();
+        });
+    });
+
+    it('should truncat too long error messages', function (done) {
+        const veryLongString = 'Very long error message '.repeat(1000);
+        const truncatedString = veryLongString.substring(0, 1024);
+
+        let error = new Error(veryLongString);
+
+        const expectedErrorHeader = {
+            statusCode: 400,
+            message: truncatedString
+	};
+
+        const res = getRes();
+
+        errorMiddleware()(error, req, res, function () {
+            assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+            assert.deepEqual(
+                res.headers['X-SQLAPI-Errors'],
+                JSON.stringify(expectedErrorHeader)
+            );
+
+            done();
+        });
+    });
+});

test/unit/error_handler_factory.test.js

@@ -0,0 +1,71 @@
+'use strict';
+
+const assert = require('assert');
+const errorHandlerFactory = require('../../app/services/error_handler_factory');
+const ErrorHandler = require('../../app/services/error_handler');
+const { codeToCondition } = require('../../app/postgresql/error_codes');
+
+let rateLimitError = new Error(
+    'You are over platform\'s limits. Please contact us to know more details'
+);
+rateLimitError.http_status = 429;
+rateLimitError.context = 'limit';
+rateLimitError.detail = 'rate-limit';
+
+const cases = [
+    {
+        title: 'postgres error',
+        error: new Error(codeToCondition['02000'])
+    },
+    {
+        title: 'rate limit error',
+        error: rateLimitError
+    }
+];
+
+describe('error-handler-factory', function () {
+    cases.forEach(({ title, error }) => {
+        it(title, function () {
+            const errorHandler = errorHandlerFactory(error);
+            const expectedError = new ErrorHandler({
+                message: error.message,
+                context: error.context,
+                detail: error.detail,
+                hint: error.hint,
+                http_status: error.http_status,
+                name: codeToCondition[error.code] || error.name
+            });
+
+            assert.deepEqual(errorHandler, expectedError);
+        });
+    });
+
+    it('timeout error', function() {
+        const error = new Error('statement timeout');
+        const errorHandler = errorHandlerFactory(error);
+        const expectedError = new ErrorHandler({
+            message: 'You are over platform\'s limits: SQL query timeout error.' +
+                ' Refactor your query before running again or contact CARTO support for more details.',
+            context: 'limit',
+            detail: 'datasource',
+            http_status: 429
+        });
+
+        assert.deepEqual(errorHandler, expectedError);
+    });
+
+    it('permission denied error', function() {
+        const error = new Error('permission denied');
+        const errorHandler = errorHandlerFactory(error);
+        const expectedError = new ErrorHandler({
+            message: error.message,
+            context: error.context,
+            detail: error.detail,
+            hint: error.hint,
+            http_status: 403,
+            name: codeToCondition[error.code] || error.name
+        });
+
+        assert.deepEqual(errorHandler, expectedError);
+    });
+});

test/unit/health_check.js

@@ -0,0 +1,47 @@
+'use strict';
+
+require('../helper');
+
+var assert      = require('assert');
+var HealthCheck = require('../../app/monitoring/health_check');
+
+var metadataBackend = {};
+
+function PSQL(dbParams) {
+    this.params = dbParams;
+}
+
+var healthCheck = new HealthCheck(metadataBackend, PSQL);
+
+describe('health checks', function() {
+
+    it('errors if disabled file exists', function(done) {
+      var fs = require('fs');
+
+      var readFileFn = fs.readFile;
+      fs.readFile = function(filename, callback) {
+        callback(null, "Maintenance");
+      };
+      healthCheck.check(function(err) {
+        assert.equal(err.message, "Maintenance");
+        assert.equal(err.http_status, 503);
+        fs.readFile = readFileFn;
+        done();
+      });
+    });
+
+    it('does not err if disabled file does not exists', function(done) {
+      var fs = require('fs');
+
+      var readFileFn = fs.readFile;
+      fs.readFile = function(filename, callback) {
+        callback(new Error("ENOENT"), null);
+      };
+      healthCheck.check(function(err) {
+        assert.equal(err, null);
+        fs.readFile = readFileFn;
+        done();
+      });
+    });
+
+});

test/unit/model/bin_encoder.js

@@ -0,0 +1,76 @@
+'use strict';
+
+require('../../helper');
+var assert = require('assert');
+
+var ArrayBufferSer = require('../../../app/models/bin_encoder');
+
+describe('ArrayBufferSer', function() {
+
+  it('calculate size for basic types', function() {
+    var b = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2,3,4]);
+    assert.equal(4*2, b.getDataSize());
+
+    b = new ArrayBufferSer(ArrayBufferSer.INT8, [1,2,3,4]);
+    assert.equal(4, b.getDataSize());
+
+    b = new ArrayBufferSer(ArrayBufferSer.INT32, [1,2,3,4]);
+    assert.equal(4*4, b.getDataSize());
+  });
+
+
+  it('calculate size for arrays', function() {
+    var b = new ArrayBufferSer(ArrayBufferSer.STRING, ["test","kease"]);
+    assert.equal((b.headerSize + 4 + 5)*2, b.getDataSize());
+
+    var ba = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2,3,4]);
+    var bc = new ArrayBufferSer(ArrayBufferSer.INT16, [1,4]);
+
+    b = new ArrayBufferSer(ArrayBufferSer.BUFFER, [ba, bc]);
+    assert.equal((b.headerSize + 4 + 2)*2, b.getDataSize());
+    assert.equal(b.type, ArrayBufferSer.BUFFER);
+  });
+
+  function assert_buffer_equals(a, b) {
+    assert.equal(a.length, b.length);
+    for(var i = 0; i < a.length; ++i) {
+      assert.equal(a[i], b[i], "byte i " + i + " is different: " + a[i] + " != " + b[i]);
+    }
+  }
+
+  it('binary data is ok', function() {
+    var b = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2,3,4]);
+    var bf = new Buffer([0, 0, 0, ArrayBufferSer.INT16, 0, 0, 0, 8, 1, 0, 2, 0, 3, 0, 4, 0]);
+    assert_buffer_equals(bf, b.buffer);
+  });
+
+  it('binary data is ok with arrays', function() {
+    var ba = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2, 3, 4]);
+    var bc = new ArrayBufferSer(ArrayBufferSer.INT16, [1,4]);
+
+    var b = new ArrayBufferSer(ArrayBufferSer.BUFFER, [ba, bc]);
+    var bf = new Buffer([
+      0, 0, 0, ArrayBufferSer.BUFFER, // type
+      0, 0, 0, 28,
+      0, 0, 0, ArrayBufferSer.INT16, 0, 0, 0, 8, 1, 0, 2, 0, 3, 0, 4, 0,
+      0, 0, 0, ArrayBufferSer.INT16, 0, 0, 0, 4, 1, 0, 4, 0]);
+    assert_buffer_equals(bf, b.buffer);
+  });
+
+  it('binary data is ok with strings', function() {
+    var s = 'test';
+    var b = new ArrayBufferSer(ArrayBufferSer.STRING, [s]);
+    var bf = new Buffer([
+      0, 0, 0, ArrayBufferSer.STRING, // type
+      0, 0, 0, 16,
+      0, 0, 0, ArrayBufferSer.UINT16,
+      0, 0, 0, 8,
+      s.charCodeAt(0), 0,
+      s.charCodeAt(1), 0,
+      s.charCodeAt(2), 0,
+      s.charCodeAt(3), 0
+    ]);
+    assert_buffer_equals(bf, b.buffer);
+  });
+
+});

test/unit/oauth.test.js

@@ -0,0 +1,176 @@
+'use strict';
+
+require('../helper');
+
+var _ = require('underscore');
+var OAuthAuth = require('../../app/auth/oauth');
+var MetadataDB = require('cartodb-redis');
+var oAuth = require('../../app/auth/oauth').backend;
+var assert = require('assert');
+var oauth_data_1 = {
+        oauth_consumer_key: "dpf43f3p2l4k3l03",
+        oauth_token: "nnch734d00sl2jdk",
+        oauth_signature_method: "HMAC-SHA1",
+        oauth_signature: "tR3%2BTy81lMeYAr%2FFid0kMTYa%2FWM%3D",
+        oauth_timestamp:"1191242096",
+        oauth_nonce:"kllo9940pd9333jh"
+};
+var oauth_data_2 = { oauth_version:"1.0" };
+var oauth_data = _.extend(oauth_data_1, oauth_data_2);
+var real_oauth_header = 'OAuth ' +
+    'realm="http://vizzuality.testhost.lan/",' +
+    'oauth_consumer_key="fZeNGv5iYayvItgDYHUbot1Ukb5rVyX6QAg8GaY2",' +
+    'oauth_token="l0lPbtP68ao8NfStCiA3V3neqfM03JKhToxhUQTR",' +
+    'oauth_signature_method="HMAC-SHA1", ' +
+    'oauth_signature="o4hx4hWP6KtLyFwggnYB4yPK8xI%3D",' +
+    'oauth_timestamp="1313581372",' +
+    'oauth_nonce="W0zUmvyC4eVL8cBd4YwlH1nnPTbxW0QBYcWkXTwe4",' +
+    'oauth_version="1.0"';
+var oauth_header_tokens = 'oauth_consumer_key="dpf43f3p2l4k3l03",' +
+    'oauth_token="nnch734d00sl2jdk",' +
+    'oauth_signature_method="HMAC-SHA1", ' +
+    'oauth_signature="tR3%2BTy81lMeYAr%2FFid0kMTYa%2FWM%3D",' +
+    'oauth_timestamp="1191242096",' +
+    'oauth_nonce="kllo9940pd9333jh",' +
+    'oauth_version="1.0"';
+var full_oauth_header = 'OAuth realm="http://photos.example.net/"' + oauth_header_tokens;
+
+var metadataBackend = new MetadataDB({
+    host: global.settings.redis_host,
+    port: global.settings.redis_port,
+    max: global.settings.redisPool,
+    idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
+    reapIntervalMillis: global.settings.redisReapIntervalMillis
+});
+
+describe('oauth', function() {
+
+it('test database number', function(){
+    assert.equal(oAuth.oauth_database, 3);
+});
+
+it('test oauth database key', function(){
+    assert.equal(oAuth.oauth_user_key, "rails:oauth_access_tokens:<%= oauth_access_key %>");
+});
+
+it('test parse tokens from full headers does not raise exception', function(){
+    var req = {query:{}, headers:{authorization:full_oauth_header}};
+    assert.doesNotThrow(function(){ oAuth.parseTokens(req); }, /incomplete oauth tokens in request/);
+});
+
+it('test parse all normal tokens raises no exception', function(){
+    var req = {query:oauth_data, headers:{}};
+    assert.doesNotThrow(function(){ oAuth.parseTokens(req); }, /incomplete oauth tokens in request/);
+});
+
+it('test headers take presedence over query parameters', function(){
+    var req = {query:{oauth_signature_method: "MY_HASH"}, headers:{authorization:full_oauth_header}};
+    var tokens = oAuth.parseTokens(req);
+    assert.equal(tokens.oauth_signature_method, "HMAC-SHA1");
+});
+
+it('test can access oauth hash for a user based on access token (oauth_token)', function(done){
+    var req = {query:{}, headers:{authorization:real_oauth_header}};
+    var tokens = oAuth.parseTokens(req);
+
+    oAuth.getOAuthHash(metadataBackend, tokens.oauth_token, function(err, data){
+        assert.equal(tokens.oauth_consumer_key, data.consumer_key);
+        done();
+    });
+});
+
+it('test non existant oauth hash for a user based on oauth_token returns empty hash', function(done){
+    var req = {query:{}, params: { user: 'vizzuality' }, headers:{authorization:full_oauth_header}};
+    var tokens = oAuth.parseTokens(req);
+
+    oAuth.getOAuthHash(metadataBackend, tokens.oauth_token, function(err, data){
+        assert.ok(!err, err);
+        assert.deepEqual(data, {});
+        done();
+    });
+});
+
+it('can return user for verified signature', function(done){
+    var req = {query:{},
+        headers:{authorization:real_oauth_header, host: 'vizzuality.testhost.lan' },
+        params: { user: 'vizzuality' },
+        protocol: 'http',
+        method: 'GET',
+        path: '/api/v1/tables'
+    };
+
+    oAuth.verifyRequest(req, metadataBackend, function(err, data){
+        assert.ok(!err, err);
+        assert.equal(data, 'master');
+        done();
+    });
+});
+
+it('can return user for verified signature (for other allowed domains)', function(done){
+    var oAuthGetAllowedHostsFn = oAuth.getAllowedHosts;
+    oAuth.getAllowedHosts = function() {
+        return ['testhost.lan', 'testhostdb.lan'];
+    };
+    var req = {query:{},
+        headers:{authorization:real_oauth_header, host: 'vizzuality.testhostdb.lan' },
+        params: { user: 'vizzuality' },
+        protocol: 'http',
+        method: 'GET',
+        path: '/api/v1/tables'
+    };
+
+    oAuth.verifyRequest(req, metadataBackend, function(err, data){
+        oAuth.getAllowedHosts = oAuthGetAllowedHostsFn;
+        assert.ok(!err, err);
+        assert.equal(data, 'master');
+        done();
+    });
+});
+
+it('returns null user for unverified signatures', function(done){
+    var req = {query:{},
+        headers:{authorization:real_oauth_header, host: 'vizzuality.testyhost.lan' },
+        params: { user: 'vizzuality' },
+        protocol: 'http',
+        method: 'GET',
+        path: '/api/v1/tables'
+    };
+
+    oAuth.verifyRequest(req, metadataBackend, function(err, data){
+        assert.equal(data, null);
+        done();
+    });
+});
+
+it('returns null user for no oauth', function(done){
+    var req = {
+        query:{},
+        headers:{},
+        params: { user: 'vizzuality' },
+        protocol: 'http',
+        method: 'GET',
+        path: '/api/v1/tables'
+    };
+
+    oAuth.verifyRequest(req, metadataBackend, function(err,data){
+        assert.equal(data, null);
+        done();
+    });
+});
+
+it('OAuthAuth reports it has credentials', function(done) {
+    var req = {query:{}, headers:{authorization:real_oauth_header}};
+    var oAuthAuth = new OAuthAuth(req);
+    assert.ok(oAuthAuth.hasCredentials());
+    done();
+});
+
+it('OAuthAuth reports it has no credentials', function(done) {
+    var req = {query:{}, headers:{}};
+    var oAuthAuth = new OAuthAuth(req);
+    assert.equal(oAuthAuth.hasCredentials(), false);
+    done();
+});
+
+
+});

test/unit/pg-entities-access-validator.test.js

@@ -0,0 +1,175 @@
+'use strict';
+
+const assert = require('assert');
+const pgEntitiesAccessValidator = require('../../app/services/pg-entities-access-validator');
+
+const fakeAffectedTables = [{
+    schema_name: 'schema',
+    table_name: 'untitled_table'
+}];
+
+const fakeAffectedTablesCarto = [{
+    schema_name: 'carto',
+    table_name: 'untitled_table'
+}];
+
+const fakeAffectedTablesCartodbOK = [{
+    schema_name: 'cartodb',
+    table_name: 'untitled_table'
+}];
+
+const fakeAffectedTablesCartodbKO = [
+    {
+        schema_name: 'cartodb',
+        table_name: 'untitled_table'
+    },
+    {
+        schema_name: 'cartodb',
+        table_name: 'cdb_tablemetadata'
+    }
+];
+
+const fakeAffectedTablesPgcatalog = [{
+    schema_name: 'pg_catalog',
+    table_name: 'pg_catalog'
+}];
+
+const fakeAffectedTablesInfo = [{
+    schema_name: 'information_schema',
+    table_name: 'untitled_table'
+}];
+
+const fakeAffectedTablesPublicOK = [{
+    schema_name: 'public',
+    table_name: 'untitled_table'
+}];
+
+const fakeAffectedTablesPublicKO = [
+    {
+        schema_name: 'public',
+        table_name: 'spatial_ref_sys'
+    },
+    {
+        schema_name: 'public',
+        table_name: 'untitled_table'
+    }
+];
+
+const fakeAffectedTablesTopologyOK = [{
+    schema_name: 'topology',
+    table_name: 'untitled_table'
+}];
+
+const fakeAffectedTablesTopologyKO = [
+    {
+        schema_name: 'topology',
+        table_name: 'layer'
+    },
+    {
+        schema_name: 'topology',
+        table_name: 'untitled_table'
+    }
+];
+
+
+
+describe('pg entities access validator with validatePGEntitiesAccess enabled', function () {
+    before(function() {
+        global.settings.validatePGEntitiesAccess = true;
+    });
+
+    after(function() {
+        global.settings.validatePGEntitiesAccess = false;
+    });
+
+    it('validate function: bad parameters', function () {
+        assert.strictEqual(pgEntitiesAccessValidator.validate(), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate(null), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate(null, null), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate([]), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate([], 3), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate({ tables: [] }, false), true);
+    });
+
+    it('validate function: should be validated', function () {
+        assert.strictEqual(pgEntitiesAccessValidator.validate({ tables: fakeAffectedTables }), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesCartodbOK }), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesPublicOK }), true);
+        assert.strictEqual(pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesTopologyOK }), true);
+    });
+
+    it('validate function: should not be validated', function () {
+        let authorizationLevel = 'master';
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesCarto }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesCartodbKO }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesPgcatalog }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesInfo }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesPublicKO }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesTopologyKO }, authorizationLevel),
+            false
+        );
+
+
+        authorizationLevel = 'regular';
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesCarto }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesCartodbKO }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesPgcatalog }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesInfo }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesPublicKO }, authorizationLevel),
+            false
+        );
+        assert.strictEqual(
+            pgEntitiesAccessValidator.validate({ tables: fakeAffectedTablesTopologyKO }, authorizationLevel),
+            false
+        );
+    });
+
+    it('hardValidation function', function () {
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTables), true);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesCartodbOK), true);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesPublicOK), true);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesTopologyOK), true);
+
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesCarto), false);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesCartodbKO), false);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesPgcatalog), false);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesInfo), false);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesPublicKO), false);
+        assert.strictEqual(pgEntitiesAccessValidator.hardValidation(fakeAffectedTablesTopologyKO), false);
+    });
+
+    it('softValidation function', function () {
+        assert.strictEqual(pgEntitiesAccessValidator.softValidation(fakeAffectedTablesCartodbKO), true);
+        assert.strictEqual(pgEntitiesAccessValidator.softValidation(fakeAffectedTablesPgcatalog), false);
+    });
+
+});

test/unit/query_info.test.js

@@ -0,0 +1,66 @@
+'use strict';
+
+const assert = require('assert');
+const queryInfo = require('../../app/utils/query_info');
+
+describe('query info', function () {
+    describe('copy format', function () {
+        describe('csv', function () {
+            const validQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT  CSV, DELIMITER ',', HEADER true)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV , DELIMITER ',', HEADER true)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV)",
+                "COPY copy_endpoints_test FROM STDIN WITH(FORMAT csv,HEADER true)"
+            ];
+
+            validQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.equal(result, 'CSV');
+                });
+            });
+        });
+
+        describe('text', function() {
+            const validQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT TEXT)",
+                "COPY copy_endpoints_test (id, name) FROM STDIN",
+            ];
+
+            validQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.equal(result, 'TEXT');
+                });
+            });
+        });
+
+        describe('binary', function() {
+            const validQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT BINARY)",
+            ];
+
+            validQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.equal(result, 'BINARY');
+                });
+            });
+        });
+
+        describe('should fail', function() {
+            const validQueries = [
+                "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT ERROR)",
+                "SELECT * from copy_endpoints_test"
+            ];
+
+            validQueries.forEach(query => {
+                it(query, function() {
+                    const result = queryInfo.getFormatFromCopyQuery(query);
+                    assert.strictEqual(result, false);
+                });
+            });
+        });
+    });
+});

test/websocket_test/app.js

@@ -0,0 +1,59 @@
+'use strict';
+
+// this is a test to understand accessing sql api via websockets
+var express = require('express')
+  , app     = express.createServer(
+    express.logger({
+        buffer: true,
+        format: '[:date] :req[X-Real-IP] \033[90m:method\033[0m \033[36m:req[Host]:url\033[0m \033[90m:status :response-time ms -> :res[Content-Type]\033[0m'
+    }))
+    , Step     = require('step')
+    , _        = require('underscore');
+
+app.use(express.bodyParser());
+app.use(express.static(__dirname + '/public'));
+app.enable('jsonp callback');
+
+var io = require('socket.io');
+io = io.listen(app);
+
+io.configure('development', function(){
+    io.set('log level', 1);
+    io.set('origins', '*:*');
+});
+
+app.listen(8080);
+
+// hacked postgres setup
+//var pg = require('pg');
+var pg = require('pg').native //native libpq bindings = `
+var conString = "tcp://postgres@localhost/cartodb_dev_user_2_db";
+
+var client = new pg.Client(conString);
+client.connect();
+
+
+io.sockets.on('connection', function (socket) {
+  socket.emit('news', { hello: 'world' });
+  socket.on('my other event', function (data) {
+    console.log(data);
+  });
+
+  socket.on('sql_query', function(data){
+
+      var query = client.query(data.sql);
+      var id = data.id;
+
+      query.on('row', function(row) {
+          socket.emit("sql_result", {r:row, id:id, state:1})
+      });
+
+      query.on('end',function(){
+          socket.emit("sql_result", {id:id, state:0});
+      });
+
+      query.on('error', function(row){
+          socket.emit("sql_result", {r:row, id:id, state:-1})
+      });
+  });
+});

test/websocket_test/public/gmaps_mercator.js

@@ -0,0 +1,151 @@
+'use strict';
+
+  function Point(x, y) {
+      this.x = x;
+      this.y = y;
+  }
+
+  /** return a copy of this point with coordinates as int */
+  Point.prototype.floor = function() {
+      return new Point(this.x>>0, this.y>>0);
+  }
+
+  function LatLng(lat, lng) {
+      this.lat = lat;
+      this.lng = lng;
+  }
+
+  LatLng.prototype.clone = function() {
+      return new LatLng(this.lat, this.lng);
+  }
+
+  var TILE_SIZE = 256;
+
+  MercatorProjection.prototype.TILE_SIZE = TILE_SIZE;
+
+  function bound(value, opt_min, opt_max) {
+    if (opt_min != null) value = Math.max(value, opt_min);
+    if (opt_max != null) value = Math.min(value, opt_max);
+    return value;
+  }
+
+  function degreesToRadians(deg) {
+    return deg * (Math.PI / 180);
+  }
+
+  function radiansToDegrees(rad) {
+    return rad / (Math.PI / 180);
+  }
+
+  function MercatorProjection() {
+    this.pixelOrigin_ = new Point(TILE_SIZE / 2,
+        TILE_SIZE / 2);
+    this.pixelsPerLonDegree_ = TILE_SIZE / 360;
+    this.pixelsPerLonRadian_ = TILE_SIZE / (2 * Math.PI);
+  }
+
+  MercatorProjection.prototype.fromLatLngToPixel = function(latLng, zoom) {
+      var p = this.fromLatLngToPoint(latLng);
+      return this.toPixelCoordinate(p, zoom);
+  };
+
+  MercatorProjection.prototype.fromLatLngToPoint = function(latLng,
+      opt_point) {
+    var me = this;
+    var point = opt_point || new Point(0, 0);
+    var origin = me.pixelOrigin_;
+
+    point.x = origin.x + latLng.lng * me.pixelsPerLonDegree_;
+
+    // NOTE(appleton): Truncating to 0.9999 effectively limits latitude to
+    // 89.189.  This is about a third of a tile past the edge of the world
+    // tile.
+    var siny = bound(Math.sin(degreesToRadians(latLng.lat)), -0.9999,
+        0.9999);
+    point.y = origin.y + 0.5 * Math.log((1 + siny) / (1 - siny)) *
+        -me.pixelsPerLonRadian_;
+    return point;
+  };
+
+  MercatorProjection.prototype.fromPointToLatLng = function(point) {
+    var me = this;
+    var origin = me.pixelOrigin_;
+    var lng = (point.x - origin.x) / me.pixelsPerLonDegree_;
+    var latRadians = (point.y - origin.y) / -me.pixelsPerLonRadian_;
+    var lat = radiansToDegrees(2 * Math.atan(Math.exp(latRadians)) -
+        Math.PI / 2);
+    return new LatLng(lat, lng);
+  };
+
+  MercatorProjection.prototype.tileBBox = function(x, y, zoom) {
+    var numTiles = 1 << zoom;
+    var inc = TILE_SIZE/numTiles;
+    var px = x*TILE_SIZE/numTiles;
+    var py = y*TILE_SIZE/numTiles;
+    return [
+        this.fromPointToLatLng(new Point(px, py + inc)),
+        this.fromPointToLatLng(new Point(px + inc, py))
+    ];
+  };
+
+  MercatorProjection.prototype.tilePoint = function(x, y, zoom) {
+        var numTiles = 1 << zoom;
+        var px = x*TILE_SIZE;
+        var py = y*TILE_SIZE;
+        return [px, py];
+  }
+  MercatorProjection.prototype.fromPixelToLatLng = function(pixel, zoom) {
+        var numTiles = 1 << zoom;
+        var p = new Point(
+                pixel.x/numTiles,
+                pixel.y/numTiles);
+        return this.fromPointToLatLng(p);
+
+
+  }
+
+  MercatorProjection.prototype.toPixelCoordinate = function(worldCoordinate, zoom) {
+        var numTiles = 1 << zoom;
+        return new Point(
+                worldCoordinate.x * numTiles,
+                worldCoordinate.y * numTiles);
+  }
+
+  MercatorProjection.prototype.latLngToTilePoint = function(latLng, x, y, zoom) {
+        var numTiles = 1 << zoom;
+        var projection = this;
+        var worldCoordinate = projection.fromLatLngToPoint(latLng);
+        var pixelCoordinate = new Point(
+                worldCoordinate.x * numTiles,
+                worldCoordinate.y * numTiles);
+        var tp = this.tilePoint(x, y, zoom);
+        return new Point(
+                Math.floor(pixelCoordinate.x - tp[0]),
+                Math.floor(pixelCoordinate.y - tp[1]));
+  }
+
+  MercatorProjection.prototype.pixelToTile = function(pixelCoordinate) {
+        return new Point(
+                Math.floor(pixelCoordinate.x / TILE_SIZE),
+                Math.floor(pixelCoordinate.y / TILE_SIZE));
+  };
+
+  MercatorProjection.prototype.pointToTile = function(point, zoom) {
+        var numTiles = 1 << zoom;
+        var pixelCoordinate = new Point(
+                point.x * numTiles,
+                point.y * numTiles);
+        return this.pixelToTile(pixelCoordinate);
+  };
+
+  MercatorProjection.prototype.latLngToTile = function(latLng, zoom) {
+        var numTiles = 1 << zoom;
+        var projection = this;
+        var worldCoordinate = projection.fromLatLngToPoint(latLng);
+        var pixelCoordinate = new Point(
+                worldCoordinate.x * numTiles,
+                worldCoordinate.y * numTiles);
+        return new Point(
+                Math.floor(pixelCoordinate.x / TILE_SIZE),
+                Math.floor(pixelCoordinate.y / TILE_SIZE));
+  }