cartodb4/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

first commit

Browse Source
This commit is contained in:
zhongjin
2023-05-19 00:42:48 +08:00
commit 53de9c6c51
243 changed files with 39485 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

83
test/acceptance/batch/batch-drain.test.js Normal file
View File

@@ -0,0 +1,83 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var batchFactory = require('../../../batch/index');
var JobPublisher = require('../../../batch/pubsub/job-publisher');
var JobQueue = require('../../../batch/job_queue');
var JobBackend = require('../../../batch/job_backend');
var JobService = require('../../../batch/job_service');
var JobCanceller = require('../../../batch/job_canceller');
var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
describe('batch module', function() {
var dbInstance = 'localhost';
var username = 'vizzuality';
var pool = redisUtils.getPool();
var jobPublisher = new JobPublisher(pool);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);
var jobCanceller = new JobCanceller();
var jobService = new JobService(jobBackend, jobCanceller);
before(function (done) {
this.batch = batchFactory(metadataBackend, pool);
this.batch.start();
this.batch.on('ready', done);
});
after(function (done) {
this.batch.stop();
redisUtils.clean('batch:*', done);
});
function createJob(sql, done) {
var data = {
user: username,
query: sql,
host: dbInstance,
dbname: 'cartodb_test_user_1_db',
dbuser: 'test_cartodb_user_1',
port: 5432,
pass: 'test_cartodb_user_1_pass',
};
jobService.create(data, function (err, job) {
if (err) {
return done(err);
}
done(null, job.serialize());
});
}
it('should drain the current job', function (done) {
var self = this;
createJob('select pg_sleep(3)', function (err, job) {
if (err) {
return done(err);
}
setTimeout(function () {
jobBackend.get(job.job_id, function (err, job) {
if (err) {
done(err);
}
assert.equal(job.status, 'running');
self.batch.drain(function () {
jobBackend.get(job.job_id, function (err, job) {
if (err) {
done(err);
}
assert.equal(job.status, 'pending');
done();
});
});
});
}, 50);
});
});
});

52
test/acceptance/batch/batch-limits.test.js Normal file
View File

@@ -0,0 +1,52 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
var redisUtils = require('../../support/redis_utils');
var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() });
const db_utils = require('../../support/db_utils');
describe('batch query statement_timeout limit', function() {
before(function(done) {
this.batchTestClient = new BatchTestClient();
this.batchQueryTimeout = global.settings.batch_query_timeout;
global.settings.batch_query_timeout = 15000;
metadataBackend.redisCmd(5, 'HMSET', ['limits:batch:vizzuality', 'timeout', 100], done);
});
before(db_utils.resetPgBouncerConnections);
after(function(done) {
global.settings.batch_query_timeout = this.batchQueryTimeout;
redisUtils.clean('limits:batch:*', function() {
this.batchTestClient.drain(done);
}.bind(this));
});
after(db_utils.resetPgBouncerConnections);
function jobPayload(query) {
return {
query: query
};
}
it('should cancel with user statement_timeout limit', function (done) {
var payload = jobPayload('select pg_sleep(10)');
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
assert.ok(job.failed_reason.match(/statement.*timeout/));
return done();
});
});
});
});

236
test/acceptance/batch/batch.multiquery.test.js Normal file
View File

@@ -0,0 +1,236 @@
'use strict';
require('../../helper');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
var assert = require('../../support/assert');
var queue = require('queue-async');
describe('batch multiquery', function() {
function jobPayload(query) {
return {
query: query
};
}
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function (done) {
this.batchTestClient.drain(done);
});
it('should perform one multiquery job with two queries', function (done) {
var queries = [
'select pg_sleep(0)',
'select pg_sleep(0)'
];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
return done();
});
});
});
it('should perform one multiquery job with two queries and fail on last one', function (done) {
var queries = [
'select pg_sleep(0)',
'select shouldFail()'
];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
return done();
});
});
});
it('should perform one multiquery job with three queries and fail on last one', function (done) {
var queries = [
'select pg_sleep(0)',
'select pg_sleep(0)',
'select shouldFail()'
];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
return done();
});
});
});
it('should perform one multiquery job with three queries and fail on second one', function (done) {
var queries = [
'select pg_sleep(0)',
'select shouldFail()',
'select pg_sleep(0)'
];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
return done();
});
});
});
it('should perform two multiquery job with two queries for each one', function (done) {
var self = this;
var jobs = [
[
'select pg_sleep(0)',
'select pg_sleep(0)'
],
[
'select pg_sleep(0)',
'select pg_sleep(0)'
]
];
var jobsQueue = queue(1);
jobs.forEach(function(job) {
jobsQueue.defer(function(payload, done) {
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(done);
});
}, jobPayload(job));
});
jobsQueue.awaitAll(function (err, jobsCreated) {
if (err) {
return done(err);
}
jobsCreated.forEach(function(job) {
assert.equal(job.status, JobStatus.DONE);
});
return done();
});
});
it('should perform two multiquery job with two queries for each one and fail the first one', function (done) {
var self = this;
var jobs = [
[
'select pg_sleep(0)',
'select shouldFail()'
],
[
'select pg_sleep(0)',
'select pg_sleep(0)'
]
];
var expectedStatus = [JobStatus.FAILED, JobStatus.DONE];
var jobsQueue = queue(1);
jobs.forEach(function(job) {
jobsQueue.defer(function(payload, done) {
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(done);
});
}, jobPayload(job));
});
jobsQueue.awaitAll(function (err, jobsCreated) {
if (err) {
return done(err);
}
var statuses = jobsCreated.map(function(job) {
return job.status;
});
assert.deepEqual(statuses, expectedStatus);
return done();
});
});
it('should perform two multiquery job with two queries for each one and fail the second one', function (done) {
var self = this;
var jobs = [
[
'select pg_sleep(0)',
'select pg_sleep(0)'
],
[
'select pg_sleep(0)',
'select shouldFail()'
]
];
var expectedStatus = [JobStatus.DONE, JobStatus.FAILED];
var jobsQueue = queue(1);
jobs.forEach(function(job) {
jobsQueue.defer(function(payload, done) {
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(done);
});
}, jobPayload(job));
});
jobsQueue.awaitAll(function (err, jobsCreated) {
if (err) {
return done(err);
}
var statuses = jobsCreated.map(function(job) {
return job.status;
});
assert.deepEqual(statuses, expectedStatus);
return done();
});
});
});

197
test/acceptance/batch/batch.test.js Normal file
View File

@@ -0,0 +1,197 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var queue = require('queue-async');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('batch happy cases', function() {
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function(done) {
this.batchTestClient.drain(done);
});
function jobPayload(query) {
return {
query: query
};
}
it('should perform job with select', function (done) {
var payload = jobPayload('select * from private_table');
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
return done();
});
});
});
it('should perform job with select into', function (done) {
var payload = jobPayload('select * into batch_test_table from (select * from private_table) as job');
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
return done();
});
});
});
it('should perform job with select from result table', function (done) {
var payload = jobPayload('select * from batch_test_table');
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
return done();
});
});
});
it('should perform all enqueued jobs', function (done) {
var self = this;
var jobs = [
'select * from private_table',
'select * from private_table',
'select * from private_table',
];
var jobsQueue = queue(1);
jobs.forEach(function(job) {
jobsQueue.defer(function(payload, done) {
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(done);
});
}, jobPayload(job));
});
jobsQueue.awaitAll(function (err, jobsCreated) {
if (err) {
return done(err);
}
jobsCreated.forEach(function(job) {
assert.equal(job.status, JobStatus.DONE);
});
return done();
});
});
it('should set all job as failed', function (done) {
var self = this;
var jobs = [
'select * from unexistent_table',
'select * from unexistent_table',
'select * from unexistent_table'
];
var jobsQueue = queue(1);
jobs.forEach(function(job) {
jobsQueue.defer(function(payload, done) {
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(done);
});
}, jobPayload(job));
});
jobsQueue.awaitAll(function (err, jobsCreated) {
if (err) {
return done(err);
}
jobsCreated.forEach(function(job) {
assert.equal(job.status, JobStatus.FAILED);
});
return done();
});
});
it('should perform job with array of select', function (done) {
var queries = ['select * from private_table limit 1', 'select * from private_table'];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
return done();
});
});
});
it('should set job as failed if last query fails', function (done) {
var queries = ['select * from private_table', 'select * from undefined_table'];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
return done();
});
});
});
it('should set job as failed if first query fails', function (done) {
var queries = ['select * from undefined_table', 'select * from private_table'];
var payload = jobPayload(queries);
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
return done();
});
});
});
});

99
test/acceptance/batch/batch.wip.test.js Normal file
View File

@@ -0,0 +1,99 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('batch work in progress endpoint happy cases', function() {
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function(done) {
this.batchTestClient.drain(done);
});
function jobPayload(query) {
return {
query: query
};
}
it('should get a list of work in progress jobs group by user', function (done) {
var self = this;
var user = 'vizzuality';
var queries = ['select pg_sleep(3)'];
var payload = jobPayload(queries);
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.RUNNING, function (err) {
if (err) {
return done(err);
}
self.batchTestClient.getWorkInProgressJobs(function (err, workInProgressJobs) {
if (err) {
return done(err);
}
if (!workInProgressJobs[user]) {
return done(new Error('User should be in work-in-progress list'));
}
assert.ok(Array.isArray(workInProgressJobs[user]));
assert.ok(workInProgressJobs[user].length >= 1);
for (var i = 0; i < workInProgressJobs[user].length; i++) {
if (workInProgressJobs[user][i] === jobResult.job.job_id) {
return jobResult.cancel(done);
}
}
return done(new Error('Job should not be in work-in-progress list'));
});
});
});
});
it('should get a list of work in progress jobs w/o the finished ones', function (done) {
var self = this;
var user = 'vizzuality';
var queries = ['select pg_sleep(0)'];
var payload = jobPayload(queries);
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err) {
if (err) {
return done(err);
}
self.batchTestClient.getWorkInProgressJobs(function (err, workInProgressJobs) {
if (err) {
return done(err);
}
if (workInProgressJobs[user]) {
assert.ok(Array.isArray(workInProgressJobs[user]));
assert.ok(workInProgressJobs[user].length >= 1);
for (var i = 0; i < workInProgressJobs[user].length; i++) {
if (workInProgressJobs[user][i] === jobResult.job.job_id) {
return done(new Error('Job should not be in work-in-progress list'));
}
}
}
done();
});
});
});
});
});

126
test/acceptance/batch/job.callback-template.test.js Normal file
View File

@@ -0,0 +1,126 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
var JobStatus = require('../../../batch/job_status');
var BatchTestClient = require('../../support/batch-test-client');
describe('Batch API callback templates', function () {
before(function () {
this.batchTestClient = new BatchTestClient();
this.testClient = new TestClient();
});
after(function (done) {
this.batchTestClient.drain(done);
});
it('should use templates for error_message and job_id onerror callback' +
' and keep the original templated query but use the error message', function (done) {
var self = this;
var payload = {
"query": {
"query": [
{
"query": "SELECT * FROM invalid_table",
"onerror": "INSERT INTO test_batch_errors " +
"values ('<%= job_id %>', '<%= error_message %>')"
}
]
}
};
var expectedQuery = {
query: [
{
"query": "SELECT * FROM invalid_table",
"onerror": "INSERT INTO test_batch_errors values ('<%= job_id %>', '<%= error_message %>')",
status: 'failed',
fallback_status: 'done'
}
]
};
self.testClient.getResult(
'create table test_batch_errors (job_id text, error_message text)', function (err) {
if (err) {
return done(err);
}
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.FAILED, function (err, job) {
if (err) {
return done(err);
}
jobResult.validateExpectedResponse(expectedQuery);
self.testClient.getResult('select * from test_batch_errors', function(err, rows) {
if (err) {
return done(err);
}
assert.equal(rows[0].job_id, job.job_id);
assert.equal(rows[0].error_message, 'relation "invalid_table" does not exist');
self.testClient.getResult('drop table test_batch_errors', done);
});
});
});
});
});
it('should use template for job_id onsuccess callback ' +
'and keep the original templated query but use the job_id', function (done) {
var self = this;
var payload = {
"query": {
"query": [
{
query: "create table batch_jobs (job_id text)"
},
{
"query": "SELECT 1",
"onsuccess": "INSERT INTO batch_jobs values ('<%= job_id %>')"
}
]
}
};
var expectedQuery = {
query: [
{
query: "create table batch_jobs (job_id text)",
status: 'done'
},
{
query: "SELECT 1",
onsuccess: "INSERT INTO batch_jobs values ('<%= job_id %>')",
status: 'done',
fallback_status: 'done'
}
]
};
self.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
jobResult.validateExpectedResponse(expectedQuery);
self.testClient.getResult('select * from batch_jobs', function(err, rows) {
if (err) {
return done(err);
}
assert.equal(rows[0].job_id, job.job_id);
self.testClient.getResult('drop table batch_jobs', done);
});
});
});
});
});

934
test/acceptance/batch/job.fallback.test.js Normal file
View File

@@ -0,0 +1,934 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var JobStatus = require('../../../batch/job_status');
var BatchTestClient = require('../../support/batch-test-client');
describe('Batc API fallback job', function () {
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function(done) {
this.batchTestClient.drain(done);
});
it('"onsuccess" on first query should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4",
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4",
"onsuccess": "SELECT * FROM untitle_table_4 limit 1",
"status": "done",
"fallback_status": "done"
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onerror" on first query should not be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4",
onerror: "SELECT * FROM untitle_table_4 limit 1"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4",
"onerror": "SELECT * FROM untitle_table_4 limit 1",
"status": "done",
"fallback_status": "skipped"
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onerror" on first query should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM nonexistent_table /* query should fail */",
onerror: "SELECT * FROM untitle_table_4 limit 1"
}]
}
};
var expectedQuery = {
query: [{
query: 'SELECT * FROM nonexistent_table /* query should fail */',
onerror: 'SELECT * FROM untitle_table_4 limit 1',
status: 'failed',
fallback_status: 'done',
failed_reason: 'relation "nonexistent_table" does not exist'
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" on first query should not be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM nonexistent_table /* query should fail */",
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}]
}
};
var expectedQuery = {
query: [{
query: 'SELECT * FROM nonexistent_table /* query should fail */',
onsuccess: 'SELECT * FROM untitle_table_4 limit 1',
status: 'failed',
fallback_status: 'skipped',
failed_reason: 'relation "nonexistent_table" does not exist'
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4",
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4",
"status": "done"
}],
"onsuccess": "SELECT * FROM untitle_table_4 limit 1"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
assert.equal(job.fallback_status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" should not be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM nonexistent_table /* query should fail */",
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM nonexistent_table /* query should fail */",
"status": "failed",
"failed_reason": 'relation "nonexistent_table" does not exist'
}],
"onsuccess": "SELECT * FROM untitle_table_4 limit 1"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
assert.equal(job.fallback_status, JobStatus.SKIPPED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onerror" should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM nonexistent_table /* query should fail */"
}],
onerror: "SELECT * FROM untitle_table_4 limit 1"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM nonexistent_table /* query should fail */",
"status": "failed",
"failed_reason": 'relation "nonexistent_table" does not exist'
}],
"onerror": "SELECT * FROM untitle_table_4 limit 1"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
assert.equal(job.fallback_status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onerror" should not be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4",
}],
onerror: "SELECT * FROM untitle_table_4 limit 1"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4",
"status": "done"
}],
"onerror": "SELECT * FROM untitle_table_4 limit 1"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
assert.equal(job.fallback_status, JobStatus.SKIPPED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" & "onsuccess" on query should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4",
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 2"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4",
"onsuccess": "SELECT * FROM untitle_table_4 limit 1",
"status": "done",
"fallback_status": "done"
}],
"onsuccess": "SELECT * FROM untitle_table_4 limit 2"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
assert.equal(job.fallback_status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for each query should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4",
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}, {
query: "SELECT * FROM untitle_table_4 limit 2",
onsuccess: "SELECT * FROM untitle_table_4 limit 3"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4",
"onsuccess": "SELECT * FROM untitle_table_4 limit 1",
"status": "done",
"fallback_status": "done"
}, {
"query": "SELECT * FROM untitle_table_4 limit 2",
"onsuccess": "SELECT * FROM untitle_table_4 limit 3",
"status": "done",
"fallback_status": "done"
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for each query should not be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM nonexistent_table /* should fail */",
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}, {
query: "SELECT * FROM untitle_table_4 limit 2",
onsuccess: "SELECT * FROM untitle_table_4 limit 3"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM nonexistent_table /* should fail */",
"onsuccess": "SELECT * FROM untitle_table_4 limit 1",
"status": "failed",
"fallback_status": "skipped",
"failed_reason": 'relation "nonexistent_table" does not exist'
}, {
"query": "SELECT * FROM untitle_table_4 limit 2",
"onsuccess": "SELECT * FROM untitle_table_4 limit 3",
"status": "skipped",
"fallback_status": "skipped"
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for second query should not be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 2",
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}, {
query: "SELECT * FROM nonexistent_table /* should fail */",
onsuccess: "SELECT * FROM untitle_table_4 limit 3"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4 limit 2",
"onsuccess": "SELECT * FROM untitle_table_4 limit 1",
"status": "done",
"fallback_status": "done"
}, {
"query": "SELECT * FROM nonexistent_table /* should fail */",
"onsuccess": "SELECT * FROM untitle_table_4 limit 3",
"status": "failed",
"fallback_status": "skipped",
"failed_reason": 'relation "nonexistent_table" does not exist'
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onerror" should not be triggered for any query and "skipped"', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1",
onerror: "SELECT * FROM untitle_table_4 limit 2"
}, {
query: "SELECT * FROM untitle_table_4 limit 3",
onerror: "SELECT * FROM untitle_table_4 limit 4"
}]
}
};
var expectedQuery = {
query: [{
query: 'SELECT * FROM untitle_table_4 limit 1',
onerror: 'SELECT * FROM untitle_table_4 limit 2',
status: 'done',
fallback_status: 'skipped'
}, {
query: 'SELECT * FROM untitle_table_4 limit 3',
onerror: 'SELECT * FROM untitle_table_4 limit 4',
status: 'done',
fallback_status: 'skipped'
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" should be "skipped"', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */",
onsuccess: "SELECT * FROM untitle_table_4 limit 2"
}]
}
};
var expectedQuery = {
query: [{
query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */',
onsuccess: 'SELECT * FROM untitle_table_4 limit 2',
status: 'failed',
fallback_status: 'skipped',
failed_reason: 'syntax error at end of input'
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" should not be triggered and "skipped"', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */",
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 2"
}
};
var expectedQuery = {
query: [{
query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */',
status: 'failed',
failed_reason: 'syntax error at end of input'
}],
onsuccess: 'SELECT * FROM untitle_table_4 limit 2'
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for first query should fail', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1",
onsuccess: "SELECT * FROM nonexistent_table /* should fail */"
}, {
query: "SELECT * FROM untitle_table_4 limit 2",
onsuccess: "SELECT * FROM untitle_table_4 limit 3"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4 limit 1",
"onsuccess": "SELECT * FROM nonexistent_table /* should fail */",
"status": "done",
"fallback_status": "failed",
"failed_reason": 'relation "nonexistent_table" does not exist'
}, {
"query": "SELECT * FROM untitle_table_4 limit 2",
"onsuccess": "SELECT * FROM untitle_table_4 limit 3",
"status": "done",
"fallback_status": "done"
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for second query should fail', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1",
onsuccess: "SELECT * FROM untitle_table_4 limit 2"
}, {
query: "SELECT * FROM untitle_table_4 limit 3",
onsuccess: "SELECT * FROM nonexistent_table /* should fail */"
}]
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4 limit 1",
"onsuccess": "SELECT * FROM untitle_table_4 limit 2",
"status": "done",
"fallback_status": "done"
}, {
"query": "SELECT * FROM untitle_table_4 limit 3",
"onsuccess": "SELECT * FROM nonexistent_table /* should fail */",
"status": "done",
"fallback_status": "failed",
"failed_reason": 'relation "nonexistent_table" does not exist'
}]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for job & "onsuccess" for each query should be triggered', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1",
onsuccess: "SELECT * FROM untitle_table_4 limit 2"
}, {
query: "SELECT * FROM untitle_table_4 limit 3",
onsuccess: "SELECT * FROM untitle_table_4 limit 4"
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 5"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4 limit 1",
"onsuccess": "SELECT * FROM untitle_table_4 limit 2",
"status": "done",
"fallback_status": "done"
}, {
"query": "SELECT * FROM untitle_table_4 limit 3",
"onsuccess": "SELECT * FROM untitle_table_4 limit 4",
"status": "done",
"fallback_status": "done"
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 5"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
assert.equal(job.fallback_status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for job & "onsuccess" for each query should be triggered ' +
'(even second "onsuccess" fails job should be done)', function (done) {
var payload = {
query: {
query: [{
query: "SELECT * FROM untitle_table_4 limit 1",
onsuccess: "SELECT * FROM untitle_table_4 limit 2"
}, {
query: "SELECT * FROM untitle_table_4 limit 3",
onsuccess: "SELECT * FROM nonexistent_table /* should fail */"
}],
onsuccess: "SELECT * FROM untitle_table_4 limit 5"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT * FROM untitle_table_4 limit 1",
"onsuccess": "SELECT * FROM untitle_table_4 limit 2",
"status": "done",
"fallback_status": "done"
}, {
"query": "SELECT * FROM untitle_table_4 limit 3",
"onsuccess": "SELECT * FROM nonexistent_table /* should fail */",
"status": "done",
"fallback_status": "failed",
"failed_reason": 'relation "nonexistent_table" does not exist'
}],
"onsuccess": "SELECT * FROM untitle_table_4 limit 5"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
assert.equal(job.fallback_status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('should fail first "onerror" and job "onerror" and skip the other ones', function (done) {
var payload = {
"query": {
"query": [{
"query": "SELECT * FROM atm_madrid limit 1, should fail",
"onerror": "SELECT * FROM atm_madrid limit 2"
}, {
"query": "SELECT * FROM atm_madrid limit 3",
"onerror": "SELECT * FROM atm_madrid limit 4"
}],
"onerror": "SELECT * FROM atm_madrid limit 5"
}
};
var expectedQuery = {
query: [{
query: 'SELECT * FROM atm_madrid limit 1, should fail',
onerror: 'SELECT * FROM atm_madrid limit 2',
status: 'failed',
fallback_status: 'failed',
failed_reason: 'relation "atm_madrid" does not exist'
}, {
query: 'SELECT * FROM atm_madrid limit 3',
onerror: 'SELECT * FROM atm_madrid limit 4',
status: 'skipped',
fallback_status: 'skipped'
}],
onerror: 'SELECT * FROM atm_madrid limit 5'
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
assert.equal(job.fallback_status, JobStatus.FAILED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('should run first "onerror" and job "onerror" and skip the other ones', function (done) {
var payload = {
"query": {
"query": [{
"query": "SELECT * FROM untitle_table_4 limit 1, should fail",
"onerror": "SELECT * FROM untitle_table_4 limit 2"
}, {
"query": "SELECT * FROM untitle_table_4 limit 3",
"onerror": "SELECT * FROM untitle_table_4 limit 4"
}],
"onerror": "SELECT * FROM untitle_table_4 limit 5"
}
};
var expectedQuery = {
"query": [
{
"query": "SELECT * FROM untitle_table_4 limit 1, should fail",
"onerror": "SELECT * FROM untitle_table_4 limit 2",
"status": "failed",
"fallback_status": "done",
"failed_reason": "LIMIT #,# syntax is not supported"
},
{
"query": "SELECT * FROM untitle_table_4 limit 3",
"onerror": "SELECT * FROM untitle_table_4 limit 4",
"status": "skipped",
"fallback_status": "skipped"
}
],
"onerror": "SELECT * FROM untitle_table_4 limit 5"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
assert.equal(job.fallback_status, JobStatus.DONE);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
it('"onsuccess" for job & "onsuccess" for each query should not be triggered ' +
' because it has been cancelled', function (done) {
var payload = {
query: {
query: [{
query: "SELECT pg_sleep(3)",
onsuccess: "SELECT pg_sleep(0)"
}],
onsuccess: "SELECT pg_sleep(0)"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT pg_sleep(3)",
"onsuccess": "SELECT pg_sleep(0)",
"status": "cancelled",
"fallback_status": "skipped"
}],
"onsuccess": "SELECT pg_sleep(0)"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.RUNNING);
jobResult.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
assert.equal(job.fallback_status, JobStatus.SKIPPED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
});
it('first "onsuccess" should be triggered and it will be cancelled', function (done) {
var payload = {
query: {
query: [{
query: "SELECT pg_sleep(0)",
onsuccess: "SELECT pg_sleep(3)"
}],
onsuccess: "SELECT pg_sleep(0)"
}
};
var expectedQuery = {
"query": [{
"query": "SELECT pg_sleep(0)",
"onsuccess": "SELECT pg_sleep(3)",
"status": "done",
"fallback_status": "cancelled"
}],
"onsuccess": "SELECT pg_sleep(0)"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.RUNNING);
jobResult.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
assert.equal(job.fallback_status, JobStatus.SKIPPED);
jobResult.validateExpectedResponse(expectedQuery);
return done();
});
});
});
});
});

125
test/acceptance/batch/job.query.limit.test.js Normal file
View File

@@ -0,0 +1,125 @@
'use strict';
/**
*
* Requires the database and tables setup in config/environments/test.js to exist
* Ensure the user is present in the pgbouncer auth file too
* TODO: Add OAuth tests.
*
* To run this test, ensure that cartodb_test_user_1_db metadata exists
* in Redis for the vizzuality.cartodb.com domain
*
* SELECT 5
* HSET rails:users:vizzuality id 1
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../../helper');
var JobController = require('../../../app/controllers/job_controller');
var redisUtils = require('../../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('qs');
function payload(query) {
return JSON.stringify({query: query});
}
function payloadSize(query) {
return payload(query).length;
}
var minPayloadSize = payloadSize('');
var queryMaxSize = new Array(JobController.MAX_LIMIT_QUERY_SIZE_IN_BYTES - minPayloadSize + 1).join('a');
var queryTooLong = queryMaxSize.concat('a');
describe('job query limit', function() {
function expectedErrorMessage(query) {
return JobController.getMaxSizeErrorMessage(payload(query));
}
after(function (done) {
redisUtils.clean('batch:*', done);
});
it('POST /api/v2/sql/job with a invalid query size should respond with 400 query too long', function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: queryTooLong
})
}, {
status: 400
}, function (err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [expectedErrorMessage(queryTooLong)] });
done();
});
});
it('POST /api/v2/sql/job with a valid query size should respond with 201 created', function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: queryMaxSize
})
}, {
status: 201
}, function (err, res) {
var job = JSON.parse(res.body);
assert.ok(job.job_id);
done();
});
});
it('POST /api/v2/sql/job with a invalid query size should consider multiple queries', function (done){
var queries = [queryTooLong, 'select 1'];
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: queries
})
}, {
status: 400
}, function (err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [expectedErrorMessage(queries)] });
done();
});
});
it('POST /api/v2/sql/job with a invalid query size should consider fallback queries/callbacks', function (done){
var fallbackQueries = {
query: [{
query: queryTooLong,
onsuccess: "SELECT * FROM untitle_table_4 limit 1"
}, {
query: "SELECT * FROM untitle_table_4 limit 2",
onsuccess: "SELECT * FROM untitle_table_4 limit 3"
}]
};
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: fallbackQueries
})
}, {
status: 400
}, function (err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [expectedErrorMessage(fallbackQueries)] });
done();
});
});
});

59
test/acceptance/batch/job.query.order.test.js Normal file
View File

@@ -0,0 +1,59 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('job query order', function() {
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function (done) {
return this.batchTestClient.drain(done);
});
function createJob(queries) {
return {
query: queries
};
}
it('should run job queries in order (single consumer)', function (done) {
var jobRequest1 = createJob(["select 1", "select 2"]);
var jobRequest2 = createJob(["select 3"]);
this.batchTestClient.createJob(jobRequest1, function(err, jobResult1) {
if (err) {
return done(err);
}
this.batchTestClient.createJob(jobRequest2, function(err, jobResult2) {
if (err) {
return done(err);
}
jobResult1.getStatus(function (err, job1) {
if (err) {
return done(err);
}
jobResult2.getStatus(function(err, job2) {
if (err) {
return done(err);
}
assert.equal(job1.status, JobStatus.DONE);
assert.equal(job2.status, JobStatus.DONE);
assert.ok(
new Date(job1.updated_at).getTime() < new Date(job2.updated_at).getTime(),
'job1 (' + job1.updated_at + ') should finish before job2 (' + job2.updated_at + ')'
);
done();
});
});
});
}.bind(this));
});
});

99
test/acceptance/batch/job.query.timeout.test.js Normal file
View File

@@ -0,0 +1,99 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('job query timeout', function() {
before(function() {
this.batchQueryTimeout = global.settings.batch_query_timeout;
this.batchTestClient = new BatchTestClient();
});
after(function (done) {
global.settings.batch_query_timeout = this.batchQueryTimeout;
return this.batchTestClient.drain(done);
});
function createTimeoutQuery(query, timeout) {
return {
query: {
query: [
{
timeout: timeout,
query: query
}
]
}
};
}
it('should run query with higher user timeout', function (done) {
var jobRequest = createTimeoutQuery("select pg_sleep(0.1)", 200);
this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function(err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
done();
});
});
});
it('should fail to run query with lower user timeout', function (done) {
var jobRequest = createTimeoutQuery("select pg_sleep(0.1)", 50);
this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function(err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
done();
});
});
});
it('should fail to run query with user timeout if it is higher than config', function (done) {
global.settings.batch_query_timeout = 100;
var jobRequest = createTimeoutQuery("select pg_sleep(1)", 2000);
this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function(err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
done();
});
});
});
it('should fail to run query with user timeout if set to 0 (ignored timeout)', function (done) {
global.settings.batch_query_timeout = 100;
var jobRequest = createTimeoutQuery("select pg_sleep(1)", 0);
this.batchTestClient.createJob(jobRequest, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function(err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.FAILED);
done();
});
});
});
});

219
test/acceptance/batch/job.test.js Normal file
View File

@@ -0,0 +1,219 @@
'use strict';
/**
*
* Requires the database and tables setup in config/environments/test.js to exist
* Ensure the user is present in the pgbouncer auth file too
* TODO: Add OAuth tests.
*
* To run this test, ensure that cartodb_test_user_1_db metadata exists
* in Redis for the vizzuality.cartodb.com domain
*
* SELECT 5
* HSET rails:users:vizzuality id 1
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../../helper');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
describe('job module', function() {
var job = {};
after(function (done) {
redisUtils.clean('batch:*', done);
});
it('POST /api/v2/sql/job should respond with 200 and the created job', function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: "SELECT * FROM untitle_table_4"
})
}, {
status: 201
}, function(err, res) {
job = JSON.parse(res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.ok(job.job_id);
assert.equal(job.query, "SELECT * FROM untitle_table_4");
assert.equal(job.user, "vizzuality");
done();
});
});
it('POST /api/v2/sql/job without query should respond with 400 and the corresponding message of error',
function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({})
}, {
status: 400
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] });
done();
});
});
it('POST /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
q: "SELECT * FROM untitle_table_4"
})
}, {
status: 400
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] });
done();
});
});
it('POST /api/v2/sql/job with wrong api key should respond with 401 permission denied', function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=wrong',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: "SELECT * FROM untitle_table_4"
})
}, {
status: 401
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'Unauthorized' ] });
done();
});
});
it('POST /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){
assert.response(server, {
url: '/api/v2/sql/job?api_key=wrong',
headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({
query: "SELECT * FROM untitle_table_4"
})
}, {
status: 404
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, {
error: [
'Sorry, we can\'t find CARTO user \'wrong-host\'. ' +
'Please check that you have entered the correct domain.'
]
});
done();
});
});
it('GET /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(err, res) {
var jobGot = JSON.parse(res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.equal(jobGot.query, "SELECT * FROM untitle_table_4");
assert.equal(jobGot.user, "vizzuality");
done();
});
});
it('GET /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 401
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: ['Unauthorized'] });
done();
});
});
it('GET /api/v2/sql/job/:jobId with wrong jobId header respond with 400 and an error', function (done){
assert.response(server, {
url: '/api/v2/sql/job/irrelevantJob?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 400
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error , {
error: ['Job with id irrelevantJob not found']
});
done();
});
});
it('DELETE /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(err, res) {
var jobCancelled = JSON.parse(res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.equal(jobCancelled.job_id, job.job_id);
assert.equal(jobCancelled.query, "SELECT * FROM untitle_table_4");
assert.equal(jobCancelled.user, "vizzuality");
assert.equal(jobCancelled.status, "cancelled");
done();
});
});
it('DELETE /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 401
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: ['Unauthorized'] });
done();
});
});
it('DELETE /api/v2/sql/job/ with wrong host header respond with 404 not found', function (done){
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 404
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error , {
error: [
'Sorry, we can\'t find CARTO user \'wrong-host\'. ' +
'Please check that you have entered the correct domain.'
]
});
done();
});
});
});

115
test/acceptance/batch/job.timing.test.js Normal file
View File

@@ -0,0 +1,115 @@
'use strict';
require('../../helper');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('Batch API query timing', function () {
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function(done) {
this.batchTestClient.drain(done);
});
it('should report start and end time for each query with fallback queries' +
'and expose started_at and ended_at for all queries with fallback mechanism', function (done) {
var expectedQuery = {
query: [{
query: 'SELECT * FROM untitle_table_4 limit 1',
onerror: 'SELECT * FROM untitle_table_4 limit 2',
status: 'done',
fallback_status: 'skipped'
}, {
query: 'SELECT * FROM untitle_table_4 limit 3',
onerror: 'SELECT * FROM untitle_table_4 limit 4',
status: 'done',
fallback_status: 'skipped'
}],
onerror: 'SELECT * FROM untitle_table_4 limit 5'
};
var payload = {
"query": {
"query": [
{
"query": "SELECT * FROM untitle_table_4 limit 1",
"onerror": "SELECT * FROM untitle_table_4 limit 2"
},
{
"query": "SELECT * FROM untitle_table_4 limit 3",
"onerror": "SELECT * FROM untitle_table_4 limit 4"
}
],
"onerror": "SELECT * FROM untitle_table_4 limit 5"
}
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(function (err) {
if (err) {
return done(err);
}
jobResult.validateExpectedResponse(expectedQuery);
done();
});
});
});
it('should report start and end time for each query also for failing queries' +
'and expose started_at and ended_at for all queries with fallback mechanism (failed)', function (done) {
var expectedQuery = {
query: [{
query: 'SELECT * FROM untitle_table_4 limit 1',
onerror: 'SELECT * FROM untitle_table_4 limit 2',
status: 'done',
fallback_status: 'skipped'
}, {
query: 'SELECT * FROM untitle_table_4 limit 3 failed',
onerror: 'SELECT * FROM untitle_table_4 limit 4',
status: 'failed',
fallback_status: 'done'
}],
onerror: 'SELECT * FROM untitle_table_4 limit 5'
};
var payload = {
"query": {
"query": [
{
"query": "SELECT * FROM untitle_table_4 limit 1",
"onerror": "SELECT * FROM untitle_table_4 limit 2"
},
{
"query": "SELECT * FROM untitle_table_4 limit 3 failed",
"onerror": "SELECT * FROM untitle_table_4 limit 4"
}
],
"onerror": "SELECT * FROM untitle_table_4 limit 5"
}
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.FAILED, function (err) {
if (err) {
return done(err);
}
jobResult.validateExpectedResponse(expectedQuery);
done();
});
});
});
});

129
test/acceptance/batch/leader-multiple-users-query-order.test.js Normal file
View File

@@ -0,0 +1,129 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('multiple batch clients and users, job query order', function() {
before(function(done) {
this.batchTestClientA = new BatchTestClient({ name: 'consumerA' });
this.batchTestClientB = new BatchTestClient({ name: 'consumerB' });
this.testClient = new TestClient();
this.testClient.getResult(
[
'drop table if exists ordered_inserts_a',
'drop table if exists ordered_inserts_bbbbb',
'create table ordered_inserts_a (status numeric)',
'create table ordered_inserts_bbbbb (status numeric)'
].join(';'),
done
);
});
after(function (done) {
this.batchTestClientA.drain(function(err) {
if (err) {
return done(err);
}
this.batchTestClientB.drain(done);
}.bind(this));
});
function createJob(queries) {
return {
query: queries
};
}
it('should run job queries in order (multiple consumers)', function (done) {
var jobRequestA1 = createJob([
"insert into ordered_inserts_a values(1)",
"select pg_sleep(0.25)",
"insert into ordered_inserts_a values(2)"
]);
var jobRequestA2 = createJob([
"insert into ordered_inserts_a values(3)"
]);
var jobRequestB1 = createJob([
"insert into ordered_inserts_bbbbb values(1)"
]);
var self = this;
this.batchTestClientA.createJob(jobRequestA1, function(err, jobResultA1) {
if (err) {
return done(err);
}
var override = { host: 'cartodb250user.cartodb.com' };
self.batchTestClientB.createJob(jobRequestB1, override, function(err, jobResultB1) {
if (err) {
return done(err);
}
// we don't care about the producer
self.batchTestClientB.createJob(jobRequestA2, function(err, jobResultA2) {
if (err) {
return done(err);
}
jobResultA1.getStatus(function (err, jobA1) {
if (err) {
return done(err);
}
jobResultA2.getStatus(function(err, jobA2) {
if (err) {
return done(err);
}
jobResultB1.getStatus(function(err, jobB1) {
assert.equal(jobA1.status, JobStatus.DONE);
assert.equal(jobA2.status, JobStatus.DONE);
assert.equal(jobB1.status, JobStatus.DONE);
assert.ok(
new Date(jobA1.updated_at).getTime() < new Date(jobA2.updated_at).getTime(),
'A1 (' + jobA1.updated_at + ') ' +
'should finish before A2 (' + jobA2.updated_at + ')'
);
assert.ok(
new Date(jobB1.updated_at).getTime() < new Date(jobA1.updated_at).getTime(),
'B1 (' + jobA1.updated_at + ') ' +
'should finish before A1 (' + jobA1.updated_at + ')'
);
function statusMapper (status) { return { status: status }; }
self.testClient.getResult('select * from ordered_inserts_a', function(err, rows) {
assert.ok(!err);
// cartodb250user and vizzuality test users share database
var expectedRows = [1, 2, 3].map(statusMapper);
assert.deepEqual(rows, expectedRows);
var query = 'select * from ordered_inserts_bbbbb';
self.testClient.getResult(query, override, function(err, rows) {
assert.ok(!err);
var expectedRows = [1].map(statusMapper);
assert.deepEqual(rows, expectedRows);
done();
});
});
});
});
});
});
});
});
});
});

88
test/acceptance/batch/leader.job.query.order.test.js Normal file
View File

@@ -0,0 +1,88 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('multiple batch clients job query order', function() {
before(function(done) {
this.batchTestClient1 = new BatchTestClient({ name: 'consumerA' });
this.batchTestClient2 = new BatchTestClient({ name: 'consumerB' });
this.testClient = new TestClient();
this.testClient.getResult(
'drop table if exists ordered_inserts; create table ordered_inserts (status numeric)',
done
);
});
after(function (done) {
this.batchTestClient1.drain(function(err) {
if (err) {
return done(err);
}
this.batchTestClient2.drain(done);
}.bind(this));
});
function createJob(queries) {
return {
query: queries
};
}
it('should run job queries in order (multiple consumers)', function (done) {
var jobRequest1 = createJob([
"insert into ordered_inserts values(1)",
"select pg_sleep(0.25)",
"insert into ordered_inserts values(2)"
]);
var jobRequest2 = createJob([
"insert into ordered_inserts values(3)"
]);
var self = this;
this.batchTestClient1.createJob(jobRequest1, function(err, jobResult1) {
if (err) {
return done(err);
}
this.batchTestClient2.createJob(jobRequest2, function(err, jobResult2) {
if (err) {
return done(err);
}
jobResult1.getStatus(function (err, job1) {
if (err) {
return done(err);
}
jobResult2.getStatus(function(err, job2) {
if (err) {
return done(err);
}
assert.equal(job1.status, JobStatus.DONE);
assert.equal(job2.status, JobStatus.DONE);
self.testClient.getResult('select * from ordered_inserts', function(err, rows) {
assert.ok(!err);
assert.deepEqual(rows, [{ status: 1 }, { status: 2 }, { status: 3 }]);
assert.ok(
new Date(job1.updated_at).getTime() < new Date(job2.updated_at).getTime(),
'job1 (' + job1.updated_at + ') should finish before job2 (' + job2.updated_at + ')'
);
done();
});
});
});
});
}.bind(this));
});
});

68
test/acceptance/batch/queued-jobs-limit.test.js Normal file
View File

@@ -0,0 +1,68 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var TestClient = require('../../support/test-client');
describe('max queued jobs', function() {
before(function(done) {
this.batch_max_queued_jobs = global.settings.batch_max_queued_jobs;
global.settings.batch_max_queued_jobs = 1;
this.server = require('../../../app/server')();
this.testClient = new TestClient();
this.testClient.getResult(
'drop table if exists max_queued_jobs_inserts; create table max_queued_jobs_inserts (status numeric)',
done
);
});
after(function (done) {
global.settings.batch_max_queued_jobs = this.batch_max_queued_jobs;
redisUtils.clean('batch:*', done);
});
function createJob(server, status, callback) {
assert.response(
server,
{
url: '/api/v2/sql/job?api_key=1234',
headers: {
host: 'vizzuality.cartodb.com',
'Content-Type': 'application/json'
},
method: 'POST',
data: JSON.stringify({
query: "insert into max_queued_jobs_inserts values (1)"
})
},
{
status: status
},
function(err, res) {
if (err) {
return callback(err);
}
return callback(null, JSON.parse(res.body));
}
);
}
it('POST /api/v2/sql/job should respond with 200 and the created job', function (done) {
var self = this;
createJob(this.server, 201, function(err) {
assert.ok(!err);
createJob(self.server, 400, function(err, res) {
assert.ok(!err);
assert.equal(res.error[0], "Failed to create job. Max number of jobs (" +
global.settings.batch_max_queued_jobs + ") queued reached");
done();
});
});
});
});

99
test/acceptance/batch/scheduler-basic.test.js Normal file
View File

@@ -0,0 +1,99 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
var BatchTestClient = require('../../support/batch-test-client');
var JobStatus = require('../../../batch/job_status');
describe('basic scheduling', function() {
before(function(done) {
this.batchTestClientA = new BatchTestClient({ name: 'consumerA' });
this.batchTestClientB = new BatchTestClient({ name: 'consumerB' });
this.testClient = new TestClient();
this.testClient.getResult(
[
'drop table if exists ordered_inserts_a',
'create table ordered_inserts_a (status numeric)'
].join(';'),
done
);
});
after(function (done) {
this.batchTestClientA.drain(function(err) {
if (err) {
return done(err);
}
this.batchTestClientB.drain(done);
}.bind(this));
});
function createJob(queries) {
return {
query: queries
};
}
it('should run job queries in order (multiple consumers)', function (done) {
var jobRequestA1 = createJob([
"insert into ordered_inserts_a values(1)",
"select pg_sleep(0.25)",
"insert into ordered_inserts_a values(2)"
]);
var jobRequestA2 = createJob([
"insert into ordered_inserts_a values(3)"
]);
var self = this;
this.batchTestClientA.createJob(jobRequestA1, function(err, jobResultA1) {
if (err) {
return done(err);
}
// we don't care about the producer
self.batchTestClientB.createJob(jobRequestA2, function(err, jobResultA2) {
if (err) {
return done(err);
}
jobResultA1.getStatus(function (err, jobA1) {
if (err) {
return done(err);
}
jobResultA2.getStatus(function(err, jobA2) {
if (err) {
return done(err);
}
assert.equal(jobA1.status, JobStatus.DONE);
assert.equal(jobA2.status, JobStatus.DONE);
assert.ok(
new Date(jobA1.updated_at).getTime() < new Date(jobA2.updated_at).getTime(),
'A1 (' + jobA1.updated_at + ') ' +
'should finish before A2 (' + jobA2.updated_at + ')'
);
function statusMapper (status) { return { status: status }; }
self.testClient.getResult('select * from ordered_inserts_a', function(err, rows) {
assert.ok(!err);
// cartodb250user and vizzuality test users share database
var expectedRows = [1, 2, 3].map(statusMapper);
assert.deepEqual(rows, expectedRows);
return done();
});
});
});
});
});
});
});

190
test/acceptance/batch/use-cases.test.js Normal file
View File

@@ -0,0 +1,190 @@
'use strict';
require('../../helper');
var assert = require('../../support/assert');
var JobStatus = require('../../../batch/job_status');
var BatchTestClient = require('../../support/batch-test-client');
describe('Use cases', function () {
before(function() {
this.batchTestClient = new BatchTestClient();
});
after(function(done) {
this.batchTestClient.drain(done);
});
it('cancel a done job should return an error', function (done) {
var payload = {
query: "SELECT * FROM untitle_table_4"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.DONE, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.DONE);
jobResult.tryCancel(function (err, body) {
assert.equal(body.error[0], "Cannot set status from done to cancelled");
done();
});
});
});
});
it('cancel a running job', function (done) {
var payload = {
query: "SELECT * FROM untitle_table_4; select pg_sleep(3)"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.RUNNING);
jobResult.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
jobResult.tryCancel(function (err, body) {
assert.equal(body.error[0], "Cannot set status from cancelled to cancelled");
done();
});
});
});
});
});
it('cancel a pending job', function (done) {
var self = this;
var payload1 = {
query: "SELECT * FROM untitle_table_4; select pg_sleep(3)"
};
this.batchTestClient.createJob(payload1, function(err, jobResult1) {
if (err) {
return done(err);
}
var payload2 = {
query: "SELECT * FROM untitle_table_4"
};
self.batchTestClient.createJob(payload2, function(err, jobResult2) {
if (err) {
return done(err);
}
jobResult2.getStatus(JobStatus.PENDING, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.PENDING);
jobResult2.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
jobResult1.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
done();
});
});
});
});
});
});
it('cancel a job with quotes', function (done) {
var payload = {
query: "SELECT name FROM untitle_table_4 WHERE name = 'Hawai'; select pg_sleep(3)"
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.RUNNING);
jobResult.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
done();
});
});
});
});
it('cancel a running multiquery job', function (done) {
var payload = {
query: [
"select pg_sleep(1)",
"select pg_sleep(1)",
"select pg_sleep(1)"
]
};
this.batchTestClient.createJob(payload, function(err, jobResult) {
if (err) {
return done(err);
}
jobResult.getStatus(JobStatus.RUNNING, function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.RUNNING);
jobResult.cancel(function (err, job) {
if (err) {
return done(err);
}
assert.equal(job.status, JobStatus.CANCELLED);
jobResult.tryCancel(function (err, body) {
assert.equal(body.error[0], "Cannot set status from cancelled to cancelled");
done();
});
});
});
});
});
});