cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'new_querytables_library' into analysis-layers

Browse Source
This commit is contained in:
Raul Ochoa
2016-03-04 12:08:57 +01:00
parent 10a602a4f3 16a7c4fa3d
commit c7bdabfc65
23 changed files with 325 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/acceptance/multilayer.js
+2 -2
View File
@@ -18,7 +18,7 @@ var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var QueryTables = require('cartodb-query-tables');
['/api/v1/map', '/user/localhost/api/v1/map'].forEach(function(layergroup_url) {
@@ -274,7 +274,7 @@ describe(suiteName, function() {
var parsedBody = JSON.parse(res.body);
expected_token = parsedBody.layergroupid.split(':')[0];
helper.checkCache(res);
helper.checkSurrogateKey(res, new TablesCacheEntry([
helper.checkSurrogateKey(res, new QueryTables.DatabaseTablesEntry([
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table", schema_name: "public"},
{dbname: "test_windshaft_cartodb_user_1_db", table_name: "test_table_2", schema_name: "public"},
]).key().join(' '));
test/acceptance/multilayer_server.js
+7 -4
View File
@@ -7,6 +7,7 @@ var _ = require('underscore');
var LayergroupToken = require('../../lib/cartodb/models/layergroup_token');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTables = require('cartodb-query-tables');
var CartodbWindshaft = require('../../lib/cartodb/server');
var serverOptions = require('../../lib/cartodb/server_options');
var server = new CartodbWindshaft(serverOptions);
@@ -360,9 +361,11 @@ describe('tests from old api translated to multilayer', function() {
keysToDelete['map_cfg|' + LayergroupToken.parse(JSON.parse(res.body).layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
var runQueryFn = PgQueryRunner.prototype.run;
PgQueryRunner.prototype.run = function(username, query, callback) {
return callback(new Error('failed to query database for affected tables'), []);
var affectedFn = QueryTables.getAffectedTablesFromQuery;
QueryTables.getAffectedTablesFromQuery = function(sql, username, query, callback) {
affectedFn({query: function(query, callback) {
return callback(new Error('fake error message'), []);
}}, username, query, callback);
};
// reset internal cacheChannel cache
@@ -387,7 +390,7 @@ describe('tests from old api translated to multilayer', function() {
},
function(res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'));
PgQueryRunner.prototype.run = runQueryFn;
QueryTables.getAffectedTablesFromQuery = affectedFn;
done();
}
);
test/acceptance/templates.js
+3 -3
View File
@@ -3,8 +3,8 @@ var _ = require('underscore');
var redis = require('redis');
var step = require('step');
var strftime = require('strftime');
var QueryTables = require('cartodb-query-tables');
var NamedMapsCacheEntry = require('../../lib/cartodb/cache/model/named_maps_entry');
var TablesCacheEntry = require('../../lib/cartodb/cache/model/database_tables_entry');
var redis_stats_db = 5;
// Pollute the PG environment to make sure
@@ -1405,7 +1405,7 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');
@@ -1489,7 +1489,7 @@ describe('template_api', function() {
// See https://github.com/CartoDB/Windshaft-cartodb/issues/176
helper.checkCache(res);
var expectedSurrogateKey = [
new TablesCacheEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
new QueryTables.DatabaseTablesEntry([{dbname: 'test_windshaft_cartodb_user_1_db', schema_name: 'public',
table_name: 'test_table_private_1'}]).key(),
new NamedMapsCacheEntry('localhost', template_acceptance_open.name).key()
].join(' ');
test/integration/overviews-metadata-api.js
+1 -3
View File
@@ -7,20 +7,18 @@ var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
describe('OverviewsMetadataApi', function() {
var queryTablesApi, overviewsMetadataApi;
var overviewsMetadataApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
});
test/integration/query-tables-api.js
-56
View File
@@ -1,56 +0,0 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var QueryTablesApi = require('../../lib/cartodb/api/query_tables_api');
describe('QueryTablesApi', function() {
var queryTablesApi;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
queryTablesApi = new QueryTablesApi(pgQueryRunner);
});
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
it('should return an object with affected tables array and last updated time', function(done) {
var query = 'select * from test_table';
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
affectedTables: [{dbname: "test_windshaft_cartodb_user_1_db", schema_name: "public",
"table_name": 'test_table', updated_at: new Date(1234567890123)}],
lastUpdatedTime: 1234567890123
});
done();
});
});
it('should work with private tables', function(done) {
var query = 'select * from test_table_private_1';
queryTablesApi.getAffectedTablesAndLastUpdatedTime('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {
affectedTables: [{dbname: "test_windshaft_cartodb_user_1_db", schema_name: "public",
"table_name": 'test_table_private_1', updated_at: new Date(1234567890123)}],
lastUpdatedTime: 1234567890123
});
done();
});
});
});
test/integration/query-tables.js
+71
View File
@@ -0,0 +1,71 @@
require('../support/test_helper');
var assert = require('assert');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var QueryTables = require('cartodb-query-tables');
describe('QueryTables', function() {
var connection;
before(function(done) {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
pgConnection.getConnection('localhost', function(err, pgConnection) {
if (err) {
return done(err);
}
connection = pgConnection;
return done();
});
});
// Check test/support/sql/windshaft.test.sql to understand where the values come from.
it('should return an object with affected tables array and last updated time', function(done) {
var query = 'select * from test_table';
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
assert.ok(!err, err);
assert.equal(result.getLastUpdatedAt(), 1234567890123);
assert.equal(result.tables.length, 1);
assert.deepEqual(result.tables[0], {
dbname: 'test_windshaft_cartodb_user_1_db',
schema_name: 'public',
table_name: 'test_table',
updated_at: new Date(1234567890123)
});
done();
});
});
it('should work with private tables', function(done) {
var query = 'select * from test_table_private_1';
QueryTables.getAffectedTablesFromQuery(connection, query, function(err, result) {
assert.ok(!err, err);
assert.equal(result.getLastUpdatedAt(), 1234567890123);
assert.equal(result.tables.length, 1);
assert.deepEqual(result.tables[0], {
dbname: 'test_windshaft_cartodb_user_1_db',
schema_name: 'public',
table_name: 'test_table_private_1',
updated_at: new Date(1234567890123)
});
done();
});
});
});
test/support/prepare_db.sh
+4 -2
View File
@@ -78,8 +78,10 @@ if test x"$PREPARE_PGSQL" = xyes; then
sed "s/:TESTPASS/${TESTPASS}/" |
psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
psql -c "CREATE LANGUAGE plpythonu;" ${TEST_DB}
for i in CDB_QueryStatements CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews
cat sql/_CDB_QueryStatements.sql | psql -v ON_ERROR_STOP=1 ${TEST_DB} || exit 1
SQL_SCRIPTS='CDB_QueryTables CDB_CartodbfyTable CDB_TableMetadata CDB_ForeignTable CDB_UserTables CDB_ColumnNames CDB_ZoomFromScale CDB_Overviews'
for i in ${SQL_SCRIPTS}
do
curl -L -s https://github.com/CartoDB/cartodb-postgresql/raw/master/scripts-available/$i.sql -o sql/$i.sql
cat sql/$i.sql | sed -e 's/cartodb\./public./g' -e "s/''cartodb''/''public''/g" \
test/support/sql/_CDB_QueryStatements.sql
+16
View File
@@ -0,0 +1,16 @@
-- DUMMY IMPLEMENTATION
-- Ref: https://github.com/CartoDB/cartodb-postgresql/blob/master/scripts-available/CDB_QueryStatements.sql
-- Originally implemented in plpython for performance reasons
-- Return an array of statements found in the given query text
--
-- Regexp curtesy of Hubert Lubaczewski (depesz)
--
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
RETURNS SETOF TEXT AS $$
with matches as (
select regexp_matches($1, $regexp$((?:[^'"$;]+|"[^"]*"|'[^']*'|(\$[^$]*\$).*?\2)+)$regexp$, 'g') as m
)
select btrim(m[1]) from matches
$$
LANGUAGE SQL IMMUTABLE STRICT;
test/unit/cartodb/overviews_query_rewriter.js
+44 -74
View File
@@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
@@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
) AS \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
) AS \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT column1, column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with selected columns and all columns', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with a semicolon', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
var sql = "SELECT column1, column2, column3 FROM table1;";
var data = {
overviews: {
table1: {
@@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
) AS _vovw_table1;\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with extra whitespace', function(done){
var sql = " SELECT table1.* , column1,column2, column3 FROM table1 ";
var sql = " SELECT column1,column2, column3 FROM table1 ";
var data = {
overviews: {
table1: {
@@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1,column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() {
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT table1.*, column1, column2, column3 FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
done();
});