From 4e40a61795468b06cfaddc5a29b66616cc1ba402 Mon Sep 17 00:00:00 2001 From: Javier Goizueta Date: Wed, 2 Mar 2016 19:25:08 +0100 Subject: [PATCH 1/2] Change form of overviews queries so they can be optimized The PostgreSQL planner wasn't applying the spatial filtering of tile bounds to the queries efficiently. --- lib/cartodb/utils/overviews_query_rewriter.js | 14 +-- test/unit/cartodb/overviews_query_rewriter.js | 118 +++++++----------- 2 files changed, 49 insertions(+), 83 deletions(-) diff --git a/lib/cartodb/utils/overviews_query_rewriter.js b/lib/cartodb/utils/overviews_query_rewriter.js index 0b6c592a..622c2018 100644 --- a/lib/cartodb/utils/overviews_query_rewriter.js +++ b/lib/cartodb/utils/overviews_query_rewriter.js @@ -74,14 +74,9 @@ function overviews_view_name(table) { } // replace a table name in a query by anoter name -function replace_table_in_query(sql, old_table_name, new_table_name) { +function replace_table_in_query(sql, old_table_name, replacement) { var old_table = TableNameParser.parse(old_table_name); - var new_table = TableNameParser.parse(new_table_name); var old_table_ident = TableNameParser.table_identifier(old_table); - var new_table_ident = TableNameParser.table_identifier(new_table); - - // text that will be substituted by the table name pattern - var replacement = new_table_ident; // regular expression prefix (beginning) to match a table name function pattern_prefix(schema, identifier) { @@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) { function overviews_query(query, overviews, zoom_level_expression) { var replaced_query = query; var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )"; + var replacement; for ( var table in overviews ) { if (overviews.hasOwnProperty(table)) { var table_overviews = overviews[table]; var table_view = overviews_view_name(table); - replaced_query = replace_table_in_query(replaced_query, table, table_view); - sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )"; + replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view; + replaced_query = replace_table_in_query(replaced_query, table, replacement); } } if ( replaced_query !== query ) { @@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) { OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { return !!sql.match( - /^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i + /^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i ); }; diff --git a/test/unit/cartodb/overviews_query_rewriter.js b/test/unit/cartodb/overviews_query_rewriter.js index 9adbe031..b58b55b0 100644 --- a/test/unit/cartodb/overviews_query_rewriter.js +++ b/test/unit/cartodb/overviews_query_rewriter.js @@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() { } }; var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ UNION ALL\ SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ @@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() { SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ UNION ALL\ SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ @@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() { SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); @@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - \"_vovw_table 1\" AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT * FROM \"_vovw_table 1\"\ + ) AS \"_vovw_table 1\"\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT * FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - \"_vovw_table 1\" AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT * FROM (\ SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT * FROM \"_vovw_table 1\"\ + ) AS \"_vovw_table 1\"\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT column1, column2, column3 FROM (\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT column1, column2, column3 FROM _vovw_table1\ - "; - assertSameSql(overviews_sql, expected_sql); - done(); - }); - - it('generates query using overviews for queries with selected columns and all columns', function(done){ - var sql = "SELECT table1.*, column1, column2, column3 FROM table1"; - var data = { - overviews: { - table1: { - 2: { table: 'table1_ov2' } - } - } - }; - var overviews_sql = overviewsQueryRewriter.query(sql, data); - var expected_sql = "\ - WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ - SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ - UNION ALL\ - SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); }); it('generates query using overviews for queries with a semicolon', function(done){ - var sql = "SELECT table1.*, column1, column2, column3 FROM table1;"; + var sql = "SELECT column1, column2, column3 FROM table1;"; var data = { overviews: { table1: { @@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() { } }; var overviews_sql = overviewsQueryRewriter.query(sql, data); + var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT column1, column2, column3 FROM (\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\ + ) AS _vovw_table1;\ "; assertSameSql(overviews_sql, expected_sql); done(); }); it('generates query using overviews for queries with extra whitespace', function(done){ - var sql = " SELECT table1.* , column1,column2, column3 FROM table1 "; + var sql = " SELECT column1,column2, column3 FROM table1 "; var data = { overviews: { table1: { @@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() { var overviews_sql = overviewsQueryRewriter.query(sql, data); var expected_sql = "\ WITH\ - _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ - _vovw_table1 AS (\ + _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\ + SELECT column1,column2, column3 FROM (\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ UNION ALL\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ - )\ - SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\ + ) AS _vovw_table1\ "; assertSameSql(overviews_sql, expected_sql); done(); @@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() { overviews_sql = overviewsQueryRewriter.query(sql, data); assert.equal(overviews_sql, sql); + sql = "SELECT table1.*, column1, column2, column3 FROM table1"; + overviews_sql = overviewsQueryRewriter.query(sql, data); + assert.equal(overviews_sql, sql); + done(); }); From e0d18e3c20cb20d16f305488cd63f2bed3450d66 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 3 Mar 2016 16:20:20 +0100 Subject: [PATCH 2/2] Back to pg 9.3 and postgresql-plpython-9.3 using sudo=true build --- .travis.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8492e564..08f37c05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,11 @@ -sudo: false +sudo: true addons: - postgresql: "9.4" - apt: - packages: - - postgresql-plpython-9.4 - - pkg-config - - libcairo2-dev - - libjpeg8-dev - - libgif-dev + postgresql: "9.3" before_install: + - sudo apt-get update + - sudo apt-get install -y postgresql-plpython-9.3 pkg-config libcairo2-dev libjpeg8-dev libgif-dev - npm install -g npm@2 - createdb template_postgis - psql -c "CREATE EXTENSION postgis" template_postgis