Release 2.26.3

Optimize overviews queries

.travis.yml

@@ -1,16 +1,11 @@
-sudo: false
+sudo: true
 
 addons:
-  postgresql: "9.4"
-  apt:
-    packages:
-    - postgresql-plpython-9.4
-    - pkg-config
-    - libcairo2-dev
-    - libjpeg8-dev
-    - libgif-dev
+  postgresql: "9.3"
 
 before_install:
+  - sudo apt-get update
+  - sudo apt-get install -y postgresql-plpython-9.3 pkg-config libcairo2-dev libjpeg8-dev libgif-dev
   - npm install -g npm@2
   - createdb template_postgis
   - psql -c "CREATE EXTENSION postgis" template_postgis

NEWS.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## 2.26.3
+
+Released 2016-03-03
+
+Improvements:
+ - Optimize overviews queries for efficient spatial filtering in PostgreSQL
+
+
+## 2.26.2
+
+Released 2016-02-25
+
+Announcements:
+ - Upgrades windshaft to [1.13.2](https://github.com/CartoDB/Windshaft/releases/tag/1.13.2)
+
+
+## 2.26.1
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.1](https://github.com/CartoDB/Windshaft/releases/tag/1.13.1)
+
+
+## 2.26.0
+
+Released 2016-02-24
+
+Announcements:
+ - Upgrades windshaft to [1.13.0](https://github.com/CartoDB/Windshaft/releases/tag/1.13.0)
+
+
 ## 2.25.2
 
 Released 2016-02-22

lib/cartodb/api/overviews_metadata_api.js

@@ -25,8 +25,7 @@ OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, c
     var query = 'SELECT * FROM CDB_Overviews(CDB_QueryTablesText($windshaft$' + prepareSql(sql) + '$windshaft$))';
     this.pgQueryRunner.run(username, query, function handleOverviewsRows(err, rows) {
         if (err){
-            var msg = err.message ? err.message : err;
-            callback(new Error('could not get overviews metadata: ' + msg));
+            callback(err);
             return;
         }
         var metadata = {};

lib/cartodb/utils/overviews_query_rewriter.js

@@ -74,14 +74,9 @@ function overviews_view_name(table) {
 }
 
 // replace a table name in a query by anoter name
-function replace_table_in_query(sql, old_table_name, new_table_name) {
+function replace_table_in_query(sql, old_table_name, replacement) {
     var old_table = TableNameParser.parse(old_table_name);
-    var new_table = TableNameParser.parse(new_table_name);
     var old_table_ident = TableNameParser.table_identifier(old_table);
-    var new_table_ident = TableNameParser.table_identifier(new_table);
-
-    // text that will be substituted by the table name pattern
-    var replacement = new_table_ident;
 
     // regular expression prefix (beginning) to match a table name
     function pattern_prefix(schema, identifier) {
@@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
 function overviews_query(query, overviews, zoom_level_expression) {
     var replaced_query = query;
     var sql = "WITH\n  _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
+    var replacement;
     for ( var table in overviews ) {
         if (overviews.hasOwnProperty(table)) {
             var table_overviews = overviews[table];
             var table_view = overviews_view_name(table);
-            replaced_query = replace_table_in_query(replaced_query, table, table_view);
-            sql += ",\n  " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n  )";
+            replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n  ) AS " + table_view;
+            replaced_query = replace_table_in_query(replaced_query, table, replacement);
         }
     }
     if ( replaced_query !== query ) {
@@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
 
 OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
   return !!sql.match(
-    /^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
+    /^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
   );
 };
 

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "2.25.2",
+    "version": "2.26.3",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"
@@ -26,7 +26,7 @@
         "node-statsd": "~0.0.7",
         "underscore"  : "~1.6.0",
         "dot": "~1.0.2",
-        "windshaft": "1.11.1",
+        "windshaft": "1.13.2",
         "step": "~0.0.6",
         "queue-async": "~1.0.7",
         "request": "~2.62.0",

test/acceptance/multilayer_server.js

@@ -334,7 +334,7 @@ describe('tests from old api translated to multilayer', function() {
 
                 var parsed = JSON.parse(res.body);
                 assert.deepEqual(parsed, {
-                    errors: ["could not get overviews metadata: fake error message"]
+                    errors: ["fake error message"]
                 });
 
                 done();

test/unit/cartodb/overviews_query_rewriter.js

@@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
         }
     };
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM  (\
             SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
             UNION ALL\
             SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
             SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
             UNION ALL\
             SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
             SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
 
     assertSameSql(overviews_sql, expected_sql);
@@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          \"_vovw_table 1\" AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM \"_vovw_table 1\"\
+          ) AS \"_vovw_table 1\"\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          \"_vovw_table 1\" AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT * FROM (\
             SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT * FROM \"_vovw_table 1\"\
+          ) AS \"_vovw_table 1\"\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT column1, column2, column3 FROM _vovw_table1\
-    ";
-    assertSameSql(overviews_sql, expected_sql);
-    done();
-  });
-
-  it('generates query using overviews for queries with selected columns and all columns', function(done){
-    var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
-    var data = {
-        overviews: {
-            table1: {
-              2: { table: 'table1_ov2' }
-            }
-        }
-    };
-    var overviews_sql = overviewsQueryRewriter.query(sql, data);
-    var expected_sql = "\
-        WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
-            SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
-            UNION ALL\
-            SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
   });
 
   it('generates query using overviews for queries with a semicolon', function(done){
-    var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
+    var sql = "SELECT column1, column2, column3 FROM table1;";
     var data = {
         overviews: {
             table1: {
@@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
         }
     };
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
+
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1, column2, column3 FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
+          ) AS _vovw_table1;\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
   });
 
   it('generates query using overviews for queries with extra whitespace', function(done){
-    var sql = "  SELECT  table1.* , column1,column2,  column3 FROM  table1  ";
+    var sql = "  SELECT  column1,column2,  column3 FROM  table1  ";
     var data = {
         overviews: {
             table1: {
@@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
     var overviews_sql = overviewsQueryRewriter.query(sql, data);
     var expected_sql = "\
         WITH\
-          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
-          _vovw_table1 AS (\
+          _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
+          SELECT column1,column2, column3 FROM (\
             SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
             UNION ALL\
             SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
-          )\
-        SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
+          ) AS _vovw_table1\
     ";
     assertSameSql(overviews_sql, expected_sql);
     done();
@@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() {
     overviews_sql = overviewsQueryRewriter.query(sql, data);
     assert.equal(overviews_sql, sql);
 
+    sql = "SELECT table1.*, column1, column2, column3 FROM table1";
+    overviews_sql = overviewsQueryRewriter.query(sql, data);
+    assert.equal(overviews_sql, sql);
+
     done();
   });