Release 2.71.0

Upgrade camshaft to 0.40.0

.travis.yml

@@ -9,14 +9,12 @@ addons:
 before_install:
   - npm install -g npm@2
   - lsb_release -a
-  - sudo mv /etc/apt/sources.list.d/pgdg-source.list* /tmp
   - sudo apt-get -qq purge postgis* postgresql*
   - sudo rm -Rf /var/lib/postgresql /etc/postgresql
   - sudo apt-add-repository --yes ppa:cartodb/postgresql-9.5
   - sudo apt-add-repository --yes ppa:cartodb/gis
   - sudo apt-get update
-  - sudo apt-get install -q postgresql-9.5-postgis-2.2
-  - sudo apt-get install -q postgresql-plpython-9.5
+  - sudo apt-get install -q --force-yes postgresql-9.5-postgis-2.2 postgresql-plpython-9.5
   - echo -e "local\tall\tall\ttrust\nhost\tall\tall\t127.0.0.1/32\ttrust\nhost\tall\tall\t::1/128\ttrust" |sudo tee /etc/postgresql/9.5/main/pg_hba.conf
   - sudo service postgresql restart
   - createdb template_postgis

NEWS.md

@@ -1,5 +1,103 @@
 # Changelog
 
+## 2.71.0
+
+Released 2016-08-17
+
+Announcements:
+ - Upgrades windshaft to [2.5.0](https://github.com/CartoDB/windshaft/releases/tag/2.5.0).
+
+
+## 2.70.0
+
+Released 2016-08-16
+
+Announcements:
+ - Upgrades camshaft to [0.40.0](https://github.com/CartoDB/camshaft/releases/tag/0.40.0).
+
+
+## 2.69.1
+
+Released 2016-08-12
+
+Announcements:
+ - Upgrades windshaft to [2.4.2](https://github.com/CartoDB/windshaft/releases/tag/2.4.2).
+
+
+## 2.69.0
+
+Released 2016-08-11
+
+Announcements:
+ - Upgrades camshaft to [0.39.0](https://github.com/CartoDB/camshaft/releases/tag/0.39.0).
+
+
+## 2.68.0
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades turbo-carto to [0.16.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.16.0).
+
+
+## 2.67.1
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades camshaft to [0.38.1](https://github.com/CartoDB/camshaft/releases/tag/0.38.1).
+
+
+## 2.67.0
+
+Released 2016-07-21
+
+Announcements:
+ - Upgrades camshaft to [0.38.0](https://github.com/CartoDB/camshaft/releases/tag/0.38.0).
+
+
+## 2.66.2
+
+Released 2016-07-20
+
+Announcements:
+ - Upgrades turbo-carto to [0.15.1](https://github.com/CartoDB/turbo-carto/releases/tag/0.15.1).
+
+
+## 2.66.1
+
+Released 2016-07-20
+
+Announcements:
+ - Upgrades turbo-carto to [0.15.0](https://github.com/CartoDB/turbo-carto/releases/tag/0.15.0).
+
+
+## 2.66.0
+
+Released 2016-07-18
+
+Announcements:
+ - Available new endpoint to check user analyses.
+ - Upgrades camshaft to [0.37.1](https://github.com/CartoDB/camshaft/releases/tag/0.37.1).
+
+
+## 2.65.0
+
+Released 2016-07-15
+
+Announcements:
+ - Upgrades cartodb-redis to 0.13.1.
+ - Upgrades camshaft to [0.37.0](https://github.com/CartoDB/camshaft/releases/tag/0.37.0).
+
+
+## 2.64.0
+
+Released 2016-07-12
+
+Announcements:
+ - Upgrades camshaft to [0.36.0](https://github.com/CartoDB/camshaft/releases/tag/0.36.0).
+
+
 ## 2.63.0
 
 Released 2016-07-11

config/environments/development.js.example

@@ -62,6 +62,7 @@ var config = {
         extent: "-180,-90,180,90",
         srid: 4326,
         */
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster

config/environments/production.js.example

@@ -56,6 +56,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         /*
          * Set persist_connection to false if you want

config/environments/staging.js.example

@@ -56,6 +56,7 @@ var config = {
         host: '127.0.0.1',
         port: 6432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster

config/environments/test.js.example

@@ -56,6 +56,7 @@ var config = {
         host: '127.0.0.1',
         port: 5432,
         extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        // max number of rows to return when querying data, 0 means no limit
         row_limit: 65535,
         simplify_geometries: true,
         use_overviews: true, // use overviews to retrieve raster

lib/cartodb/backends/turbo-carto-postgres-datasource.js

@@ -32,14 +32,9 @@ methodTemplates.category = dot.template([
     '  ORDER BY 2 DESC',
     '),',
     'agg_categories AS (',
-    '  SELECT \'__other\' category',
+    '  SELECT category',
     '  FROM categories',
-    '  WHERE rank >= {{=it._buckets}}',
-    '  GROUP BY 1',
-    '  UNION ALL',
-    '  SELECT CAST(category AS text)',
-    '  FROM categories',
-    '  WHERE rank < {{=it._buckets}}',
+    '  WHERE rank <= {{=it._buckets}}',
     ')',
     'SELECT array_agg(category) AS category FROM agg_categories'
 ].join('\n'));

lib/cartodb/controllers/analyses.js

@@ -0,0 +1,169 @@
+var step = require('step');
+var assert = require('assert');
+var dot = require('dot');
+dot.templateSettings.strip = false;
+var PSQL = require('cartodb-psql');
+
+var util = require('util');
+var BaseController = require('./base');
+
+var cors = require('../middleware/cors');
+var userMiddleware = require('../middleware/user');
+
+function AnalysesController(authApi, pgConnection) {
+    BaseController.call(this, authApi, pgConnection);
+}
+
+util.inherits(AnalysesController, BaseController);
+
+module.exports = AnalysesController;
+
+AnalysesController.prototype.register = function(app) {
+    app.get(app.base_url_mapconfig + '/analyses/catalog', cors(), userMiddleware, this.catalog.bind(this));
+};
+
+AnalysesController.prototype.sendResponse = function(req, res, resource) {
+    res.set('Cache-Control', 'public,max-age=10,must-revalidate');
+    this.send(req, res, resource, 200);
+};
+
+AnalysesController.prototype.catalog = function(req, res) {
+    var self = this;
+    var username = req.context.user;
+
+    step(
+        function reqParams() {
+            self.req2params(req, this);
+        },
+        function catalogQuery(err) {
+            assert.ifError(err);
+            var pg = new PSQL(dbParamsFromReqParams(req.params));
+            getMetadata(username, pg, this);
+        },
+        function prepareResponse(err, results) {
+            assert.ifError(err);
+
+            var analysisIdToTable = results.tables.reduce(function(analysisIdToTable, table) {
+                var analysisId = table.relname.split('_')[2];
+                if (analysisId && analysisId.length === 40) {
+                    analysisIdToTable[analysisId] = table;
+                }
+                return analysisIdToTable;
+            }, {});
+
+            var catalogWithTables = results.catalog.map(function(analysis) {
+                if (analysisIdToTable.hasOwnProperty(analysis.node_id)) {
+                    analysis.table = analysisIdToTable[analysis.node_id];
+                }
+                return analysis;
+            });
+
+            return catalogWithTables.sort(function(analysisA, analysisB) {
+                if (!!analysisA.table && !!analysisB.table) {
+                    return analysisB.table.size - analysisA.table.size;
+                }
+
+                if (!!analysisA.table) {
+                    return -1;
+                }
+
+                if (!!analysisB.table) {
+                    return 1;
+                }
+
+                return -1;
+            });
+        },
+        function sendResponse(err, catalogWithTables) {
+            if (err) {
+                if (err.message.match(/permission\sdenied/)) {
+                    err = new Error('Unauthorized');
+                    err.http_status = 401;
+                }
+                self.sendError(req, res, err);
+            } else {
+                self.sendResponse(req, res, { catalog: catalogWithTables });
+            }
+        }
+    );
+};
+
+var catalogQueryTpl = dot.template(
+    'SELECT analysis_def->>\'type\' as type, * FROM cartodb.cdb_analysis_catalog WHERE username = \'{{=it._username}}\''
+);
+
+var tablesQueryTpl = dot.template([
+    "WITH analysis_tables AS (",
+    "    SELECT",
+    "        n.nspname AS nspname,",
+    "        c.relname AS relname,",
+    "        pg_total_relation_size(",
+    "            format('%s.%s', pg_catalog.quote_ident(n.nspname), pg_catalog.quote_ident(c.relname))",
+    "        ) AS size,",
+    "        format('%s.%s', pg_catalog.quote_ident(nspname), pg_catalog.quote_ident(relname)) AS fully_qualified_name",
+    "    FROM pg_catalog.pg_class c, pg_catalog.pg_namespace n",
+    "    WHERE c.relnamespace = n.oid",
+    "    AND pg_catalog.quote_ident(c.relname) ~ '^analysis_[a-z0-9]{10}_[a-z0-9]{40}$'",
+    "    AND n.nspname IN ('{{=it._username}}', 'public')",
+    ")",
+    "SELECT *, pg_size_pretty(size) as size_pretty",
+    "FROM analysis_tables",
+    "ORDER BY size DESC"
+].join('\n'));
+
+
+function getMetadata(username, pg, callback) {
+    var results = {
+        catalog: [],
+        tables: []
+    };
+    step(
+        function getCatalog() {
+            pg.query(catalogQueryTpl({_username: username}), this, true); // use read-only transaction
+        },
+        function handleCatalog(err, resultSet) {
+            assert.ifError(err);
+            resultSet = resultSet || {};
+            results.catalog = resultSet.rows || [];
+            this();
+        },
+        function getTables(err) {
+            assert.ifError(err);
+            pg.query(tablesQueryTpl({_username: username}), this, true); // use read-only transaction
+        },
+        function handleTables(err, resultSet) {
+            assert.ifError(err);
+            resultSet = resultSet || {};
+            results.tables = resultSet.rows || [];
+            this();
+        },
+        function finish(err) {
+            if (err) {
+                return callback(err);
+            }
+
+            return callback(null, results);
+        }
+    );
+}
+
+
+function dbParamsFromReqParams(params) {
+    var dbParams = {};
+    if ( params.dbuser ) {
+        dbParams.user = params.dbuser;
+    }
+    if ( params.dbpassword ) {
+        dbParams.pass = params.dbpassword;
+    }
+    if ( params.dbhost ) {
+        dbParams.host = params.dbhost;
+    }
+    if ( params.dbport ) {
+        dbParams.port = params.dbport;
+    }
+    if ( params.dbname ) {
+        dbParams.dbname = params.dbname;
+    }
+    return dbParams;
+}

lib/cartodb/controllers/index.js

@@ -1,4 +1,5 @@
 module.exports = {
+    Analyses: require('./analyses'),
     Layergroup: require('./layergroup'),
     Map: require('./map'),
     NamedMaps: require('./named_maps'),

lib/cartodb/models/mapconfig/adapter/analysis-mapconfig-adapter.js

@@ -65,6 +65,7 @@ AnalysisMapConfigAdapter.prototype.getMapConfig = function(user, requestMapConfi
                 error.type = 'analysis';
                 error.analysis = {
                     id: analysisDefinition.id,
+                    node_id: err.node_id,
                     type: analysisDefinition.type
                 };
                 return done(error);

lib/cartodb/server.js

@@ -222,6 +222,8 @@ module.exports = function(serverOptions) {
 
     new controller.NamedMapsAdmin(authApi, pgConnection, templateMaps).register(app);
 
+    new controller.Analyses(authApi, pgConnection).register(app);
+
     new controller.ServerInfo(versions).register(app);
 
     /*******************************************************************************************************************

package.json

@@ -1,7 +1,7 @@
 {
   "private": true,
   "name": "windshaft-cartodb",
-  "version": "2.63.0",
+  "version": "2.71.0",
   "description": "A map tile server for CartoDB",
   "keywords": [
     "cartodb"
@@ -20,15 +20,15 @@
   ],
   "dependencies": {
     "body-parser": "~1.14.0",
-    "camshaft": "0.35.0",
+    "camshaft": "0.40.0",
     "cartodb-psql": "~0.6.1",
     "cartodb-query-tables": "~0.1.0",
-    "cartodb-redis": "~0.13.0",
+    "cartodb-redis": "0.13.1",
     "debug": "~2.2.0",
     "dot": "~1.0.2",
     "express": "~4.13.3",
     "fastly-purge": "~1.0.1",
-    "log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
+    "log4js": "cartodb/log4js-node#cdb",
     "lru-cache": "2.6.5",
     "lzma": "~2.3.2",
     "node-statsd": "~0.0.7",
@@ -37,9 +37,9 @@
     "request": "~2.62.0",
     "step": "~0.0.6",
     "step-profiler": "~0.3.0",
-    "turbo-carto": "0.14.0",
+    "turbo-carto": "0.16.0",
     "underscore": "~1.6.0",
-    "windshaft": "2.4.0"
+    "windshaft": "2.5.0"
   },
   "devDependencies": {
     "istanbul": "~0.4.3",

test/acceptance/analysis/error-cases.js

@@ -233,7 +233,7 @@ describe('analysis-layers error cases', function() {
                     "type": "buffer",
                     "params": {
                         "source": {
-                            "id": "HEAD",
+                            "id": "HEAD2",
                             "type": "source",
                             "params": {
                                 "query": "select * from populated_places_simple_reduced"
@@ -264,5 +264,114 @@ describe('analysis-layers error cases', function() {
         });
     });
 
+    it('should return missing param error of outer node indicating the node_id and context', function(done) {
+        var mapConfig = createMapConfig([{
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": '#polygons { polygon-fill: red; }',
+                "cartocss_version": "2.3.0"
+            }
+        }], {}, [{
+            "id": "HEAD",
+            "type": "buffer",
+            "params": {
+                "source": {
+                    "id": "HEAD2",
+                    "type": "buffer",
+                    "params": {
+                        "source": {
+                            "id": "HEAD3",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from populated_places_simple_reduced"
+                            }
+                        },
+                        "radius": 10
+                    }
+                }
+            }
+            // radius: 'missing'
+        }]);
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(
+                layergroupResult.errors[0],
+                'Missing required param "radius"'
+            );
+
+            assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
+            assert.equal(layergroupResult.errors_with_context[0].message, 'Missing required param "radius"');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD');
+
+            testClient.drain(done);
+        });
+    });
+
+    it('should return invalid param type error of inner node indicating the node_id and context', function(done) {
+        var mapConfig = createMapConfig([{
+            "type": "cartodb",
+            "options": {
+                "source": {
+                    "id": "HEAD"
+                },
+                "cartocss": '#polygons { polygon-fill: red; }',
+                "cartocss_version": "2.3.0"
+            }
+        }], {}, [{
+            "id": "HEAD",
+            "type": "buffer",
+            "params": {
+                "source": {
+                    "id": "HEAD2",
+                    "type": "buffer",
+                    "params": {
+                        "source": {
+                            "id": "HEAD3",
+                            "type": "source",
+                            "params": {
+                                "query": "select * from populated_places_simple_reduced"
+                            }
+                        },
+                        "radius": 'invalid_radius'
+                    }
+                },
+                "radius": 10
+            }
+        }]);
+
+        var testClient = new TestClient(mapConfig, 1234);
+
+        testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroupResult) {
+            assert.ok(!err, err);
+
+            assert.equal(layergroupResult.errors.length, 1);
+            assert.equal(
+                layergroupResult.errors[0],
+                'Invalid type for param "radius", expects "number" type, got `"invalid_radius"`'
+            );
+
+            assert.equal(layergroupResult.errors_with_context[0].type, 'analysis');
+            assert.equal(
+                layergroupResult.errors_with_context[0].message,
+                'Invalid type for param "radius", expects "number" type, got `"invalid_radius"`'
+            );
+            assert.equal(layergroupResult.errors_with_context[0].analysis.id, 'HEAD');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.type, 'buffer');
+            assert.equal(layergroupResult.errors_with_context[0].analysis.node_id, 'HEAD2');
+
+            testClient.drain(done);
+        });
+    });
+
 
 });

test/acceptance/turbo-carto/error-cases.js

@@ -50,14 +50,14 @@ describe('turbo-carto error cases', function() {
     });
 
     it('should return invalid number of ramp error', function(done) {
-        this.testClient = new TestClient(makeMapconfig('ramp([pop_max], (8,24,96), (8,24,96,128))'));
+        this.testClient = new TestClient(makeMapconfig('ramp([pop_max], 8, 96, 3, (8,24,96,128))'));
         this.testClient.getLayergroup(ERROR_RESPONSE, function(err, layergroup) {
             assert.ok(!err, err);
 
             assert.ok(layergroup.hasOwnProperty('errors'));
             assert.equal(layergroup.errors.length, 1);
-            assert.ok(layergroup.errors[0].match(/^Failed\sto\sprocess/));
-            assert.ok(layergroup.errors[0].match(/invalid\sramp\slength/i));
+            assert.ok(layergroup.errors[0].match(/^Failed\sto\sprocess/), layergroup.errors[0]);
+            assert.ok(layergroup.errors[0].match(/invalid\sramp\slength/i), layergroup.errors[0]);
 
             done();
         });

test/acceptance/turbo-carto/regressions.js

@@ -287,4 +287,97 @@ describe('turbo-carto regressions', function() {
         });
     });
 
+    var scenarios = [
+        {
+            desc: 'numeric datasource',
+            cartocss: [
+                "#points {",
+                "  marker-fill: ramp([scalerank], colorbrewer(Reds), category);",
+                "}"
+            ].join('\n'),
+            expected: [
+                '#points {',
+                '  marker-fill: #fee5d9;',
+                '  [ scalerank = 6 ] {',
+                '    marker-fill: #fcae91',
+                '  }',
+                '  [ scalerank = 8 ] {',
+                '    marker-fill: #fb6a4a',
+                '  }',
+                '  [ scalerank = 4 ] {',
+                '    marker-fill: #de2d26',
+                '  }',
+                '  [ scalerank = 10 ] {',
+                '    marker-fill: #a50f15',
+                '  }',
+                '}',
+            ].join('\n')
+        },
+        {
+            desc: 'string datasource',
+            cartocss: [
+                "#points {",
+                "  marker-fill: ramp([adm0name], colorbrewer(Reds), category);",
+                "}"
+            ].join('\n'),
+            expected: [
+                '#points {',
+                '  marker-fill: #fee5d9;',
+                '  [ adm0name = "Russia" ] {',
+                '    marker-fill: #fcae91',
+                '  }',
+                '  [ adm0name = "China" ] {',
+                '    marker-fill: #fb6a4a',
+                '  }',
+                '  [ adm0name = "Brazil" ] {',
+                '    marker-fill: #de2d26',
+                '  }',
+                '  [ adm0name = "Canada" ] {',
+                '    marker-fill: #a50f15',
+                '  }',
+                '}',
+            ].join('\n')
+        },
+        {
+            desc: 'numeric manual',
+            cartocss: [
+                "#points {",
+                "  marker-fill: ramp([scalerank], colorbrewer(Reds), (-1, 6, 8, 4, 10), category);",
+                "}"
+            ].join('\n'),
+            expected: [
+                '#points {',
+                '  marker-fill: #fee5d9;',
+                '  [ scalerank = 6 ] {',
+                '    marker-fill: #fcae91',
+                '  }',
+                '  [ scalerank = 8 ] {',
+                '    marker-fill: #fb6a4a',
+                '  }',
+                '  [ scalerank = 4 ] {',
+                '    marker-fill: #de2d26',
+                '  }',
+                '  [ scalerank = 10 ] {',
+                '    marker-fill: #a50f15',
+                '  }',
+                '}',
+            ].join('\n')
+        }
+    ];
+
+    scenarios.forEach(function(scenario) {
+        it('category ramps should use original type: ' + scenario.desc, function(done) {
+            var mapConfig = makeMapconfig('SELECT * FROM populated_places_simple_reduced', scenario.cartocss);
+            this.testClient = new TestClient(mapConfig);
+            this.testClient.getLayergroup(function(err, layergroup) {
+                assert.ok(!err, err);
+
+                assert.ok(layergroup.hasOwnProperty('layergroupid'));
+                assert.deepEqual(layergroup.metadata.layers[0].meta.cartocss, scenario.expected);
+
+                done();
+            });
+        });
+    });
+
 });