Release 1.6.2

Closes #110

NEWS.md

@@ -1,3 +1,16 @@
+1.6.2 -- 2014-01-23
+-------------------
+
+Bug fixes:
+
+* Fix support for long (>64k chars) queries in layergroup creation (#111)
+
+Enhancements:
+
+* Enhance tools/show_style to accept an environment parameter and
+  print XML style now it is not in redis anymore (#110)
+* Support CORS in template instanciation endpoint (#113)
+
 1.6.1 -- 2014-01-15
 -------------------
 
@@ -14,6 +27,7 @@ New features:
  * Add 'user_from_host' directive to generalize username extraction (#100)
  * Implement signed template maps (#98)
 
+
 Other changes:
 
  * Update cartodb-redis dependency to "~0.3.0"

lib/cartodb/cartodb_windshaft.js

@@ -387,6 +387,11 @@ var CartodbWindshaft = function(serverOptions) {
       );
     };
 
+    ws.options(template_baseurl + '/:template_id', function(req, res) {
+      ws.doCORS(res, "Content-Type");
+      return next();
+    });
+
     // Instantiate a template
     ws.post(template_baseurl + '/:template_id', function(req, res) {
       ws.doCORS(res);

lib/cartodb/server_options.js

@@ -78,7 +78,15 @@ module.exports = function(){
       if (_.isString(api_key) && api_key != '') { qs.api_key = api_key; }
 
       // call sql api
-      request.get({url:sqlapi, qs:qs, json:true}, function(err, res, body){
+      //
+      // NOTE: using POST to avoid size limits:
+      // Seehttp://github.com/CartoDB/Windshaft-cartodb/issues/111
+      //
+      // TODO: use "host" header to allow IP based specification
+      //       of sqlapi address (and avoid a DNS lookup)
+      //
+      request.post({url:sqlapi, body:qs, json:true},
+        function(err, res, body){
           if (err){
             console.log('ERROR connecting to SQL API on ' + sqlapi + ': ' + err);
             callback(err);

npm-shrinkwrap.json

@@ -1,6 +1,6 @@
 {
   "name": "windshaft-cartodb",
-  "version": "1.6.1",
+  "version": "1.6.2",
   "dependencies": {
     "node-varnish": {
       "version": "0.1.1"
@@ -378,12 +378,12 @@
     "lzma": {
       "version": "1.2.3"
     },
-    "semver": {
-      "version": "1.1.4"
-    },
     "strftime": {
       "version": "0.6.2"
     },
+    "semver": {
+      "version": "1.1.4"
+    },
     "redis": {
       "version": "0.8.6"
     },

package.json

@@ -1,7 +1,7 @@
 {
     "private": true,
     "name": "windshaft-cartodb",
-    "version": "1.6.1",
+    "version": "1.6.2",
     "description": "A map tile server for CartoDB",
     "keywords": [
       "cartodb"

test/acceptance/multilayer.js

@@ -900,6 +900,71 @@ suite('multilayer', function() {
       );
     });
 
+    // SQL strings can be of arbitrary length, when using POST
+    // See https://github.com/CartoDB/Windshaft-cartodb/issues/111
+    test("sql string can be very long", function(done){
+      var long_val = 'pretty';
+      for (var i=0; i<1024; ++i) long_val += ' long'
+      long_val += ' string';
+      var sql = "SELECT ";
+      for (var i=0; i<16; ++i) 
+        sql += "'" + long_val + "'::text as pretty_long_field_name_" + i + ", ";
+      sql += "cartodb_id, the_geom_webmercator FROM gadm4 g";
+      var layergroup =  {
+        version: '1.0.0',
+        layers: [
+           { options: {
+               sql: sql,
+               cartocss: '#layer { marker-fill:red; }', 
+               cartocss_version: '2.0.1'
+             } }
+        ]
+      };
+      var errors = [];
+      var expected_token; 
+      Step(
+        function do_post()
+        {
+          var data = JSON.stringify(layergroup);
+          assert.ok(data.length > 1024*64);
+          var next = this;
+          assert.response(server, {
+              url: '/tiles/layergroup?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost', 'Content-Type': 'application/json' },
+              data: data
+          }, {}, function(res) { next(null, res); });
+        },
+        function check_result(err, res) {
+          if ( err ) throw err;
+          assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
+          var parsedBody = JSON.parse(res.body);
+          var token_components = parsedBody.layergroupid.split(':');
+          expected_token = token_components[0];
+          return null;
+        },
+        function cleanup(err) {
+          if ( err ) errors.push(err.message);
+          if ( ! expected_token ) return null;
+          var next = this;
+          redis_client.keys("map_style|test_cartodb_user_1_db|~" + expected_token, function(err, matches) {
+              if ( err ) errors.push(err.message);
+              assert.equal(matches.length, 1, "Missing expected token " + expected_token + " from redis: " + matches);
+              redis_client.del(matches, function(err) {
+                if ( err ) errors.push(err.message);
+                next();
+              });
+          });
+        },
+        function finish(err) {
+          if ( err ) errors.push('' + err);
+          if ( errors.length ) done(new Error(errors.join(',')));
+          else done(null);
+        }
+      );
+    });
+
+
     suiteTeardown(function(done) {
 
         // This test will add map_style records, like

test/acceptance/templates.js

@@ -131,6 +131,32 @@ suite('template_api', function() {
       );
     });
 
+    test("instance endpoint should return CORS headers", function(done){
+      Step(function postTemplate1(err, res) {
+          var next = this;
+          var post_request = {
+              url: '/tiles/template?api_key=1234',
+              method: 'POST',
+              headers: {host: 'localhost.localhost', 'Content-Type': 'application/json' },
+              data: JSON.stringify(template_acceptance1)
+          };
+          assert.response(server, post_request, {}, function(res) { next(null, res); });
+        },
+        function testCORS() {
+          assert.response(server, {
+              url: '/tiles/template/acceptance1',
+              method: 'OPTIONS'
+          },{
+              status: 200,
+              headers: {
+                'Access-Control-Allow-Headers': 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Content-Type',
+                'Access-Control-Allow-Origin': '*'
+              }
+          }, function() { done(); });
+      });
+    });
+
+
     test("can list templates", function(done) {
 
       var errors = [];

test/support/SQLAPIEmu.js

@@ -5,31 +5,60 @@ var o = function(port, cb) {
 
   this.queries = [];
   var that = this;
+
   this.sqlapi_server = http.createServer(function(req,res) {
-        var query = url.parse(req.url, true).query;
-        that.queries.push(query);
-        if ( query.q.match('SQLAPIERROR') ) {
-          res.statusCode = 400;
-          res.write(JSON.stringify({'error':'Some error occurred'}));
-        } else if ( query.q.match('EPOCH.* as max') ) {
-          // This is the structure of the known query sent by tiler
-          var row = {
-            'max': 1234567890.123
-          };
-          res.write(JSON.stringify({rows: [ row ]}));
-        } else {
-          var qs = JSON.stringify(query);
-          var row = {
-            // This is the structure of the known query sent by tiler
-            'cdb_querytables': '{' + qs + '}',
-            'max': qs
-          };
-          res.write(JSON.stringify({rows: [ row ]}));
-        }
-        res.end();
+      //console.log("server got request with method " + req.method);
+      var query;
+      if ( req.method == 'GET' ) {
+        query = url.parse(req.url, true).query;
+        that.handleQuery(query, res);
+      }
+      else if ( req.method == 'POST') {
+        var data = '';
+        req.on('data', function(chunk) {
+          //console.log("GOT Chunk  " + chunk);
+          data += chunk;
+        });
+        req.on('end', function() {
+          //console.log("Data is: "); console.dir(data);
+          query = JSON.parse(data);
+          //console.log("Parsed is: "); console.dir(query);
+          //console.log("handleQuery is " + that.handleQuery);
+          that.handleQuery(query, res);
+        });
+      }
+      else {
+        that.handleQuery('SQLAPIEmu does not support method' + req.method, res);
+      }
    }).listen(port, cb);
 };
 
+o.prototype.handleQuery = function(query, res) {
+    this.queries.push(query);
+    if ( query.q.match('SQLAPIERROR') ) {
+      res.statusCode = 400;
+      res.write(JSON.stringify({'error':'Some error occurred'}));
+    } else if ( query.q.match('EPOCH.* as max') ) {
+      // This is the structure of the known query sent by tiler
+      var row = {
+        'max': 1234567890.123
+      };
+      res.write(JSON.stringify({rows: [ row ]}));
+    } else {
+      var qs = JSON.stringify(query);
+      var row = {
+        // This is the structure of the known query sent by tiler
+        'cdb_querytables': '{' + qs + '}',
+        'max': qs
+      };
+      var out_obj = {rows: [ row ]};
+      var out = JSON.stringify(out_obj);
+      res.write(out);
+    }
+    res.end();
+  };
+
+
 o.prototype.close = function(cb) {
   this.sqlapi_server.close(cb);
 };

tools/show_style

@@ -2,28 +2,84 @@
 
 # TODO: port to node, if you really need it
 
-REDIS_PORT=6379 # default port
-
+ENV='development';
+BASEDIR=`cd $(dirname $0)/../; pwd`
 
 if test -z "$1"; then
-  echo "Usage: $0 <username> [<tablename>|~<token>]" >&2
+  echo "Usage: $0 [--env <environment>] <username> [<tablename>|~<token>]" >&2
+  echo " environment defaults to 'development'"
+  exit 1
+fi
+
+username=""
+token=""
+
+while test -n "$1"; do
+  if test "$1" = "--env"; then
+    shift; ENV="$1"; shift
+  elif test -z "$username"; then
+    username="$1"; shift
+  elif test -z "$token"; then
+    token="$1"; shift
+  else
+    echo "Unused option $1" >&2
+    shift
+  fi
+done
+
+echo "Using environment '${ENV}'"
+
+CONFIG="${BASEDIR}/config/environments/${ENV}.js"
+REDIS_PORT=`node -e "console.log(require('${CONFIG}').redis.port)"`
+if test $? -ne 0; then
   exit 1
 fi
 
-username="$1"
-token="$2"
 
 dbname=`redis-cli -p ${REDIS_PORT} -n 5 hget "rails:users:${username}" "database_name"`
 if test $? -ne 0; then
   exit 1
 fi
 if test -z "${dbname}"; then
-  echo "Username ${username} unknown by redis (try CARTODB/script/restore_redis?)" >&2
+  echo "Username ${username} unknown by redis on port ${REDIS_PORT} (try CARTODB/script/restore_redis?)" >&2
   exit 1
 fi
 echo "Database name for user ${username}: ${dbname}" # only if verbose?
 if test -n "$token"; then
-  redis-cli get "map_style|${dbname}|${token}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
+  rec=`redis-cli get "map_style|${dbname}|${token}"`
+  if test -z "${rec}"; then
+    echo "${token}: no such map style known by redis on port ${REDIS_PORT}" >&2
+    exit 1
+  fi
+  #echo "${rec}" 
+  escrec=`echo "${rec}" | sed -e 's/\\\\/\\\\\\\\/g'`
+  #echo "${escrec}" 
+  node <<EOF
+var x=JSON.parse('${escrec}');
+console.log('style: ' + x.style);
+console.log('version: ' + x.version);
+
+global.environment = require('${CONFIG}');
+var serverOptions = require('${BASEDIR}/lib/cartodb/server_options'); // _after_ setting global.environment
+var grainstore = require('${BASEDIR}/node_modules/windshaft/node_modules/grainstore/lib/grainstore');
+var mml_store = new grainstore.MMLStore(serverOptions.redis, serverOptions.grainstore);
+var builderconfig = {dbname:'${dbname}'};
+if ( '${token}'.match(/^~/) ) {
+  builderconfig.token = '${token}'.substring(1);
+} else {
+  builderconfig.table = '${token}';
+}
+var mml_builder = mml_store.mml_builder(builderconfig,
+      function(err, payload) {
+        if ( err ) throw err;
+        mml_builder.toXML(function(err, xml) {
+        if ( err ) throw err;
+          console.log('- XML - ');
+          console.log(xml);
+        });
+      });
+EOF
+  #echo "${rec}" | sed -e 's/\\n/\n/g' -e 's/\\//g'
 else
   redis-cli keys "map_style|${dbname}|*"
 fi