cdb

lib/assets/javascripts/cdb/scripts/cdn_invalidation.rb

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+#
+# install gem install ruby-hmac
+# from https://gist.github.com/589132
+#
+require 'rubygems'
+require 'hmac-sha1'
+require 'net/https'
+require 'base64'
+require 'json'
+
+
+secrets = JSON.parse(File.read('secrets.json'))
+
+if ARGV.length < 1
+  puts "usage: aws_cf_invalidate.rb file1.html dir1/file2.jpg ..."
+  exit
+end
+
+s3_access = secrets['AWS_ACCESS_KEY']
+s3_secret = secrets['AWS_SECRET_KEY']
+cf_distribution = secrets['AWS_DISTRIBUTION']
+
+paths = '<Path>/' + ARGV.join('</Path><Path>/') + '</Path>'
+
+date = Time.now.utc
+date = date.strftime("%a, %d %b %Y %H:%M:%S %Z")
+digest = HMAC::SHA1.new(s3_secret)
+digest << date
+
+uri = URI.parse('https://cloudfront.amazonaws.com/2010-08-01/distribution/' + cf_distribution + '/invalidation')
+
+req = Net::HTTP::Post.new(uri.path)
+req.initialize_http_header({
+  'x-amz-date' => date,
+  'Content-Type' => 'text/xml',
+  'Authorization' => "AWS %s:%s" % [s3_access, Base64.encode64(digest.digest)]
+})
+
+req.body = "<InvalidationBatch>" + paths + "<CallerReference>ref_#{Time.now.utc.to_i}</CallerReference></InvalidationBatch>"
+
+http = Net::HTTP.new(uri.host, uri.port)
+http.use_ssl = true
+http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+res = http.request(req)
+
+puts res.code
+puts res.body

lib/assets/javascripts/cdb/scripts/check_size.js

@@ -0,0 +1,24 @@
+/*
+ * this utility reports the size of a layergroup compressed
+ * usage:
+ *  node check_size.js layergroup.json
+ */
+
+var lzma = require("lzma").LZMA();
+var fs = require('fs');
+function _array2hex (byteArr) {
+  var encoded = [];
+  for(var i = 0; i < byteArr.length; ++i) {
+    encoded.push(String.fromCharCode(byteArr[i] + 128));
+  }
+  buffer = new Buffer(encoded.join(''), 'binary');
+  return buffer.toString('base64');
+}
+
+var str = fs.readFileSync(process.argv[2]);
+lzma.compress(str, 1, function(result) {
+  console.log("size (raw/compressed/base64 compressed): ", str.length, "/", result.length, "/", _array2hex(result).length);
+});
+
+
+

lib/assets/javascripts/cdb/scripts/compress.js

@@ -0,0 +1,104 @@
+
+fs = require('fs')
+var _exec = require('child_process').exec;
+var package_ = require('../package')
+var _ = require('underscore');
+
+INCLUDE_DEPS = false;
+if(process.argv.length >= 3) {
+  INCLUDE_DEPS = process.argv[2] === 'include_deps'
+}
+
+require('../src/cartodb');
+
+cdb.files.splice(0, 0, 'cartodb.js');
+var files = cdb.files;
+
+var c = 0;
+cmds = [
+  'rm -rf  dist/_cartodb.js',
+  "echo // cartodb.js v" + package_.version + " >> dist/_cartodb.js",
+  "echo // uncompressed version: cartodb.uncompressed.js >> dist/_cartodb.js"
+];
+
+
+function concat_files(_files, ignore, callback) {
+  var files = _.clone(_files);
+  var all = '';
+  var _r = function(f) {
+    if(_.contains(ignore, f)) {
+      var next = files.shift();
+      if(next) _r(next);
+      else callback(all);
+      console.log(f, "..... IGNORED");
+    } else {
+      console.log(f);
+      fs.readFile(f, 'utf8', function (err, data) {
+        if (err) { throw new Error(err); }
+        all += data;
+        var next = files.shift();
+        if(next) _r(next);
+        else callback(all);
+      });
+    }
+  }
+  _r(files.shift());
+}
+
+var vendor_files = [];
+var cdb_files = [];
+for(var i = 0; i < files.length; ++i) {
+  var f = files[i];
+  if(f.indexOf('vendor') === -1) {
+    cdb_files.push('./src/' + f);
+  } else {
+    vendor_files.push('./vendor/' + f.split('/')[2]);
+  }
+}
+
+function create_dist_file(vendor_files, cdb_files, ignore, name) {
+  require('git-rev').long(function (sha) {
+    concat_files(vendor_files, ignore, function(vendor_js) {
+      concat_files(cdb_files, ignore, function(cdb_js) {
+        fs.readFile('scripts/wrapper.js', 'utf8', function (err, final_js) {
+          fs.writeFile(name, _.template(final_js)({
+            CDB_DEPS: vendor_js,
+            CDB_LIB: cdb_js,
+            version: package_.version,
+            sha: sha,
+            load_jquery: true
+          }));
+        });
+      });
+    });
+  });
+}
+
+cdb_ui_files = [
+  './src/ui/common/tabpane.js',
+  './src/ui/common/dialog.js',
+  './src/ui/common/notification.js',
+  './src/ui/common/table.js',
+  './src/geo/leaflet/leaflet.geometry.js',
+  './src/geo/gmaps/gmaps.geometry.js',
+];
+
+create_dist_file(vendor_files, cdb_files, [], "dist/_cartodb.js");
+create_dist_file(vendor_files, cdb_files, ['./vendor/jquery.min.js'], "dist/_cartodb_nojquery.js");
+create_dist_file(vendor_files, cdb_files, ['./vendor/leaflet.js'], "dist/_cartodb_noleaflet.js");
+create_dist_file(vendor_files, cdb_files.concat(cdb_ui_files), [], "dist/cartodb.full.uncompressed.js");
+
+
+//exec batch commands
+
+function batch() {
+  var cmd = cmds.shift();
+  if(cmd !== undefined) {
+    _exec(cmd, batch);
+  }
+}
+
+batch();
+
+
+

lib/assets/javascripts/cdb/scripts/core_footer.js

@@ -0,0 +1,3 @@
+
+})();
+//cartodb.core end

lib/assets/javascripts/cdb/scripts/core_header.js

@@ -0,0 +1,4 @@
+;(function() {
+  this.cartodb = {};
+  var Backbone = {};
+

lib/assets/javascripts/cdb/scripts/get.js

@@ -0,0 +1,17 @@
+
+var package_ = require('../package')
+
+if(process.argv.length >= 3) {
+  if(process.argv[2] === 'version') {
+    console.log(package_.version);
+  } else if(process.argv[2] === 'sha') {
+    require('git-rev').long(function (sha) {
+      console.log(sha);
+    })
+  } else if(process.argv[2] === 'header') {
+    require('git-rev').long(function (sha) {
+      console.log("// version: " + package_.version);
+      console.log("// sha: " + sha);
+    })
+  }
+}

lib/assets/javascripts/cdb/scripts/mod.torque.footer.js

@@ -0,0 +1,5 @@
+
+cartodb.moduleLoad('torque', torque);
+
+Profiler = cartodb.core.Profiler
+

lib/assets/javascripts/cdb/scripts/print_lzma.js

@@ -0,0 +1,21 @@
+/*
+ * this utility decompress layergroup lzma
+ * usage:
+ *  node print_lzma.js lzma_param
+ */
+
+var LZMA       = require('../vendor/lzma').LZMA
+
+var str = decodeURIComponent(process.argv[2]);
+
+
+// Decode (from base64)
+var lzma = (new Buffer(str, 'base64').toString('binary')).split('').map(function(c) { return c.charCodeAt(0) - 128 })
+
+// Decompress
+LZMA.decompress(
+  lzma,
+  function(result) {
+    console.log(result);
+  }
+);

lib/assets/javascripts/cdb/scripts/publish.js

@@ -0,0 +1,218 @@
+
+var secrets = require('../secrets.json');
+var fs = require('fs');
+var http = require('http');
+var package_ = require('../package');
+var _exec = require('child_process').exec;
+var AWS = require('aws-sdk');
+
+//
+// contants
+// 
+
+var S3_BUCKET_URL = 'libs.cartocdn.com.s3.amazonaws.com';
+
+var JS_FILES = [
+  'cartodb.js',
+  'cartodb.uncompressed.js',
+  'cartodb.core.js',
+  'cartodb.core.uncompressed.js',
+  'cartodb.nojquery.js',
+  'cartodb.mod.torque.js',
+  'cartodb.mod.torque.uncompressed.js',
+  'cartodb.noleaflet.js'
+]
+
+var CSS_FILES = [
+  'cartodb.css',
+  'cartodb.ie.css'
+]
+
+var CSS_IMAGE_FILES = fs.readdirSync('themes/css/images')
+var IMG_FILES = fs.readdirSync('themes/img')
+
+var content_type = {
+  'png': 'image/png',
+  'gif': 'image/gif',
+  'css': 'text/css',
+  'js': 'application/x-javascript'
+};
+
+
+AWS.config.update({ accessKeyId: secrets.S3_KEY, secretAccessKey: secrets.S3_SECRET });
+var s3 = new AWS.S3({params: {Bucket: secrets.S3_BUCKET}});
+
+
+//
+// global flags
+//
+
+var only_invalidate = process.argv.length > 2 && process.argv[2] == '--invalidate';
+var only_current_version = process.argv.length > 2 && process.argv[2] == '--current_version';
+var version = 'v' + package_.version.split('.')[0];
+
+/**
+ * check that the remote file is the same than local one to avoid errors
+ */
+function check_file_upload(local, remote, done) {
+  var fs = require('fs');
+  var file = fs.createWriteStream();
+  var request = http.get("http://im.glogster.com/media/2/5/24/10/5241033.png", function(response) {
+    response.pipe(file);
+    done && done();
+  });
+}
+
+function put_file(local_path, remote_path, file, errors, done) {
+    var local_file = file;
+
+    // extract extension
+    var ext = file.split('.');
+    ext = ext[ext.length - 1];
+
+    var headers = { 'Content-Type': content_type[ext] };
+
+    // define the function that actually puts the tile is S3
+    var _put = function() {
+      var local = local_path + '/' + local_file;
+      var remote = remote_path + '/' + file;
+
+      // dont need to update content length, knox do it
+      var size = fs.statSync(local).size;
+
+      fs.readFile(local, function (err, data) {
+        var opts = {
+          Key: remote,
+          Body: data,
+          ACL:'public-read',
+          ContentType: headers['Content-Type'],
+          ContentLength: size
+        };
+        if(headers['Content-Encoding']) {
+          opts.ContentEncoding = headers['Content-Encoding'];
+        }
+        // send file to S3
+        s3.putObject(opts, function (err, data) {
+          console.log("===>", file);
+          if(err || !data) {
+            errors.push(file + ' Failed to upload file to Amazon S3', err);
+            console.log('Failed to upload file to Amazon S3', err); 
+          }
+          console.log( local, ' => ', remote, size, "bytes");
+          done && done();
+        });
+      });
+    }
+
+    // compress depending on the extension
+    if(ext === 'js' || ext === 'css') {
+      _exec('gzip -c -9 ' + local_path + '/' + file + ' > ' + local_path + '/' + file + '.gz', function(err) {
+        if(err) {
+          console.log("there was an error compressing file");
+        }
+        local_file = file + '.gz';
+        headers['Content-Encoding'] = 'gzip';
+        _put();
+      });
+    } else {
+      _put();
+    }
+
+
+
+
+}
+
+function put_files(files, local_path, remote_path, content_type, done) {
+  var total = files.length;
+  var errors = [];
+  for(var i in  files) {
+    var file = files[i];
+    put_file(local_path, remote_path, file, errors, function() {
+      total--;
+      if(total === 0) {
+        console.log("FINISHED");
+        done && done(errors);
+      }
+    });
+  }
+}
+
+function invalidate_files(files, remote_path) {
+  var total = files.length;
+  var uploaded = 0;
+  var to_invalidate = files.map(function(f) {
+    return remote_path + '/' + f;
+  });
+
+  for(var i in to_invalidate) {
+    var cmd = 'ruby ./scripts/cdn_invalidation.rb ' + to_invalidate[i];
+    _exec(cmd, function (error, stdout, stderr){
+      console.log('stdout: ' + stdout);
+      console.log('stderr: ' + stderr);
+      if(error) console.log(error);
+    });
+    console.log(cmd);
+  }
+}
+
+
+
+/**
+ * invalidates faslty files.
+ * this function invalidates all the files, including
+ * previous or development versions but it shouldn't be a 
+ * big deal and it simplifies a lot the task
+ */
+function invalidate_fastly() {
+  var cmd = "curl -H 'Fastly-Key: " + secrets.FASTLY_API_KEY + "' -X POST 'https://api.fastly.com/service/" + secrets.FASTLY_CARTODB_SERVICE +"/purge_all'";
+  console.log(cmd);
+  _exec(cmd, function (error, stdout, stderr) {
+    var status = '';
+    try {
+      status = JSON.parse(stdout).status;
+    } catch(e) {
+    }
+    if (!error && status === 'ok') {
+      console.log(" *** faslty invalidated")
+    } else {
+      console.log(" *** faslty invalidated FAIL", error, stdout);
+    }
+  });
+}
+
+function invalidate_cdn() {
+  invalidate_fastly();
+  return;
+  // invalidate cloudfront
+  console.log(" *** flushing cdn cache")
+  if(!only_current_version) {
+    invalidate_files(JS_FILES,  'cartodb.js/' + version + '')
+    invalidate_files(CSS_FILES, 'cartodb.js/' + version + '/themes/css')
+    invalidate_files(CSS_IMAGE_FILES, 'cartodb.js/' + version + '/themes/css/images')
+    invalidate_files(IMG_FILES, 'cartodb.js/' + version + '/themes/img')
+  }
+  invalidate_files(JS_FILES , 'cartodb.js/' + version + '/' + package_.version)
+  invalidate_files(CSS_FILES, 'cartodb.js/' + version + '/' + package_.version + '/themes/css')
+  invalidate_files(CSS_IMAGE_FILES, 'cartodb.js/' + version + '/' + package_.version + '/themes/css/images')
+  invalidate_files(IMG_FILES, 'cartodb.js/' + version + '/' + package_.version + '/themes/img')
+}
+
+
+if(!only_invalidate) {
+  if(!only_current_version) {
+    put_files(JS_FILES, '' + version + '', 'cartodb.js/' + version + '')
+    put_files(CSS_FILES, '' + version + '/themes/css', 'cartodb.js/' + version + '/themes/css')
+    put_files(CSS_IMAGE_FILES, '' + version + '/themes/css/images', 'cartodb.js/' + version + '/themes/css/images')
+    put_files(IMG_FILES, '' + version + '/themes/img', 'cartodb.js/' + version + '/themes/img')
+  }
+
+  put_files(JS_FILES, '' + version + '', 'cartodb.js/' + version + '/' + package_.version)
+  put_files(CSS_FILES, '' + version + '/themes/css', 'cartodb.js/' + version + '/' + package_.version + '/themes/css')
+  put_files(CSS_IMAGE_FILES, '' + version + '/themes/css/images', 'cartodb.js/' + version + '/' + package_.version + '/themes/css/images')
+  put_files(IMG_FILES, '' + version + '/themes/img', 'cartodb.js/' + version + '/' + package_.version + '/themes/img')
+}
+
+invalidate_cdn();
+
+

lib/assets/javascripts/cdb/scripts/release.js

@@ -0,0 +1,36 @@
+
+var _exec = require('child_process').exec;
+
+
+//TODO: 
+// - check the branch is empty
+// - checkout gh-pages
+// - push
+// - create tag
+// - check is working
+
+
+// copy files to version
+var package_ = require('../package')
+
+var version = 'v' + package_.version.split('.')[0]
+
+var full_version = version + '/' + package_.version
+cmds = [
+  'mkdir ' + version,
+  'cp dist/* ' + version,
+  'cp -r themes ' + version,
+  'mkdir ' + version + '/' + package_.version,
+  'cp dist/* ' + full_version,
+  'cp -r themes ' + full_version
+];
+
+function batch() {
+  var cmd = cmds.shift();
+  if(cmd !== undefined) {
+    console.log("***", cmd);
+    _exec(cmd, batch);
+  }
+}
+
+batch();

lib/assets/javascripts/cdb/scripts/wrapper.js

@@ -0,0 +1,64 @@
+// cartodb.js version: <%= version %>
+// uncompressed version: cartodb.uncompressed.js
+// sha: <%= sha %>
+(function() {
+  var define;  // Undefine define (require.js), see https://github.com/CartoDB/cartodb.js/issues/543
+  var root = this;
+
+  if(!<%= load_jquery %>) {
+    if(root.jQuery === undefined) {
+      throw "jQuery should be loaded before include cartodb.js";
+    }
+  }
+
+  // save current libraries
+  var __prev = {
+    jQuery: root.jQuery,
+    $: root.$,
+    L: root.L,
+    Mustache: root.Mustache,
+    Backbone: root.Backbone,
+    _: root._
+  };
+
+
+  <%=CDB_DEPS%>
+
+
+
+
+  // mustache does no defined a global var, defines a var Mustache instead
+  // so add it to root
+  root.Mustache = Mustache;
+  (function() {
+    var $ = root.$;
+    var jQuery = root.jQuery;
+    var L = root.L;
+    var Mustache = root.Mustache;
+    var Backbone = root.Backbone;
+    var _ = root._;
+
+
+    <%=CDB_LIB%>
+
+    cdb.$ = $;
+    cdb.L = L;
+    cdb.Mustache = Mustache;
+    cdb.Backbone = Backbone;
+    cdb._ = _;
+
+  })();
+
+
+
+
+  ;
+  for(var i in __prev) {
+    // keep it at global context if it didn't exist
+    if(__prev[i]) {
+      window[i] = __prev[i];
+    }
+  }
+
+
+})();