Initial commit

lib/assets/javascripts/builder/data/background-importer/background-importer-imports-collection.js

@@ -0,0 +1,102 @@
+var _ = require('underscore');
+var Backbone = require('backbone');
+var ImportsModel = require('./imports-model');
+var POLL_TIMER = 30000;
+
+/**
+ *  Imports collection
+ *
+ *  If it is fetched, it will add the import
+ *
+ */
+
+module.exports = Backbone.Collection.extend({
+
+  initialize: function (models, opts) {
+    if (!opts.userModel) throw new Error('userModel is required');
+    if (!opts.configModel) throw new Error('configModel is required');
+
+    this._userModel = opts.userModel;
+    this._configModel = opts.configModel;
+  },
+
+  model: function (attrs, opts) {
+    return new ImportsModel(attrs, {
+      userModel: opts.collection._userModel,
+      configModel: opts.collection._configModel
+    });
+  },
+
+  url: function (method) {
+    var version = this._configModel.urlVersion('import');
+    var baseUrl = this._configModel.get('base_url');
+    return baseUrl + '/api/' + version + '/imports';
+  },
+
+  parse: function (r) {
+    var self = this;
+
+    if (r.imports.length === 0) {
+      this.destroyCheck();
+    } else {
+      _.each(r.imports, function (id) {
+        // Check if that import exists...
+        var imports = self.filter(function (mdl) {
+          return mdl._importModel.get('item_queue_id') === id;
+        });
+
+        if (imports.length === 0) {
+          var importsModel = new ImportsModel({
+            id: id
+          }, {
+            userModel: self._userModel,
+            configModel: self._configModel
+          });
+
+          self.add(importsModel);
+        }
+      });
+    }
+
+    return this.models;
+  },
+
+  canImport: function () {
+    var importQuota = this._userModel.getMaxConcurrentImports();
+    var total = this.size();
+    var finished = 0;
+
+    this.each(function (m) {
+      if (m.hasFailed() || m.hasCompleted()) {
+        ++finished;
+      }
+    });
+
+    return (total - finished) < importQuota;
+  },
+
+  pollCheck: function (i) {
+    if (this.pollTimer) {
+      return;
+    }
+
+    var self = this;
+
+    this.pollTimer = setInterval(function () {
+      self.fetch();
+    }, POLL_TIMER || 2000);
+
+    this.fetch();
+  },
+
+  destroyCheck: function () {
+    clearInterval(this.pollTimer);
+    delete this.pollTimer;
+  },
+
+  failedItems: function () {
+    return this.filter(function (item) {
+      return item.hasFailed();
+    });
+  }
+});

lib/assets/javascripts/builder/data/background-importer/background-polling-model.js

@@ -0,0 +1,100 @@
+var Backbone = require('backbone');
+var ImportsCollection = require('./background-importer-imports-collection');
+
+var POLLINGS_TIMER = 3000;
+
+/**
+ *  Background polling default model
+ *
+ */
+
+module.exports = Backbone.Model.extend({
+
+  defaults: {
+    showSuccessDetailsButton: true,
+    importsPolling: false // enable imports polling
+  },
+
+  initialize: function (attrs, opts) {
+    if (!opts.userModel) throw new Error('userModel is required');
+    if (!opts.configModel) throw new Error('configModel is required');
+
+    this._userModel = opts.userModel;
+    this._configModel = opts.configModel;
+
+    this.importsCollection = opts.importsCollection || new ImportsCollection(null, {
+      userModel: this._userModel,
+      configModel: this._configModel
+    });
+
+    this._initBinds();
+    this.startPollings();
+  },
+
+  _initBinds: function () {
+    this.importsCollection.bind('change:state', function (mdl) {
+      this.trigger('change', mdl, this);
+      this._onImportsStateChange(mdl);
+    }, this);
+
+    this.importsCollection.bind('remove', function (mdl) {
+      this.trigger('importRemoved', mdl, this);
+    }, this);
+
+    this.importsCollection.bind('add', function (mdl) {
+      this.trigger('importAdded', mdl, this);
+    }, this);
+  },
+
+  getTotalFailedItems: function () {
+    return this.importsCollection.failedItems().length;
+  },
+
+  removeImportItem: function (mdl) {
+    if (!mdl) {
+      return false;
+    }
+    this.importsCollection.remove(mdl);
+  },
+
+  addImportItem: function (mdl) {
+    if (!mdl) {
+      return false;
+    }
+    this.importsCollection.add(mdl);
+  },
+
+  canAddImport: function () {
+    return this.importsCollection.canImport();
+  },
+
+  getTotalImports: function () {
+    return this.importsCollection.size();
+  },
+
+  getTotalPollings: function () {
+    return this.importsCollection.size();
+  },
+
+  stopPollings: function () {
+    if (this.get('importsPolling')) {
+      this.importsCollection.destroyCheck();
+    }
+  },
+
+  startPollings: function () {
+    var self = this;
+    // Don't start pollings inmediately
+    setTimeout(function () {
+      if (self.get('importsPolling')) {
+        self.importsCollection.pollCheck();
+      }
+    }, POLLINGS_TIMER);
+  },
+
+  _onImportsStateChange: function () {},
+
+  clean: function () {
+    this.importsCollection.unbind(null, null, this);
+  }
+});

lib/assets/javascripts/builder/data/background-importer/import-model-poller.js

@@ -0,0 +1,27 @@
+var _ = require('underscore');
+var Poller = require('./poller');
+
+var ImportModelPoller = function (model) {
+  var POLLING_INTERVAL = 2000; // Interval time between poll checkings
+  var POLLING_INTERVAL_MULTIPLIER = 2.5; // Multiply interval by this number
+  var POLLING_REQUESTS_BEFORE_INTERVAL_CHANGE = 30; // Max tries until interval change
+
+  var options = {
+    interval: function (numberOfRequests) {
+      if (numberOfRequests >= POLLING_REQUESTS_BEFORE_INTERVAL_CHANGE) {
+        return POLLING_INTERVAL * POLLING_INTERVAL_MULTIPLIER;
+      }
+      return POLLING_INTERVAL;
+    },
+    stopWhen: function (model) {
+      var state = model.get('state');
+      return (state === 'complete' || state === 'failure');
+    }
+  };
+
+  Poller.call(this, model, options);
+};
+
+ImportModelPoller.prototype = _.extend({}, Poller.prototype);
+
+module.exports = ImportModelPoller;

lib/assets/javascripts/builder/data/background-importer/import-model.js

@@ -0,0 +1,162 @@
+var _ = require('underscore');
+var Backbone = require('backbone');
+var ImportModelPoller = require('./import-model-poller');
+var SynchronizationModel = require('builder/data/synchronization-model');
+
+/**
+ *  New import model that controls
+ *  the state of an import
+ *
+ */
+module.exports = Backbone.Model.extend({
+
+  idAttribute: 'item_queue_id',
+
+  initialize: function (attrs, opts) {
+    if (!opts.configModel) throw new Error('configModel is required');
+
+    this._configModel = opts.configModel;
+
+    this._initBinds();
+    this.poller = new ImportModelPoller(this);
+  },
+
+  urlRoot: function () {
+    var version = this._configModel.urlVersion('import');
+    var baseUrl = this._configModel.get('base_url');
+
+    return baseUrl + '/api/' + version + '/imports';
+  },
+
+  _initBinds: function () {
+    this.bind('change:item_queue_id', this._checkQueueId, this);
+  },
+
+  createImport: function (data) {
+    var d = this._prepareData(data);
+    this[d.interval === 0 ? '_createRegularImport' : '_createSyncImport'](d);
+  },
+
+  _checkQueueId: function () {
+    if (this.get('item_queue_id')) {
+      this.pollCheck();
+    }
+  },
+
+  _prepareData: function (data) {
+    var d = {
+      create_vis: data.create_vis,
+      privacy: data.privacy
+    };
+
+    var type = data.type;
+
+    if (type !== 'remote') {
+      _.extend(d, {
+        type_guessing: data.type_guessing,
+        content_guessing: data.content_guessing,
+        interval: data.interval
+      });
+    }
+
+    var service = data.service_name;
+
+    if (type === 'url') {
+      _.extend(d, {
+        url: data.value
+      });
+    }
+
+    if (type === 'remote') {
+      _.extend(d, {
+        type: 'remote',
+        interval: null,
+        remote_visualization_id: data.remote_visualization_id,
+        create_vis: false,
+        value: data.value
+      });
+    }
+
+    if (type === 'sql') {
+      _.extend(d, {
+        table_name: data.table_name,
+        sql: data.value
+      });
+    }
+
+    if (type === 'duplication') {
+      _.extend(d, {
+        table_name: data.table_name,
+        table_copy: data.value
+      });
+    }
+
+    if (type === 'service') {
+      // If service is Twitter, service_item_id should be
+      // sent stringified
+      var service_item_id = (service === 'twitter_search') ? JSON.stringify(data.service_item_id) : data.service_item_id;
+
+      if (data.user_defined_limits) {
+        d.user_defined_limits = data.user_defined_limits;
+      }
+
+      _.extend(d, {
+        value: data.value,
+        service_name: data.service_name,
+        service_item_id: service_item_id
+      });
+    }
+
+    return d;
+  },
+
+  _createSyncImport: function (d) {
+    var self = this;
+    this._synchronizationModel = new SynchronizationModel(d, {
+      configModel: this._configModel
+    });
+
+    this._synchronizationModel.save(null, {
+      success: function (m) {
+        self.set('item_queue_id', m.get('data_import').item_queue_id);
+      },
+      error: function (mdl, r, opts) {
+        self._setErrorState(r);
+      }
+    });
+  },
+
+  _createRegularImport: function (d) {
+    var self = this;
+
+    this.save(d, {
+      error: function (mdl, r, opts) {
+        self._setErrorState(r);
+      }
+    });
+  },
+
+  _setErrorState: function (r) {
+    var msg;
+    try {
+      msg = r && JSON.parse(r.responseText).errors.imports;
+    } catch (err) {
+      msg = _t('data.import-model.error-starting-import');
+    }
+    this.set({
+      state: 'failure',
+      get_error_text: {
+        title: _t('data.import-model.error-title'),
+        what_about: msg
+      }
+    });
+  },
+
+  pollCheck: function () {
+    this.poller.start();
+  },
+
+  destroyCheck: function () {
+    this.poller.stop();
+  }
+});

lib/assets/javascripts/builder/data/background-importer/imports-model.js

@@ -0,0 +1,276 @@
+var _ = require('underscore');
+var Backbone = require('backbone');
+var ImportModel = require('./import-model');
+var UploadModel = require('builder/data/upload-model');
+var VisDefinitionModel = require('builder/data/vis-definition-model');
+var PermissionModel = require('builder/data/permission-model');
+
+/**
+ *  Upload/import model
+ *
+ *  It takes the control of the upload and import,
+ *  listening the change of any of these steps.
+ *
+ *  Steps:
+ *  - upload
+ *  - import
+ *
+ */
+
+module.exports = Backbone.Model.extend({
+
+  defaults: {
+    step: 'upload',
+    state: ''
+  },
+
+  initialize: function (attrs, opts) {
+    if (!opts.userModel) throw new Error('userModel is required');
+    if (!opts.configModel) throw new Error('configModel is required');
+
+    this._userModel = opts.userModel;
+    this._configModel = opts.configModel;
+
+    this._uploadModel = new UploadModel(opts.upload, {
+      userModel: this._userModel,
+      configModel: this._configModel
+    });
+
+    if (_.isEmpty(opts)) {
+      opts = {};
+    }
+
+    this._importModel = new ImportModel(opts.import, {
+      configModel: this._configModel
+    });
+    this._initBinds();
+    this._checkStatus();
+  },
+
+  _initBinds: function () {
+    this.bind('change:import', this._onImportChange, this);
+    this.bind('change:upload', this._onUploadChange, this);
+    this.bind('change:id', this._onIdChange, this);
+
+    this._importModel.bind('change', function () {
+      this.trigger('change:import', this);
+      this.trigger('change', this);
+    }, this);
+
+    this._uploadModel.bind('change', function () {
+      this.trigger('change:upload', this);
+      this.trigger('change', this);
+    }, this);
+  },
+
+  _destroyBinds: function () {
+    this._uploadModel.unbind(null, null, this);
+    this._importModel.unbind(null, null, this);
+  },
+
+  _onIdChange: function () {
+    var item_queue_id = this.get('id');
+    if (item_queue_id) this._importModel.set('item_queue_id', item_queue_id);
+    this.set('step', 'import');
+  },
+
+  _onUploadChange: function (m, i) {
+    if (this.get('step') === 'upload') {
+      var item_queue_id = this._uploadModel.get('item_queue_id');
+      var state = this._uploadModel.get('state');
+
+      if (item_queue_id) this.set('id', item_queue_id);
+      if (state) this.set('state', state);
+    }
+  },
+
+  _onImportChange: function () {
+    if (this.get('step') === 'import') {
+      var state = this._importModel.get('state');
+      if (state) this.set('state', state);
+    }
+  },
+
+  _checkStatus: function () {
+    if (!this.get('id') && !this._uploadModel.isValid()) {
+      this.trigger('change:upload');
+      return;
+    }
+
+    if (this._uploadModel.get('type') === 'file') {
+      this._uploadModel.upload();
+    } else if (this.get('id')) {
+      this.set('step', 'import');
+      this._importModel.set('item_queue_id', this.get('id'));
+    } else if (!this._importModel.get('item_queue_id') && this._uploadModel.get('type') !== '') {
+      this.set('step', 'import');
+      this._importModel.createImport(this._uploadModel.toJSON());
+    }
+  },
+
+  getImportModel: function () {
+    return this._importModel;
+  },
+
+  pause: function () {
+    this.stopUpload();
+    this.stopImport();
+  },
+
+  hasFailed: function () {
+    var state = this.get('state');
+    var step = this.get('step');
+
+    return (step === 'import' && state === 'failure') || (step === 'upload' && state === 'error');
+  },
+
+  hasCompleted: function () {
+    return this.get('step') === 'import' && this._importModel && this._importModel.get('state') === 'complete';
+  },
+
+  getWarnings: function () {
+    return this.get('step') === 'import' ? this._importModel.get('warnings') : '';
+  },
+
+  getError: function () {
+    if (this.hasFailed()) {
+      var step = this.get('step');
+      return _.extend(
+        {
+          errorCode: this[step === 'upload' ? '_uploadModel' : '_importModel'].get('error_code'),
+          itemQueueId: step === 'import' ? this._importModel.get('id') : '',
+          originalUrl: step === 'import' ? this._importModel.get('original_url') : '',
+          dataType: step === 'import' ? this._importModel.get('data_type') : '',
+          httpResponseCode: step === 'import' ? this._importModel.get('http_response_code') : '',
+          httpResponseCodeMessage: step === 'import' ? this._importModel.get('http_response_code_message') : ''
+        },
+        this[step === 'upload' ? '_uploadModel' : '_importModel'].get('get_error_text'));
+    }
+
+    return {
+      title: '',
+      what_about: '',
+      error_code: ''
+    };
+  },
+
+  importedVis: function () {
+    if (this.get('import').derived_visualization_id) {
+      return this._getMapVis();
+    } else {
+      return this._getDatasetVis();
+    }
+  },
+
+  getNumberOfTablesCreated: function () {
+    return this._importModel.get('tables_created_count');
+  },
+
+  _getServiceName: function () {
+    return this._importModel.get('service_name');
+  },
+
+  isTwitterImport: function () {
+    return this._getServiceName() === 'twitter_search';
+  },
+
+  isCartoImport: function () {
+    return this._getDisplayName() && this._getDisplayName().match(/\.carto$/i);
+  },
+
+  _getDisplayName: function () {
+    return this._importModel.get('display_name');
+  },
+
+  _getMapVis: function () {
+    var derivedVisId = this._importModel.get('derived_visualization_id');
+
+    if (!derivedVisId) {
+      return false;
+    }
+
+    return this._createVis({
+      type: 'derived',
+      id: derivedVisId
+    });
+  },
+
+  _getDatasetVis: function () {
+    var tableName = this._importModel.get('table_name');
+
+    if (!tableName) {
+      return false;
+    }
+
+    return this._createVis({
+      type: 'table',
+      table: {
+        name: tableName
+      }
+    });
+  },
+
+  _createVis: function (attrs) {
+    var vis = new VisDefinitionModel(attrs, {
+      configModel: this._configModel
+    });
+
+    vis.permission = new PermissionModel({
+      // TODO: check that this works in builder
+      owner: this._userModel.attributes
+    }, {
+      configModel: this._configModel,
+      userModel: this._userModel
+    });
+
+    return vis;
+  },
+
+  setError: function (opts) {
+    var stepModel = this[this.get('step') === 'upload' ? '_uploadModel' : '_importModel'];
+
+    this.stopUpload();
+    this.stopImport();
+
+    stepModel.set(opts);
+
+    this.set('state', 'error');
+  },
+
+  stopUpload: function () {
+    this._uploadModel.stopUpload();
+  },
+
+  stopImport: function () {
+    this._importModel.destroyCheck();
+  },
+
+  get: function (attr) {
+    if (attr === 'upload') {
+      return this._uploadModel.toJSON();
+    }
+
+    if (attr === 'import') {
+      return this._importModel.toJSON();
+    }
+
+    return Backbone.Model.prototype.get.call(this, attr);
+  },
+
+  getRedirectUrl: function (user) {
+    var vis = this.importedVis();
+    if (vis) {
+      return encodeURI(vis.viewUrl(user).edit());
+    }
+  },
+
+  toJSON: function () {
+    return {
+      step: this.get('step'),
+      id: this.get('id'),
+      state: this.get('state'),
+      upload: this._uploadModel.toJSON(),
+      import: this._importModel.toJSON()
+    };
+  }
+});

lib/assets/javascripts/builder/data/background-importer/poller.js

@@ -0,0 +1,83 @@
+var _ = require('underscore');
+
+/*
+ * Periodically fetches a model/collection. It waits for ongoing
+ * fetch requests before trying to fetch again. A stop condition
+ * can be specified.
+ *
+ * Usage example:
+ *
+ * var poller = new Poller(model, {
+ *   interval: 1000,
+ *   stopWhen: function (model) {
+ *     return model.get('state') === 'completed';
+ *   }
+ * });
+ *
+ * poller.start();
+ *
+ * // ...
+ *
+ * poller.stop();
+ *
+ */
+var Poller = function (model, opts) {
+  this.model = model;
+  this.numberOfRequests = 0;
+  this.polling = false;
+  this.interval = opts.interval;
+  if (typeof this.interval !== 'function') {
+    this.interval = function () { return opts.interval; };
+  }
+  this.stopWhen = opts.stopWhen;
+  this.error = opts.error;
+  this.autoStart = opts.autoStart;
+
+  if (this.autoStart) {
+    this.start();
+  }
+};
+
+Poller.prototype.start = function () {
+  if (this.timeout) {
+    return;
+  }
+
+  this._scheduleFetch();
+};
+
+Poller.prototype._scheduleFetch = function () {
+  this.timeout = setTimeout(this._fetch.bind(this), this.interval(this.numberOfRequests));
+};
+
+Poller.prototype._fetch = function () {
+  var self = this;
+  if (!self.polling) {
+    self.polling = true;
+    self.model.fetch({
+      success: function () {
+        self.polling = false;
+        self.numberOfRequests++;
+        if (self._continuePolling()) {
+          self._scheduleFetch();
+        }
+      },
+      error: function (e) {
+        _.isFunction(self.error) && self.error(self.model);
+      }
+    });
+  }
+};
+
+Poller.prototype._continuePolling = function () {
+  return !this.stopWhen ||
+    (_.isFunction(this.stopWhen) && !this.stopWhen(this.model));
+};
+
+Poller.prototype.stop = function () {
+  this.polling = false;
+  clearTimeout(this.timeout);
+  delete this.timeout;
+};
+
+module.exports = Poller;