Initial commit

lib/assets/javascripts/dashboard/data/table/carto-table-data.js

@@ -0,0 +1,813 @@
+const $ = require('jquery');
+const Backbone = require('backbone');
+const _ = require('underscore');
+const TableDataCollection = require('dashboard/data/table/table-data-collection');
+const RowModel = require('dashboard/data/table/row-model');
+const checkAndBuildOpts = require('builder/helpers/required-opts');
+const WKT = require('dashboard/common/wkt');
+const safeTableNameQuoting = require('dashboard/helpers/safe-table-name-quoting');
+const cartoMetadataStatic = require('dashboard/views/public-dataset/carto-table-metadata-static');
+
+const REQUIRED_OPTS = [
+  'configModel'
+];
+
+module.exports = TableDataCollection.extend({
+  _ADDED_ROW_TEXT: 'Row added correctly',
+  _ADDING_ROW_TEXT: 'Adding a new row',
+  _GEOMETRY_UPDATED: 'Table geometry updated',
+
+  model: function (attrs, opts) {
+    var configModel = opts.collection._configModel;
+    return new RowModel(attrs, {
+      configModel,
+      // TODO: Check this
+      collection: opts.collection
+    });
+  },
+
+  initialize: function (models, options) {
+    checkAndBuildOpts(options, REQUIRED_OPTS, this);
+    this.table = options ? options.table : null;
+    this.model.prototype.idAttribute = 'cartodb_id';
+    this.initOptions();
+    this.filter = null;
+    this._fetching = false;
+    this.pages = [];
+    this.lastPage = false;
+    this.bind('newPage', this.newPage, this);
+    this.bind('reset', function () {
+      var pages = Math.floor(this.size() / this.options.get('rows_per_page'));
+      this.pages = [];
+
+      for (var i = 0; i < pages; ++i) {
+        this.pages.push(i);
+      }
+    }, this);
+
+    if (this.table) {
+      this.bind('add change:the_geom', function (row) {
+        var gt = this.table.get('geometry_types');
+        if (gt && gt.length > 0) return;
+        if (row.get('the_geom')) {
+          // we set it to silent because a change in geometry_types
+          // raises rendering and column feching
+          this.table.addGeomColumnType(row.getGeomType());
+        }
+      }, this);
+    }
+    TableDataCollection.prototype.initialize.call(this);
+  },
+
+  initOptions: function () {
+    this.options = new Backbone.Model({
+      rows_per_page: 40,
+      page: 0,
+      sort_order: 'asc',
+      order_by: 'cartodb_id',
+      filter_column: '',
+      filter_value: ''
+    });
+    this.options.bind('change', () => {
+      if (this._fetching) {
+        return;
+      }
+      this._fetching = true;
+      var opt = {};
+      var previous = this.options.previous('page');
+
+      if (this.options.hasChanged('page')) {
+        opt.add = true;
+        opt.changingPage = true;
+        // if user is going backwards insert new rows at the top
+        if (previous > this.options.get('page')) {
+          opt.at = 0;
+        }
+      } else {
+        if (this.options.hasChanged('mode')) {
+          this.options.set({
+            'page': 0
+          }, { silent: true });
+        }
+      }
+
+      opt.success = (_coll, resp) => {
+        this.trigger('loaded');
+        if (resp.rows && resp.rows.length !== 0) {
+          if (opt.changingPage) {
+            this.trigger('newPage', this.options.get('page'), opt.at === 0 ? 'up' : 'down');
+          }
+        } else {
+          // no data so do not change the page
+          this.options.set({page: previous});//, { silent: true });
+        }
+        this.trigger('endLoadingRows', this.options.get('page'), opt.at === 0 ? 'up' : 'down');
+        this._fetching = false;
+      };
+
+      opt.error = () => {
+        console.error('there was some problem fetching rows');
+        this.trigger('endLoadingRows');
+        this._fetching = false;
+      };
+
+      this.trigger('loadingRows', opt.at === 0 ? 'up' : 'down');
+
+      this.fetch(opt);
+    }, this);
+  },
+
+  parse: function (d) {
+    // when the query modifies the data modified flag is true
+    // TODO: change this when SQL API was able to say if a
+    // query modify some data
+    // HACK, it will fail if using returning sql statement
+    this.modify_rows = d.rows.length === 0 && _.size(d.fields) === 0;
+    this.affected_rows = d.affected_rows;
+    this.lastPage = false;
+    if (d.rows.length < this.options.get('rows_per_page')) {
+      this.lastPage = true;
+    }
+    return d.rows;
+  },
+
+  // given fields array as they come from SQL create a map name -> type
+  _schemaFromQueryFields: function (fields) {
+    var sc = {};
+    for (var k in fields) {
+      sc[k] = fields[k].type;
+    }
+    return sc;
+  },
+
+  _createUrlOptions: function (filter) {
+    var attr;
+    if (filter) {
+      var a = {};
+      for (var k in this.options.attributes) {
+        if (filter(k)) {
+          a[k] = this.options.attributes[k];
+        }
+      }
+      attr = _(a);
+    } else {
+      attr = _(this.options.attributes);
+    }
+    var params = attr.map(function (v, k) {
+      return k + '=' + encodeURIComponent(v);
+    }).join('&');
+    params += '&api_key=' + this._configModel.get('api_key');
+    return params;
+  },
+
+  _geometryColumnSQL: function (c) {
+    return [
+      'CASE',
+      'WHEN GeometryType(' + c + ") = 'POINT' THEN",
+      'ST_AsGeoJSON(' + c + ',8)',
+      'WHEN (' + c + ' IS NULL) THEN',
+      'NULL',
+      'ELSE',
+      'GeometryType(' + c + ')',
+      'END ' + c
+    ].join(' ');
+  },
+
+  // return wrapped SQL removing the_geom and the_geom_webmercator
+  // to avoid fetching those columns.
+  // So for a sql like
+  // select * from table the returned value is
+  // select column1, column2, column3... from table
+  wrappedSQL: function (schema, exclude, fetchGeometry) {
+    exclude = exclude || ['the_geom_webmercator'];
+    schema = _.clone(schema);
+
+    var select_columns = _.chain(schema).omit(exclude).map((v, k) => {
+      if (v === 'geometry') {
+        if (fetchGeometry) {
+          return 'st_astext("' + k + '") ' + 'as ' + k;
+        }
+        return this._geometryColumnSQL(k);
+      }
+      return '"' + k + '"';
+    }).value();
+
+    select_columns = select_columns.join(',');
+
+    var mode = this.options.get('sort_order') === 'desc' ? 'desc' : 'asc';
+
+    var q = 'select ' + select_columns + ' from (' + this.getSQL() + ') __wrapped';
+    var order_by = this.options.get('order_by');
+    if (order_by && order_by.length > 0) {
+      q += ' order by ' + order_by + ' ' + mode;
+    }
+    return q;
+  },
+
+  url: function () {
+    return this.sqlApiUrl();
+  },
+
+  /**
+  * we need to override sync to avoid the sql request to be sent by GET.
+  * For security reasons, we need them to be send as a PUT request.
+  * @method sync
+  * @param method {'save' || 'read' || 'delete' || 'create'}
+  * @param model {Object}
+  * @param options {Object}
+  */
+  sync: function (method, model, options) {
+    if (!options) { options = {}; }
+    options.data = this._createUrlOptions(function (d) {
+      return d !== 'sql';
+    });
+
+    if (cartoMetadataStatic.alterTableData(this.options.get('sql') || '')) {
+      options.data += '&q=' + encodeURIComponent(this.options.get('sql'));
+      options.type = 'POST';
+    } else {
+      // when a geometry can be lazy fetched, don't fetch it
+      var fetchGeometry = 'cartodb_id' in this.query_schema;
+      options.data += '&q=' + encodeURIComponent(this.wrappedSQL(this.query_schema, [], !fetchGeometry));
+
+      if (options.data.length > 2048) {
+        options.type = 'POST';
+      }
+    }
+
+    return Backbone.sync.call(this, method, this, options);
+  },
+
+  sqlApiUrl: function () {
+    return this._configModel.getSqlApiUrl();
+  },
+
+  setOptions: function (opt) {
+    this.options.set(opt);
+  },
+
+  // Refresh all table data
+  refresh: function () {
+    this.fetch();
+  },
+
+  isFetchingPage: function () {
+    return this._fetching;
+  },
+
+  loadPageAtTop: function () {
+    if (!this._fetching) {
+      var first = this.pages[0];
+
+      if (first > 0) {
+        this.options.set('page', first - 1);
+      }
+    }
+  },
+
+  loadPageAtBottom: function () {
+    if (!this._fetching) {
+      var last = this.pages[this.pages.length - 1];
+
+      if (!this.lastPage) {
+        this.options.set('page', last + 1);
+      }
+    }
+  },
+
+  /**
+   * called when a new page is loaded
+   * removes the models to max
+   */
+  newPage: function (currentPage, direction) {
+    if (this.pages.indexOf(currentPage) < 0) {
+      this.pages.push(currentPage);
+    }
+    this.pages.sort(function (a, b) {
+      return Number(a) > Number(b);
+    });
+    // remove blocks if there are more rows than allowed
+    var rowspp = this.options.get('rows_per_page');
+    var max_items = rowspp * 4;
+    if (this.size() > max_items) {
+      if (direction == 'up') { // eslint-disable-line eqeqeq
+        // remove page from the bottom (the user is going up)
+        this.pages.pop();
+        this.remove(this.models.slice(max_items, this.size()));
+      } else {
+        // remove page from the top (the user is going down)
+        this.pages.shift();
+        this.remove(this.models.slice(0, rowspp));
+      }
+    }
+  },
+
+  addRow: function (opts) {
+    this.table.notice(this._ADDING_ROW_TEXT, 'load', 0);
+    opts = opts || {};
+    _.extend(opts, {
+      wait: true,
+      success: () => {
+        this.table.notice(this._ADDED_ROW_TEXT);
+      },
+      error: (e, resp) => {
+        // TODO: notice user
+        this.table.error(this._ADDING_ROW_TEXT, resp);
+      }
+    });
+    return this.create(null, opts);
+  },
+
+  /**
+   * creates a new row model in local, it is NOT serialized to the server
+   */
+  newRow: function (attrs) {
+    var r = new RowModel(attrs, {
+      configModel: this._configModel
+    });
+    r.table = this.table;
+    r.bind('saved', function _saved () {
+      if (r.table.data().length === 0) {
+        r.table.data().fetch();
+        r.unbind('saved', _saved, r.table);
+      }
+    }, r.table);
+    return r;
+  },
+
+  /**
+   * return a model row
+   */
+  getRow: function (id, options) {
+    options = options || {};
+    var r = this.get(id);
+    if (!r) {
+      r = new RowModel({cartodb_id: id}, { configModel: this._configModel });
+    }
+    if (!options.no_add) {
+      this.table._data.add(r);
+    }
+    r.table = this.table;
+    return r;
+  },
+
+  getRowAt: function (index) {
+    var r = this.at(index);
+    r.table = this.table;
+    return r;
+  },
+
+  deleteRow: function (row_id) {
+  },
+
+  isReadOnly: function () {
+    return false;
+  },
+
+  quartiles: function (nslots, column, callback, error) {
+    var tmpl = _.template('select quartile, max(<%= column %>) as maxamount from (select <%= column %>, ntile(<%= slots %>) over (order by <%= column %>) as quartile from (<%= sql %>) _table_sql where <%= column %> is not null) x group by quartile order by quartile');
+    this._sqlQuery(tmpl({
+      slots: nslots,
+      sql: this.getSQL(),
+      column: column
+    }),
+    function (data) {
+      callback(_(data.rows).pluck('maxamount'));
+    },
+    error);
+  },
+
+  equalInterval: function (nslots, column, callback, error) {
+    var tmpl = _.template(`
+      with params as (select min(a), max(a) from ( select <%= column %> as a from (<%= sql %>) _table_sql where <%= column %> is not null ) as foo )
+      select (max-min)/<%= slots %> as s, min, max from params`
+    );
+    this._sqlQuery(tmpl({
+      slots: nslots,
+      sql: this.getSQL(),
+      column: column
+    }),
+    function (data) {
+      var min = data.rows[0].min;
+      var max = data.rows[0].max;
+      var range = data.rows[0].s;
+      var values = [];
+
+      for (var i = 1, l = nslots; i < l; i++) {
+        values.push((range * i) + min);
+      }
+
+      // Add last value
+      values.push(max);
+      // Callback
+      callback(values);
+    },
+    error);
+  },
+
+  _quantificationMethod: function (functionName, nslots, column, distinct, callback, error) {
+    var tmpl = _.template('select unnest(<%= functionName %>(array_agg(<%= simplify_fn %>((<%= column %>::numeric))), <%= slots %>)) as buckets from (<%= sql %>) _table_sql where <%= column %> is not null');
+    this._sqlQuery(tmpl({
+      slots: nslots,
+      sql: this.getSQL(),
+      column: column,
+      functionName: functionName,
+      simplify_fn: 'distinct'
+    }),
+    function (data) {
+      callback(_(data.rows).pluck('buckets'));
+    },
+    error);
+  },
+
+  discreteHistogram: function (nbuckets, column, callback, error) {
+    var query = 'SELECT DISTINCT(<%= column %>) AS bucket, count(*) AS value FROM (<%= sql %>) _table_sql GROUP BY <%= column %> ORDER BY value DESC LIMIT <%= nbuckets %> + 1';
+
+    var sql = _.template(query, {
+      column: column,
+      nbuckets: nbuckets,
+      sql: this.getSQL()
+    });
+
+    this._sqlQuery(sql, function (data) {
+      var count = data.rows.length;
+      var reached_limit = false;
+
+      if (count > nbuckets) {
+        data.rows = data.rows.slice(0, nbuckets);
+        reached_limit = true;
+      }
+
+      callback({ rows: data.rows, reached_limit: reached_limit }); // eslint-disable-line
+    });
+  },
+
+  date_histogram: function (nbuckets, column, callback, error) {
+    column = 'EXTRACT(EPOCH FROM ' + column + '::TIMESTAMP WITH TIME ZONE )';
+
+    var tmpl = _.template(
+      'with bounds as ( ' +
+     'SELECT  ' +
+      'current_timestamp as tz, ' +
+      'min(<%= column %>) as lower,  ' +
+      'max(<%= column %>) as upper,  ' +
+      '(max(<%= column %>) - min(<%= column %>)) as span,  ' +
+      'CASE WHEN ABS((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>) <= 0 THEN 1 ELSE GREATEST(1.0, pow(10,ceil(log((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>)))) END as bucket_size ' +
+      'FROM  (<%= sql %>) _table_sql ' +
+    ')  ' +
+    'select array_agg(v) val, array_agg(bucket) buckets, tz, bounds.upper, bounds.lower, bounds.span, bounds.bucket_size from ' +
+    '( ' +
+    'select  ' +
+      'count(<%= column %>) as v,   ' +
+      'round((<%= column %> - bounds.lower)/bounds.bucket_size) as bucket  ' +
+       'from (<%= sql %>) _table_sql, bounds  ' +
+       'where <%= column %> is not null ' +
+       'group by bucket order by bucket ' +
+    ') a, bounds ' +
+     'group by ' +
+    'bounds.upper, bounds.lower, bounds.span, bounds.bucket_size, bounds.tz ');
+
+    // transform array_agg from postgres to a js array
+    function agg_array (a) {
+      return a.map(function (v) { return parseFloat(v); });
+    }
+
+    this._sqlQuery(tmpl({
+      nbuckets: nbuckets,
+      sql: this.getSQL(),
+      column: column
+    }),
+
+    function (data) {
+      if (!data.rows || data.rows.length === 0) {
+        callback(null, null);
+        return;
+      }
+
+      data = data.rows[0];
+      data.val = agg_array(data.val);
+      data.buckets = agg_array(data.buckets);
+
+      var hist = [];
+      var bounds = {};
+
+      // create a sorted array and normalize
+      var upper = data.upper;
+      var lower = data.lower;
+      var tz = data.tz;
+      var bucket_size = data.bucket_size;
+      var max;
+
+      max = data.val[0];
+
+      for (var r = 0; r < data.buckets.length; ++r) {
+        var b = data.buckets[r];
+        var v = hist[b] = data.val[r];
+        max = Math.max(max, v);
+      }
+
+      // var maxBucket = _.max(data.buckets)
+      for (var i = 0; i < hist.length; ++i) {
+        if (hist[i] === undefined) {
+          hist[i] = 0;
+        } else {
+          hist[i] = hist[i] / max;
+        }
+      }
+
+      bounds.upper = parseFloat(upper);
+      bounds.lower = parseFloat(lower);
+      bounds.bucket_size = parseFloat(bucket_size);
+      bounds.tz = tz;
+
+      callback(hist, bounds);
+    },
+
+    error);
+  },
+
+  histogram: function (nbuckets, column, callback, error) {
+    var tmpl = _.template(
+      'with bounds as ( ' +
+     'SELECT  ' +
+      'min(<%= column %>) as lower,  ' +
+      'max(<%= column %>) as upper,  ' +
+      '(max(<%= column %>) - min(<%= column %>)) as span,  ' +
+      'CASE WHEN ABS((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>) <= 0 THEN 1 ELSE GREATEST(1.0, pow(10,ceil(log((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>)))) END as bucket_size ' +
+      'FROM  (<%= sql %>) _table_sql  ' +
+    ')  ' +
+    'select array_agg(v) val, array_agg(bucket) buckets, bounds.upper, bounds.lower, bounds.span, bounds.bucket_size from ' +
+    '( ' +
+    'select  ' +
+      'count(<%= column %>) as v,   ' +
+      'round((<%= column %> - bounds.lower)/bounds.bucket_size) as bucket  ' +
+       'from (<%= sql %>) _table_sql, bounds  ' +
+       'where <%= column %> is not null ' +
+       'group by bucket order by bucket ' +
+    ') a, bounds ' +
+     'group by ' +
+    'bounds.upper, ' +
+    'bounds.lower, bounds.span, bounds.bucket_size ');
+
+    // transform array_agg from postgres to a js array
+    function agg_array (a) {
+      return a.map(function (v) { return parseFloat(v); });
+      // return JSON.parse(a.replace('{', '[').replace('}', ']'))
+    }
+
+    this._sqlQuery(tmpl({
+      nbuckets: nbuckets,
+      sql: this.getSQL(),
+      column: column
+    }),
+
+    function (data) {
+      if (!data.rows || data.rows.length === 0) {
+        callback(null, null);
+        return;
+      }
+
+      data = data.rows[0];
+
+      data.val = agg_array(data.val);
+      data.buckets = agg_array(data.buckets);
+
+      var hist = [];
+      var bounds = {};
+
+      // create a sorted array and normalize
+      var upper = data.upper;
+      var lower = data.lower;
+      var bucket_size = data.bucket_size;
+      var max;
+
+      max = data.val[0];
+
+      for (var r = 0; r < data.buckets.length; ++r) {
+        var b = data.buckets[r];
+        var v = hist[b] = data.val[r];
+        max = Math.max(max, v);
+      }
+
+      // var maxBucket = _.max(data.buckets)
+      for (var i = 0; i < hist.length; ++i) {
+        if (hist[i] === undefined) {
+          hist[i] = 0;
+        } else {
+          hist[i] = hist[i] / max;
+        }
+      }
+
+      bounds.upper = parseFloat(upper);
+      bounds.lower = parseFloat(lower);
+      bounds.bucket_size = parseFloat(bucket_size);
+
+      callback(hist, bounds);
+    },
+
+    error);
+  },
+
+  jenkBins: function (nslots, column, callback, error) {
+    this._quantificationMethod('CDB_JenksBins', nslots, column, true, callback, error);
+  },
+
+  headTails: function (nslots, column, callback, error) {
+    this._quantificationMethod('CDB_HeadsTailsBins', nslots, column, false, callback, error);
+  },
+
+  quantileBins: function (nslots, column, callback, error) {
+    this._quantificationMethod('CDB_QuantileBins', nslots, column, false, callback, error);
+  },
+
+  categoriesForColumn: function (max_values, column, callback, error) {
+    var tmpl = _.template('SELECT <%= column %>, count(<%= column %>) FROM (<%= sql %>) _table_sql ' +
+      'GROUP BY <%= column %> ORDER BY count DESC LIMIT <%= max_values %> '
+    );
+
+    this._sqlQuery(tmpl({
+      sql: this.getSQL(),
+      column: column,
+      max_values: max_values + 1
+    }),
+    function (data) {
+      callback({// eslint-disable-line
+        type: data.fields[column].type || 'string',
+        categories: _(data.rows).pluck(column)
+      });
+    },
+    error);
+  },
+
+  /**
+   * call callback with the geometry bounds
+   */
+  geometryBounds: function (callback) {
+    var tmpl = _.template('SELECT ST_XMin(ST_Extent(the_geom)) as minx,ST_YMin(ST_Extent(the_geom)) as miny, ST_XMax(ST_Extent(the_geom)) as maxx,ST_YMax(ST_Extent(the_geom)) as maxy from (<%= sql %>) _table_sql');
+    this._sqlQuery(tmpl({
+      sql: this.getSQL()
+    }),
+    function (result) {
+      var coordinates = result.rows[0];
+
+      var lon0 = coordinates.maxx;
+      var lat0 = coordinates.maxy;
+      var lon1 = coordinates.minx;
+      var lat1 = coordinates.miny;
+
+      var minlat = -85.0511;
+      var maxlat = 85.0511;
+      var minlon = -179;
+      var maxlon = 179;
+
+      var clampNum = function (x, min, max) {
+        return x < min ? min : x > max ? max : x;
+      };
+
+      lon0 = clampNum(lon0, minlon, maxlon);
+      lon1 = clampNum(lon1, minlon, maxlon);
+      lat0 = clampNum(lat0, minlat, maxlat);
+      lat1 = clampNum(lat1, minlat, maxlat);
+      callback([ [lat0, lon0], [lat1, lon1]]); // eslint-disable-line
+    }
+    );
+  },
+
+  _sqlQuery: function (sql, callback, error, type) {
+    var s = encodeURIComponent(sql);
+    return $.ajax({
+      type: type || 'POST',
+      data: 'q=' + s + '&api_key=' + this._configModel.get('api_key'),
+      url: this.url(),
+      success: callback,
+      error: error
+    });
+  },
+
+  getSQL: function () {
+    // use table.id to fetch data because if always contains the real table name
+    return 'select * from ' + safeTableNameQuoting(this.table.get('id'));
+  },
+
+  fetch: function (opts) {
+    opts = opts || {};
+    if (!opts || !opts.add) {
+      this.options.attributes.page = 0;
+      this.options._previousAttributes.page = 0;
+      this.pages = [];
+    }
+    var error = opts.error;
+    opts.error = (model, resp) => {
+      this.fetched = true;
+      this.trigger('error', model, resp);
+      error && error(model, resp);
+    };
+    var success = opts.success;
+    opts.success = (model, resp) => {
+      this.fetched = true;
+      success && success.apply(this, arguments);
+    };
+    this._fetch(opts);
+  },
+
+  _fetch: function (opts) {
+    var MAX_GET_LENGTH = 1024;
+    this.trigger('loading', opts);
+
+    var sql = this.getSQL();
+    // if the query changes the database just send it
+    if (cartoMetadataStatic.alterTableData(sql)) {
+      TableDataCollection.prototype.fetch.call(this, opts);
+      return;
+    }
+
+    // use get to fetch the schema, probably cached
+    this._sqlQuery(_.template('select * from (<%= sql %>) __wrapped limit 0')({ sql: sql }), (data) => {
+      // get schema
+      this.query_schema = this._schemaFromQueryFields(data.fields);
+      if (!this.table.isInSQLView()) {
+        if ('the_geom' in this.query_schema) {
+          delete this.query_schema['the_geom_webmercator'];
+        }
+      }
+      TableDataCollection.prototype.fetch.call(this, opts);
+    }, (err) => {
+      this.trigger('error', this, err);
+    }, sql.length > MAX_GET_LENGTH ? 'POST' : 'GET');
+  },
+
+  /**
+   * with the data from the rows fetch create an schema
+   * if the schema from original table is passed the method
+   * set the column types according to it
+   * return an empty list if no data was fetch
+   */
+  schemaFromData: function (originalTableSchema) {
+    // build schema in format [ [field, type] , ...]
+    return cartoMetadataStatic.sortSchema(_(this.query_schema).map(function (v, k) {
+      return [k, v];
+    }));
+  },
+
+  geometryTypeFromGeoJSON: function (geojson) {
+    try {
+      var geo = JSON.parse(geojson);
+      return geo.type;
+    } catch (e) {
+    }
+  },
+
+  geometryTypeFromWKT: function (wkt) {
+    if (!wkt) return null;
+    var types = WKT.types;
+    wkt = wkt.toUpperCase();
+    for (var i = 0; i < types.length; ++i) {
+      var t = types[i];
+      if (wkt.indexOf(t) !== -1) {
+        return t;
+      }
+    }
+  },
+
+  geometryTypeFromWKB: function (wkb) {
+    if (!wkb) return null;
+
+    var typeMap = {
+      '0001': 'Point',
+      '0002': 'LineString',
+      '0003': 'Polygon',
+      '0004': 'MultiPoint',
+      '0005': 'MultiLineString',
+      '0006': 'MultiPolygon'
+    };
+
+    var bigendian = wkb[0] === '0' && wkb[1] === '0';
+    var type = wkb.substring(2, 6);
+    if (!bigendian) {
+      // swap '0100' => '0001'
+      type = type[2] + type[3] + type[0] + type[1];
+    }
+    return typeMap[type];
+  },
+
+  //
+  // guesses from the first row the geometry types involved
+  // returns an empty array where there is no rows
+  // return postgist types, like st_GEOTYPE
+  //
+  getGeometryTypes: function () {
+    var row = null;
+    var i = this.size();
+    while (i-- && !(row && row.get('the_geom'))) {
+      row = this.at(i);
+    }
+    if (!row) return [];
+    var geom = row.get('the_geom') || row.get('the_geom_webmercator');
+    var geoType = this.geometryTypeFromWKB(geom) || this.geometryTypeFromWKT(geom);
+    if (geoType) {
+      return ['ST_' + geoType[0].toUpperCase() + geoType.substring(1).toLowerCase()];
+    }
+    return [];
+  }
+
+});

lib/assets/javascripts/dashboard/data/table/column-model.js

@@ -0,0 +1,48 @@
+const _ = require('underscore');
+const Backbone = require('backbone');
+const checkAndBuildOpts = require('builder/helpers/required-opts');
+
+const REQUIRED_OPTS = [
+  'configModel'
+];
+
+module.exports = Backbone.Model.extend({
+
+  idAttribute: 'name',
+
+  url: function (method) {
+    var version = this._configModel.urlVersion('column', method);
+    var table = this.table || this.collection.table;
+    if (!table) {
+      console.error('column has no table assigned');
+    }
+
+    var base = '/api/' + version + '/tables/' + table.get('name') + '/columns/';
+    if (this.isNew()) {
+      return base;
+    }
+    return base + this.id;
+  },
+
+  initialize: function (opts) {
+    checkAndBuildOpts(opts, REQUIRED_OPTS, this);
+    this.table = this.get('table');
+    if (!this.table) {
+      throw 'you should specify a table model'; // eslint-disable-line
+    }
+    this.unset('table', { silent: true });
+  },
+
+  toJSON: function () {
+    var c = _.clone(this.attributes);
+    // this hack is created to create new column
+    // if you set _name instead name backbone does not get
+    // it as idAttribute so launch a POST instead of a PUT
+    if (c._name) {
+      c.name = c._name;
+      delete c._name;
+    }
+    return c;
+  }
+
+});

lib/assets/javascripts/dashboard/data/table/row-model.js

@@ -0,0 +1,158 @@
+const _ = require('underscore');
+const Backbone = require('backbone');
+const SQL = require('internal-carto.js').SQL;
+const WKT = require('dashboard/common/wkt');
+const checkAndBuildOpts = require('builder/helpers/required-opts');
+
+const REQUIRED_OPTS = [
+  'configModel'
+];
+
+const RESERVED_COLUMNS = 'the_geom the_geom_webmercator cartodb_id created_at updated_at'.split(' ');
+
+module.exports = Backbone.Model.extend({
+
+  _GEOMETRY_TYPES: {
+    'point': 'st_point',
+    'multipoint': 'st_multipoint',
+    'linestring': 'st_linestring',
+    'multilinestring': 'st_multilinestring',
+    'polygon': 'st_polygon',
+    'multipolygon': 'st_multipolygon'
+  },
+
+  initialize: function (attrs, opts) {
+    checkAndBuildOpts(opts, REQUIRED_OPTS, this);
+  },
+
+  url: function (method) {
+    var version = this._configModel.urlVersion('record', method);
+    var table = this.table || this.collection.table;
+    if (!table) {
+      console.error('row has no table assigned');
+    }
+
+    var base = '/api/' + version + '/tables/' + table.get('name') + '/records/';
+    if (this.isNew()) {
+      return base;
+    }
+    return base + this.id;
+  },
+
+  fetch: function (opts) {
+    opts = opts || {};
+    var username = (this.options && this.options.user_data) ? this.options.user_data.username
+      : (window.user_data ? window.user_data.username : window.user_name);
+    var api_key = (this.options && this.options.user_data) ? this.options.user_data.api_key
+      : (window.user_data ? window.user_data.api_key : window.api_key);
+
+    var table = this.table || this.collection.table;
+
+    var sqlApi = new SQL({
+      user: username,
+      version: 'v2',
+      api_key: api_key,
+      sql_api_template: this._configModel.getSqlApiBaseUrl(),
+      extra_params: ['skipfields']
+    });
+    // this.trigger('loading')
+    var sql = null;
+    var columns = table.columnNames();
+    if (opts.no_geom) {
+      columns = _.without(columns, 'the_geom', 'the_geom_webmercator');
+    } else {
+      columns = _.without(columns, 'the_geom');
+    }
+    sql = 'SELECT ' + columns.join(',') + ' ';
+    if (table.containsColumn('the_geom') && !opts.no_geom) {
+      sql += ',ST_AsGeoJSON(the_geom, 8) as the_geom ';
+    }
+    sql += ' from (' + table.data().getSQL() + ') _table_sql WHERE cartodb_id = ' + this.get('cartodb_id');
+    // Added opts to sql execute function to apply
+    // parameters ( like cache ) to the ajax request
+    if (opts.no_geom) {
+      opts.skipfields = 'the_geom,the_geom_webmercator';
+    } else {
+      opts.skipfields = 'the_geom_webmercator';
+    }
+    sqlApi.execute(sql, {}, opts).done(function (data) {
+      if (this.parse(data.rows[0])) {
+        this.set(data.rows[0]);//, {silent: silent});
+        this.trigger('sync');
+      }
+    });
+  },
+
+  toJSON: function () {
+    var attr = _.clone(this.attributes);
+    // remove read-only attributes
+    delete attr['updated_at'];
+    delete attr['created_at'];
+    delete attr['the_geom_webmercator'];
+    if (!this.isGeometryGeoJSON()) {
+      delete attr['the_geom'];
+    }
+    return attr;
+  },
+
+  isGeomLoaded: function () {
+    var geojson = this.get('the_geom');
+    var column_types_WKT = WKT.types;
+    return (geojson !== 'GeoJSON' && geojson !== -1 && !_.contains(column_types_WKT, geojson));
+  },
+
+  hasGeometry: function () {
+    var the_geom = this.get('the_geom');
+    return !!(the_geom != null && the_geom != undefined && the_geom != ''); // eslint-disable-line eqeqeq
+  },
+  /**
+   * Checks if the_geom contains a valid geoJson
+   */
+  isGeometryGeoJSON: function () {
+    var the_geom = this.get('the_geom');
+    if (the_geom && typeof the_geom === 'object') {
+      return !!the_geom.coordinates;
+    } else if (typeof the_geom !== 'string') {
+      return false;
+    }
+    // if the geom contains GeoJSON, the row has a valid geometry, but is not loaded yet
+    if (the_geom === 'GeoJSON') {
+      return true;
+    }
+
+    try {
+      var g = JSON.parse(the_geom);
+      return !!g.coordinates;
+    } catch (e) {
+      return false;
+    }
+  },
+
+  getFeatureType: function () {
+    if (this.isGeomLoaded()) {
+      // Problem geometry type from a WKB format
+      // Not possible for the moment
+      try {
+        var geojson = JSON.parse(this.get('the_geom'));
+        return geojson.type.toLowerCase();
+      } catch (e) {
+        console.info('Not possible to parse geometry type');
+      }
+    }
+    return null;
+  },
+
+  getGeomType: function () {
+    try {
+      return this._GEOMETRY_TYPES[this.getFeatureType()];
+    } catch (e) {
+      console.info('Not possible to parse geometry type');
+    }
+  }
+
+}, {
+  RESERVED_COLUMNS,
+  isReservedColumn: function (c) {
+    return _(RESERVED_COLUMNS).indexOf(c) !== -1;
+  }
+});

lib/assets/javascripts/dashboard/data/table/sqlviewdata-model.js

@@ -0,0 +1,102 @@
+const _ = require('underscore');
+const CartoTableData = require('dashboard/data/table/carto-table-data');
+const checkAndBuildOpts = require('builder/helpers/required-opts');
+
+const REQUIRED_OPTS = [
+  'configModel'
+];
+
+/**
+ * contains data for a sql view
+ * var s = new cdb.admin.SQLViewData({ sql : "select...." });
+ * s.fetch();
+ */
+module.exports = CartoTableData.extend({
+
+  UNDEFINED_TYPE_COLUMN: 'undefined',
+
+  initialize: function (models, options) {
+    checkAndBuildOpts(options, REQUIRED_OPTS, this);
+    this.model.prototype.idAttribute = 'cartodb_id';
+    CartoTableData.prototype.initialize.call(this, models, options);
+
+    this.bind('error', function () {
+      this.reset([]);
+    });
+    // this.initOptions();
+    if (options && options.sql) {
+      this.setSQL(options.sql);
+    }
+  },
+
+  _parseSQL: function (sql) {
+    sql = sql.replace(/([^\\]){x}/g, '$10').replace(/\\{x}/g, '{x}');
+    sql = sql.replace(/([^\\]){y}/g, '$10').replace(/\\{y}/g, '{y}');
+    sql = sql.replace(/([^\\]){z}/g, '$10').replace(/\\{z}/g, '{z}');
+
+    // Substitute mapnik tokens
+    // resolution at zoom level 0
+    var res = '156543.03515625';
+    // full webmercator extent
+    var ext = 'ST_MakeEnvelope(-20037508.5,-20037508.5,20037508.5,20037508.5,3857)';
+    sql = sql.replace('!bbox!', ext)
+      .replace('!pixel_width!', res)
+      .replace('!pixel_height!', res);
+
+    return sql;
+  },
+
+  sqlSource: function () {
+    return this.options.get('sql_source');
+  },
+
+  setSQL: function (sql, opts) {
+    opts = opts || {};
+    // reset options whiout changing raising a new fetchs
+    this.options.set({
+      page: 0,
+      sort_order: 'asc',
+      order_by: '',
+      filter_column: '',
+      filter_value: '',
+      sql_source: opts.sql_source || null
+    }, { silent: true });
+
+    var silent = opts.silent;
+    opts.silent = true;
+    this.options.set({ sql: sql ? this._parseSQL(sql) : '' }, opts);
+    if (!silent) {
+      this.options.trigger('change:sql', this.options, sql);
+    }
+  },
+
+  getSQL: function () {
+    return this.options.get('sql');
+  },
+
+  url: function () {
+    return this.sqlApiUrl();
+  },
+
+  isReadOnly: function () {
+    return this.sqlSource() !== 'filters';
+  },
+
+  quartiles: function (nslots, column, callback, error) {
+    var tmpl = _.template('SELECT quartile, max(<%= column %>) as maxAmount FROM (SELECT <%= column %>, ntile(<%= slots %>) over (order by <%= column %>) as quartile FROM (<%= sql %>) as _rambo WHERE <%= column %> IS NOT NULL) x GROUP BY quartile ORDER BY quartile');
+    this._sqlQuery(tmpl({
+      slots: nslots,
+      sql: this.options.get('sql'),
+      column: column
+    }),
+    function (data) {
+      callback(_(data.rows).pluck('maxamount'));
+    },
+    error);
+  },
+
+  // returns if the query contains geo data
+  isGeoreferenced: function () {
+    return this.getGeometryTypes().length > 0;
+  }
+});

lib/assets/javascripts/dashboard/data/table/table-data-collection.js

@@ -0,0 +1,27 @@
+const Backbone = require('backbone');
+
+module.exports = Backbone.Collection.extend({
+  fetched: false,
+
+  initialize: function () {
+    this.bind('sync', () => {
+      this.fetched = true;
+    });
+  },
+
+  /**
+   * get value for row index and columnName
+   */
+  getCell: function (index, columnName) {
+    var r = this.at(index);
+    if (!r) {
+      return null;
+    }
+    return r.get(columnName);
+  },
+
+  isEmpty: function () {
+    return this.length === 0;
+  }
+
+});