Initial commit

app/models/access_token.rb

@@ -0,0 +1,38 @@
+class AccessToken < OauthToken
+
+  def before_create
+    self.authorized_at = Time.now
+    self.created_at = Time.now
+    super
+  end
+
+  def before_save
+    self.updated_at = Time.now
+    super
+  end
+
+  def after_create
+    store_api_credentials
+  end
+  
+  def after_destroy
+    $api_credentials.del metadata_key
+    super
+  end
+  
+  def store_api_credentials
+    $api_credentials.hset metadata_key, "consumer_key", client_application.key
+    $api_credentials.hset metadata_key, "consumer_secret", client_application.secret
+    $api_credentials.hset metadata_key, "access_token_token", token
+    $api_credentials.hset metadata_key, "access_token_secret", secret
+    $api_credentials.hset metadata_key, "user_id", user_id
+    $api_credentials.hset metadata_key, "time", authorized_at
+  end
+  
+  private
+  
+  def metadata_key
+    "rails:oauth_access_tokens:#{token}"
+  end
+  
+end

app/models/asset.rb

@@ -0,0 +1,184 @@
+require 'open-uri'
+require_relative '../../lib/cartodb/image_metadata.rb'
+require_relative '../helpers/file_upload'
+require_dependency 'carto/configuration'
+
+class Asset < Sequel::Model
+  include Carto::Configuration
+
+  many_to_one :user
+
+  KIND_ORG_AVATAR = 'orgavatar'
+
+  VALID_EXTENSIONS = %w{ .jpeg .jpg .gif .png .svg }
+
+  attr_accessor :asset_file, :url
+
+ def before_create
+  store
+  super
+ end
+
+  def after_destroy
+    super
+    remove unless self.public_url.blank?
+  end
+
+  def validate
+    super
+    errors.add(:user_id, "can't be blank") if user_id.blank?
+
+    download_file if url.present?
+    validate_file if asset_file.present?
+  end
+
+  def download_file
+    dir = Dir.mktmpdir
+    stdout, stderr, status = Open3.capture3('wget', '-nv', '-P', dir, '-E', url)
+    self.asset_file = Dir[File.join(dir, '*')][0]
+    errors.add(:url, "is invalid") unless status.exitstatus == 0
+  end
+
+  def max_size
+    Cartodb::config[:assets]["max_file_size"]
+  end
+
+  def validate_file
+    extension = asset_file_extension
+    unless VALID_EXTENSIONS.include?(extension)
+      errors.add(:file, "has invalid format")
+      return
+    end
+
+    @file = open_file(asset_file)
+    unless @file && File.readable?(@file.path)
+      errors.add(:file, "is invalid")
+      return
+    end
+
+    max_size_in_mb = (max_size.to_f / (1024 * 1024).to_f).round(2)
+    if @file.size > max_size
+      errors.add(:file, "is too big, #{max_size_in_mb}MB max")
+      return
+    end
+
+    metadata = CartoDB::ImageMetadata.new(@file.path, extension: extension)
+    errors.add(:file, "is too big, 1024x1024 max") if metadata.width > 1024 || metadata.height > 1024
+    # If metadata reports no size, 99% sure not valid, so out
+    errors.add(:file, "doesn't appear to be an image") if metadata.width == 0 || metadata.height == 0
+  rescue => e
+    errors.add(:file, "error while uploading: #{e.message}")
+  end
+
+  def asset_file_extension
+    filename = asset_file.respond_to?(:original_filename) ? asset_file.original_filename : asset_file
+    extension = File.extname(filename).downcase
+
+    # Filename might include a postfix hash -- Rack::Test::UploadedFile adds it
+    extension.gsub!(/\d+-\w+-\w+\z/, '') if Rails.env.test?
+
+    extension if VALID_EXTENSIONS.include?(extension)
+  end
+
+  ##
+  # Tries to open the specified file object or full path
+  #
+  def open_file(handle)
+    (handle.respond_to?(:path) ? handle : File.open(handle.to_s))
+  rescue Errno::ENOENT
+    nil
+  end
+
+  def store
+    return unless @file
+    filename = (@file.respond_to?(:original_filename) ? @file.original_filename : File.basename(@file))
+    filename = "#{Time.now.strftime("%Y%m%d%H%M%S")}#{filename}"
+
+    remote_url = (use_s3? ? save_to_s3(filename) : save_local(filename))
+    self.set(public_url: remote_url)
+    self.this.update(public_url: remote_url)
+  end
+
+  def save_to_s3(filename)
+    obj = s3_bucket.object("#{target_asset_path}#{filename}")
+    obj.upload_file(@file.path, acl: 'public-read', content_type: MIME::Types.type_for(filename).first.to_s)
+
+    obj.public_url.to_s
+  end
+
+  def local_dir
+    @local_dir ||= Pathname.new(public_uploaded_assets_path).join(target_asset_path)
+  end
+
+  def local_filename(filename)
+    local_dir.join(filename)
+  end
+
+  def save_local(filename)
+    FileUtils.mkdir_p local_dir
+    FileUtils.cp @file.path, local_filename(filename)
+
+    mode = chmod_mode
+    FileUtils.chmod(mode, local_filename(filename)) if mode
+
+    File.join('/', ASSET_SUBFOLDER, target_asset_path, ERB::Util.url_encode(filename))
+  end
+
+  def use_s3?
+    Cartodb.get_config(:assets, "s3_bucket_name") && Cartodb.get_config(:aws, "s3")
+  end
+
+  def remove
+    unless use_s3?
+      local_url = CGI.unescape(public_url.gsub(/(http:)?\/\/#{CartoDB.account_host}/, ''))
+      begin
+        FileUtils.rm((public_uploaded_assets_path + local_url).gsub('/uploads/uploads/', '/uploads/'))
+      rescue => e
+        CartoDB::Logger.error(message: "Error removing asset", asset: self, exception: e)
+      end
+      return
+    end
+    basename = File.basename(public_url)
+    obj = s3_bucket.object("#{target_asset_path}#{basename}")
+    obj.delete
+  end
+
+  def target_asset_path
+    "#{Rails.env}/#{self.user.username}/assets/"
+  end
+
+  def s3_bucket
+    s3_config = Cartodb.config[:aws]["s3"].symbolize_keys
+    Aws.config = s3_config
+    s3 = Aws::S3::Resource.new
+    @s3_bucket ||= s3.bucket(Cartodb.config[:assets]["s3_bucket_name"])
+  end
+
+  ASSET_SUBFOLDER = 'uploads'.freeze
+
+  def absolute_public_url
+    uri = URI.parse(public_url)
+    (uri.absolute? ? uri : URI.join(base_domain, uri)).to_s
+  rescue URI::InvalidURIError
+    public_url
+  end
+
+  private
+
+  def base_domain
+    CartoDB.base_domain_from_name(user ? user.subdomain : organization.name)
+  end
+
+  def chmod_mode
+    # Example in case asset kind should change mode
+    # kind == KIND_ORG_AVATAR ? 0644 : nil
+    0644
+  end
+
+  def asset_protocol
+    # Avatars without protocol to allow the browser pick http/https.
+    # Other assets with http, because, for example, tiler needs access to landmark images.
+    kind == KIND_ORG_AVATAR ? '' : 'http:'
+  end
+
+end

app/models/automatic_geocoding.rb

@@ -0,0 +1,52 @@
+class AutomaticGeocoding < Sequel::Model
+
+  MAX_RETRIES = 5
+
+  many_to_one :table, class: :UserTable
+  one_to_many :geocodings, order: :created_at
+
+  def self.active
+    AutomaticGeocoding.where("state IN ?", ['created', 'idle']).all.select do |ag|
+      last_modified = ag.table.data_last_modified || 1.year.ago
+      last_modified > ag.ran_at
+    end
+  end # active
+
+  def before_create
+    super
+    self.created_at ||= Time.now
+    self.ran_at     ||= Time.now
+    self.state      ||= 'created'
+  end # before_create
+
+  def before_save
+    super
+    self.updated_at = Time.now
+  end # before_save
+
+  def validate
+    super
+    validates_presence :table_id
+  end # validate
+
+  def enqueue
+    self.update(state: 'on_queue')
+    Resque.enqueue(Resque::AutomaticGeocoderJobs, job_id: id)
+  end # enqueue
+
+  def run
+    self.update(state: 'running')
+    options = { 
+      user_id:                table.owner.id,
+      table_id:               table.id,
+      formatter:              geocodings.first.formatter,
+      automatic_geocoding_id: self.id
+    }
+
+    Geocoding.create(options).run!
+    self.update(state: 'idle', ran_at: Time.now, retried_times: 0)
+  rescue => e
+    self.update(state: 'failed') and raise(e) if retried_times.to_i >= MAX_RETRIES
+    self.update(retried_times: retried_times.to_i + 1, state: 'idle')
+  end # run
+end # AutomaticGeocoding

app/models/carto/account_type.rb

@@ -0,0 +1,31 @@
+module Carto
+  class AccountType < ActiveRecord::Base
+
+    belongs_to :rate_limit, dependent: :destroy
+
+    def soft_geocoding_limit?(user)
+      !!user[:soft_geocoding_limit]
+    end
+
+    def soft_here_isolines_limit?(user)
+      !!user[:soft_here_isolines_limit]
+    end
+
+    def soft_obs_snapshot_limit?(user)
+      !!user[:soft_obs_snapshot_limit]
+    end
+
+    def soft_obs_general_limit?(user)
+      !!user[:soft_obs_general_limit]
+    end
+
+    def soft_mapzen_routing_limit?(user)
+      !!user[:soft_mapzen_routing_limit]
+    end
+
+    def mailchimp?(_user)
+      # Mailchimp is currently not supported
+      false
+    end
+  end
+end

app/models/carto/analysis.rb

@@ -0,0 +1,114 @@
+require 'json'
+require_relative './carto_json_serializer'
+
+require_dependency 'carto/table_utils'
+
+class Carto::Analysis < ActiveRecord::Base
+  extend Carto::TableUtils
+
+  serialize :analysis_definition, ::Carto::CartoJsonSymbolizerSerializer
+  validates :analysis_definition, carto_json_symbolizer: true
+  validate :validate_user_not_viewer
+
+  before_destroy :validate_user_not_viewer
+
+  belongs_to :visualization, class_name: Carto::Visualization
+  belongs_to :user, class_name: Carto::User
+
+  after_save :update_map_dataset_dependencies, :notify_map_change
+  after_destroy :update_map_dataset_dependencies, :notify_map_change
+
+  def self.find_by_natural_id(visualization_id, natural_id)
+    analysis = find_by_sql(
+      [
+        "select id from analyses where visualization_id = :visualization_id and analysis_definition ->> 'id' = :natural_id",
+        { visualization_id: visualization_id, natural_id: natural_id }
+      ]
+    ).first
+    # Load all data
+    analysis.reload if analysis
+    analysis
+  end
+
+  def self.source_analysis_for_layer(layer, index)
+    map = layer.map
+    visualization = map.visualization
+    user = visualization.user
+
+    visualization_id = visualization.id
+    user_id = user.id
+
+    layer_options = layer.options
+    username = layer_options[:user_name] || layer.user.username
+    table_name = layer_options[:table_name]
+
+    qualified_table_name = if layer.user.organization_user?
+                             safe_schema_and_table_quoting(username, table_name)
+                           else
+                             table_name
+                           end
+
+    analysis_definition = {
+      id: 'abcdefghijklmnopqrstuvwxyz'[index] + '0',
+      type: 'source',
+      params: { query: layer.default_query(user) },
+      options: { table_name: qualified_table_name }
+    }
+
+    new(visualization_id: visualization_id, user_id: user_id, analysis_definition: analysis_definition)
+  end
+
+  def update_table_name(old_name, new_name)
+    analysis_node.fix_analysis_node_queries(user.username, user, old_name => new_name)
+    save
+  end
+
+  def analysis_definition_for_api
+    filter_valid_properties(analysis_node)
+  end
+
+  def natural_id
+    pj = analysis_definition
+    return nil unless pj
+    pj[:id]
+  end
+
+  def map
+    return nil unless visualization
+    visualization.map
+  end
+
+  def analysis_node
+    Carto::AnalysisNode.new(analysis_definition)
+  end
+
+  private
+
+  # Analysis definition contains attributes not needed by Analysis API (see #7128).
+  # This methods extract the needed ones.
+  VALID_ANALYSIS_PROPERTIES = [:id, :type, :params].freeze
+
+  def filter_valid_properties(node)
+    valid = node.definition.select { |property, _| VALID_ANALYSIS_PROPERTIES.include?(property) }
+    node.children_and_location.each do |location, child|
+      child_in_hash = location.reduce(valid, :[])
+      child_in_hash.replace(filter_valid_properties(child))
+    end
+    valid
+  end
+
+  def update_map_dataset_dependencies
+    map.update_dataset_dependencies
+  end
+
+  def notify_map_change
+    map.notify_map_change if map
+  end
+
+  def validate_user_not_viewer
+    if user.viewer
+      errors.add(:user, "Viewer users can't edit analyses")
+      return false
+    end
+  end
+end

app/models/carto/analysis_node.rb

@@ -0,0 +1,100 @@
+require_dependency 'carto/query_rewriter'
+
+class Carto::AnalysisNode
+  include Carto::QueryRewriter
+
+  def initialize(definition)
+    @definition = definition
+  end
+
+  attr_reader :definition
+
+  def self.find_by_natural_id(visualization_id, natural_id)
+    analyses = Carto::Analysis.where(visualization_id: visualization_id).all
+    analyses.lazy.map { |analysis| analysis.analysis_node.find_by_id(natural_id) }.find(&:present?)
+  end
+
+  def id
+    definition[:id]
+  end
+
+  def type
+    definition[:type]
+  end
+
+  def params
+    definition[:params]
+  end
+
+  def non_child_params
+    param_locations = children_and_location.keys.select { |cal| cal.first == :params }.map(&:second)
+    params && params.reject { |key| param_locations.include?(key) }
+  end
+
+  def options
+    definition[:options] ||= Hash.new
+  end
+
+  def children
+    children_and_location.values
+  end
+
+  def children_and_location
+    @children ||= get_children(@definition)
+  end
+
+  def find_by_id(node_id)
+    return self if node_id == id
+    children.lazy.map { |child| child.find_by_id(node_id) }.find { |child| child }
+  end
+
+  def source?
+    type == 'source'
+  end
+
+  def table_source?(table_name)
+    # Maybe check params[:query]
+    source? && options && options[:table_name] == table_name
+  end
+
+  def source_descendants
+    return [self] if source?
+    children.map(&:source_descendants).flatten
+  end
+
+  def descendants
+    [self] + children.map(&:descendants).flatten
+  end
+
+  def fix_analysis_node_queries(old_username, new_user, renamed_tables)
+    if options && options.key?(:table_name)
+      old_table_name = options[:table_name]
+      old_username, old_table_name = old_table_name.split('.') if old_table_name.include?('.')
+      options[:table_name] = renamed_tables.fetch(old_table_name, old_table_name)
+    end
+
+    if params && old_username
+      query = params[:query]
+      params[:query] = rewrite_query(query, old_username, new_user, renamed_tables) if query.present?
+    end
+
+    children.each { |child| child.fix_analysis_node_queries(old_username, new_user, renamed_tables) }
+  end
+
+  private
+
+  MANDATORY_KEYS_FOR_ANALYSIS_NODE = [:id, :type, :params].freeze
+  def get_children(definition, path = [])
+    children = definition.map do |k, v|
+      if v.is_a?(Hash)
+        this_path = path + [k]
+        if (MANDATORY_KEYS_FOR_ANALYSIS_NODE - v.keys).empty?
+          { this_path => Carto::AnalysisNode.new(v) }
+        else
+          get_children(v, this_path)
+        end
+      end
+    end
+    children.flatten.compact.reduce({}, :merge)
+  end
+end

app/models/carto/api_key.rb

@@ -0,0 +1,647 @@
+require 'securerandom'
+require_dependency 'carto/errors'
+require_dependency 'carto/helpers/auth_token_generator'
+require_dependency 'carto/oauth_provider/scopes/scopes'
+require_dependency 'carto/api_key_permissions'
+
+class ApiKeyGrantsValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    return record.errors[attribute] = ['grants has to be an array'] unless value&.is_a?(Array)
+
+    record.errors[attribute] << 'only one apis section is allowed' unless value.count { |v| v[:type] == 'apis' } == 1
+
+    max_one_sections = ['database', 'dataservices', 'user']
+    max_one_sections.each do |section|
+      if value.count { |v| v[:type] == section } > 1
+        record.errors[attribute] << "only one #{section} section is allowed"
+      end
+    end
+  end
+end
+
+module Carto
+  class ApiKey < ActiveRecord::Base
+
+    include Carto::AuthTokenGenerator
+
+    TYPE_REGULAR = 'regular'.freeze
+    TYPE_MASTER = 'master'.freeze
+    TYPE_DEFAULT_PUBLIC = 'default'.freeze
+    TYPE_OAUTH = 'oauth'.freeze
+    VALID_TYPES = [TYPE_REGULAR, TYPE_MASTER, TYPE_DEFAULT_PUBLIC, TYPE_OAUTH].freeze
+
+    NAME_MASTER = 'Master'.freeze
+    NAME_DEFAULT_PUBLIC = 'Default public'.freeze
+
+    API_SQL       = 'sql'.freeze
+    API_MAPS      = 'maps'.freeze
+    API_DO        = 'do'.freeze
+
+    GRANTS_ALL_APIS = { type: "apis", apis: [API_SQL, API_MAPS] }.freeze
+    GRANTS_ALL_DATA_SERVICES = {
+      type: 'dataservices',
+      services: ['geocoding', 'routing', 'isolines', 'observatory']
+    }.freeze
+    GRANTS_ALL_USER_DATA = {
+      type: 'user',
+      data: ['profile']
+    }.freeze
+
+    MASTER_API_KEY_GRANTS = [GRANTS_ALL_APIS, GRANTS_ALL_DATA_SERVICES, GRANTS_ALL_USER_DATA].freeze
+
+    TOKEN_DEFAULT_PUBLIC = 'default_public'.freeze
+
+    TYPE_WEIGHTED_ORDER = "CASE WHEN type = '#{TYPE_MASTER}' THEN 3 " \
+                          "WHEN type = '#{TYPE_DEFAULT_PUBLIC}' THEN 2 " \
+                          "WHEN type = '#{TYPE_REGULAR}' THEN 1 " \
+                          "ELSE 0 END DESC".freeze
+
+    CDB_CONF_KEY_PREFIX = 'api_keys_'.freeze
+
+    self.inheritance_column = :_type
+
+    belongs_to :user
+    has_one :oauth_access_token, inverse_of: :api_key
+
+    before_create :create_token, if: ->(k) { k.needs_setup? && !k.token }
+    before_create :create_db_config, if: ->(k) { k.needs_setup? && !(k.db_role && k.db_password) }
+
+    serialize :grants, Carto::CartoJsonSymbolizerSerializer
+
+    validates :grants, carto_json_symbolizer: true, api_key_grants: true, json_schema: true
+    validates :type, inclusion: { in: VALID_TYPES }
+    validates :type, uniqueness: { scope: :user_id }, unless: :needs_setup?
+    validates :name, presence: true, uniqueness: { scope: :user_id }
+
+    validate :valid_name_for_type
+    validate :check_permissions, unless: :skip_role_setup
+    validate :valid_master_key, if: :master?
+    validate :valid_default_public_key, if: :default_public?
+
+    after_create :setup_db_role, if: ->(k) { k.needs_setup? && !k.skip_role_setup }
+    after_save { remove_from_redis(redis_key(token_was)) if token_changed? }
+    after_save { invalidate_cache if token_changed? }
+    after_save :add_to_redis, if: :valid_user?
+    after_save :save_cdb_conf_info, unless: :skip_cdb_conf_info?
+
+    after_destroy :reassign_owner, :drop_db_role, if: ->(k) { k.needs_setup? && !k.skip_role_setup }
+    after_destroy :remove_from_redis
+    after_destroy :invalidate_cache
+    after_destroy :remove_cdb_conf_info, unless: :skip_cdb_conf_info?
+
+    scope :master, -> { where(type: TYPE_MASTER) }
+    scope :default_public, -> { where(type: TYPE_DEFAULT_PUBLIC) }
+    scope :regular, -> { where(type: TYPE_REGULAR) }
+    scope :user_visible, -> { where(type: [TYPE_MASTER, TYPE_DEFAULT_PUBLIC, TYPE_REGULAR]) }
+    scope :by_type, ->(types) { types.blank? ? user_visible : where(type: types) }
+    scope :order_weighted_by_type, -> { order(TYPE_WEIGHTED_ORDER) }
+
+    attr_accessor :skip_role_setup, :ownership_role_name
+    attr_writer :skip_cdb_conf_info
+
+    private_class_method :new, :create, :create!
+
+    def self.create_master_key!(user: Carto::User.find(scope_attributes['user_id']))
+      create!(
+        user: user,
+        type: TYPE_MASTER,
+        name: NAME_MASTER,
+        token: user.api_key,
+        grants: MASTER_API_KEY_GRANTS,
+        db_role: user.database_username,
+        db_password: user.database_password
+      )
+    end
+
+    def self.create_default_public_key!(user: Carto::User.find(scope_attributes['user_id']))
+      create!(
+        user: user,
+        type: TYPE_DEFAULT_PUBLIC,
+        name: NAME_DEFAULT_PUBLIC,
+        token: TOKEN_DEFAULT_PUBLIC,
+        grants: [GRANTS_ALL_APIS],
+        db_role: user.database_public_username,
+        db_password: CartoDB::PUBLIC_DB_USER_PASSWORD
+      )
+    end
+
+    def self.create_regular_key!(user: Carto::User.find(scope_attributes['user_id']), name:, grants:)
+      over_regular_quota = CartoDB::QuotaChecker.new(user).will_be_over_regular_api_key_quota?
+      raise CartoDB::QuotaExceeded.new('You have reached the limit of API keys for your plan') if over_regular_quota
+
+      create!(
+        user: user,
+        type: TYPE_REGULAR,
+        name: name,
+        grants: grants
+      )
+    end
+
+    def self.create_oauth_key!(user: Carto::User.find(scope_attributes['user_id']), name:, grants:, ownership_role_name:)
+      create!(
+        user: user,
+        type: TYPE_OAUTH,
+        name: name,
+        grants: grants,
+        ownership_role_name: ownership_role_name
+      )
+    end
+
+    def self.new_from_hash(api_key_hash)
+      new(
+        id: api_key_hash[:id],
+        created_at: api_key_hash[:created_at],
+        db_password: api_key_hash[:db_password],
+        db_role: api_key_hash[:db_role],
+        name: api_key_hash[:name],
+        token: api_key_hash[:token],
+        type: api_key_hash[:type],
+        updated_at: api_key_hash[:updated_at],
+        grants: api_key_hash[:grants],
+        user_id: api_key_hash[:user_id],
+        skip_role_setup: true,
+        skip_cdb_conf_info: true
+      )
+    end
+
+    def revoke_permissions(table, revoked_permissions)
+      Carto::TableAndFriends.apply(db_connection, table.database_schema, table.name) do |s, t, qualified_name|
+        query = %{
+          REVOKE #{revoked_permissions.join(', ')}
+          ON TABLE #{qualified_name}
+          FROM \"#{db_role}\"
+        }
+        db_run(query)
+        sequences_for_table(s, t).each do |seq|
+          db_run("REVOKE ALL ON SEQUENCE #{seq} FROM \"#{db_role}\"")
+        end
+      end
+    end
+
+    def granted_apis
+      @granted_apis ||= process_granted_apis
+    end
+
+    def table_permissions
+      @table_permissions_cache ||= process_table_permissions
+      @table_permissions_cache.values
+    end
+
+    def schema_permissions
+      @schema_permissions_cache ||= process_schema_permissions
+      @schema_permissions_cache.values
+    end
+
+    def dataset_metadata_permissions
+      @dataset_metadata_permissions ||= process_dataset_metadata_permissions
+    end
+
+    def data_observatory_permissions?
+      granted_apis&.include?(API_DO)
+    end
+
+    def table_permissions_from_db
+      query = %{
+          WITH permissions AS (
+            SELECT
+                table_schema,
+                table_name,
+                string_agg(DISTINCT lower(privilege_type),',') privilege_types
+            FROM
+                information_schema.table_privileges tp
+            WHERE
+                tp.grantee = '#{db_role}'
+            GROUP BY
+                table_schema,
+                table_name
+          ),
+          ownership AS (
+            SELECT
+                n.nspname as table_schema,
+                relname as table_name
+            FROM pg_class
+            JOIN pg_catalog.pg_namespace n ON n.oid = pg_class.relnamespace
+            WHERE pg_catalog.pg_get_userbyid(relowner) = '#{db_role}'
+          )
+          SELECT
+              p.table_name,
+              p.table_schema,
+              p.privilege_types,
+              CASE WHEN o.table_name IS NULL THEN false
+                  ELSE true
+              END AS owner
+          FROM permissions p
+          LEFT JOIN ownership o ON (p.table_name = o.table_name AND p.table_schema = o.table_schema)
+        }
+      db_run(query).map do |line|
+        TablePermissions.new(schema: line['table_schema'],
+                             name: line['table_name'],
+                             owner: line['owner'] == 't' ? true : false,
+                             permissions: line['privilege_types'].split(','))
+      end
+    end
+
+    def schema_permissions_from_db
+      user.db_service.all_schemas_granted_hashed(db_role).map do |schema, permissions|
+        SchemaPermissions.new(name: schema, permissions: permissions)
+      end
+    end
+
+    def data_services
+      @data_services ||= process_data_services
+    end
+
+    def user_data
+      @user_data ||= process_user_data_grants
+    end
+
+    def regenerate_token!
+      if master?
+        # Send all master key updates through the user model, avoid circular updates
+        ::User[user.id].regenerate_api_key
+        reload
+      else
+        create_token
+        save!
+      end
+    end
+
+    def master?
+      type == TYPE_MASTER
+    end
+
+    def default_public?
+      type == TYPE_DEFAULT_PUBLIC
+    end
+
+    def regular?
+      type == TYPE_REGULAR
+    end
+
+    def oauth?
+      type == TYPE_OAUTH
+    end
+
+    def needs_setup?
+      regular? || oauth?
+    end
+
+    def data_services?
+      data_services.present?
+    end
+
+    def valid_name_for_type
+      if !master? && name == NAME_MASTER || !default_public? && name == NAME_DEFAULT_PUBLIC
+        errors.add(:name, "api_key name cannot be #{NAME_MASTER} nor #{NAME_DEFAULT_PUBLIC}")
+      end
+    end
+
+    def can_be_deleted?
+      regular?
+    end
+
+    def role_creation_queries
+      ["CREATE ROLE \"#{db_role}\" NOSUPERUSER NOCREATEDB LOGIN ENCRYPTED PASSWORD '#{db_password}'"]
+    end
+
+    def role_permission_queries
+      queries = [
+        "GRANT \"#{user.service.database_public_username}\" TO \"#{db_role}\"",
+        "ALTER ROLE \"#{db_role}\" SET search_path TO #{user.db_service.build_search_path}",
+      ]
+
+      # This is GRANTED to the organizational role for organization users, and the PUBLIC users for non-orgs
+      # We do not want to grant the organization role to the Api Keys, since that also opens access to the analysis
+      # catalog and tablemetadata. To be more consistent, we should probably GRANT this to the organization public
+      # user instead, but that has the downside of leaking quotas to the public.
+      # This works for now, but if you are adding new permissions, please reconsider this decision.
+      if user.organization_user?
+        queries << "GRANT ALL ON FUNCTION \"#{user.database_schema}\"._CDB_UserQuotaInBytes() TO \"#{db_role}\""
+      end
+      if regular?
+        queries << "GRANT \"#{db_role}\" TO \"#{user.database_username}\""
+      end
+      queries
+    end
+
+    def set_enabled_for_engine
+      # We enable/disable API keys for engine usage by adding/removing them from Redis
+      valid_user? ? add_to_redis : remove_from_redis
+    end
+
+    def save_cdb_conf_info
+      db_run("SELECT cartodb.cdb_conf_setconf('#{CDB_CONF_KEY_PREFIX}#{db_role}', '#{cdb_conf_info.to_json}');")
+    end
+
+    def remove_cdb_conf_info
+      db_run("SELECT cartodb.CDB_Conf_RemoveConf('#{CDB_CONF_KEY_PREFIX}#{db_role}');")
+    end
+
+    def cdb_conf_info
+      {
+        username: user.username,
+        permissions: data_services || [],
+        ownership_role_name: effective_ownership_role_name || ''
+      }
+    end
+
+    def skip_cdb_conf_info
+      @skip_cdb_conf_info || false
+    end
+
+    def skip_cdb_conf_info?
+      skip_cdb_conf_info.present?
+    end
+
+    def effective_ownership_role_name
+      return if schema_permissions.all? { |s| s.permissions.empty? }
+      ownership_role_name || oauth_access_token.try(:ownership_role_name)
+    end
+
+    def grant_ownership_role_privileges
+      db_run("GRANT \"#{effective_ownership_role_name}\" TO \"#{db_role}\"") if effective_ownership_role_name.present?
+    end
+
+    private
+
+    PASSWORD_LENGTH = 40
+
+    REDIS_KEY_PREFIX = 'api_keys:'.freeze
+
+    def raise_unprocessable_entity_error(error)
+      raise Carto::UnprocesableEntityError.new(/PG::Error: ERROR:  (.+)/ =~ error.message && $1 || 'Unexpected error')
+    end
+
+    def invalidate_cache
+      return unless user
+      user.invalidate_varnish_cache
+    end
+
+    def create_token
+      begin
+        self.token = generate_auth_token
+      end while self.class.exists?(user_id: user_id, token: token)
+    end
+
+    def add_to_redis
+      redis_client.hmset(redis_key, redis_hash_as_array)
+    end
+
+    def process_granted_apis
+      apis = grants.find { |v| v[:type] == 'apis' }[:apis]
+      raise UnprocesableEntityError.new('apis array is needed for type "apis"') unless apis
+      apis
+    end
+
+    def process_table_permissions
+      table_permissions = {}
+
+      databases = grants.find { |v| v[:type] == 'database' }
+      return table_permissions unless databases.try(:[], :tables).present?
+
+      databases[:tables].each do |table|
+        table_id = "#{table[:schema]}.#{table[:name]}"
+        table_permissions[table_id] ||= Carto::TablePermissions.new(schema: table[:schema], name: table[:name])
+        table_permissions[table_id].merge!(table[:permissions])
+      end
+
+      table_permissions
+    end
+
+    def process_schema_permissions
+      schema_permissions = {}
+
+      databases = grants.find { |v| v[:type] == 'database' }
+      return schema_permissions unless databases.try(:[], :schemas).present?
+
+      databases[:schemas].each do |schema|
+        schema_id = schema[:name]
+        schema_permissions[schema_id] ||= Carto::SchemaPermissions.new(name: schema[:name])
+        schema_permissions[schema_id].merge!(schema[:permissions])
+      end
+
+      schema_permissions
+    end
+
+    def process_data_services
+      data_services_grants = grants.find { |v| v[:type] == 'dataservices' }
+      return nil unless data_services_grants.present?
+
+      data_services_grants[:services]
+    end
+
+    def process_user_data_grants
+      user_data_grants = grants.find { |v| v[:type] == 'user' }
+      return nil unless user_data_grants.present?
+
+      user_data_grants[:data]
+    end
+
+    def process_dataset_metadata_permissions
+      dataset_metadata_grants = grants.find { |v| v[:type] == 'database' }
+      dataset_metadata_grants.try(:[], :table_metadata)
+    end
+
+    def check_permissions
+      # Only checks if no previous errors in JSON definition
+      check_table_permissions
+      check_schema_permissions
+    end
+
+    def check_table_permissions
+      if errors[:grants].empty? && invalid_tables_permissions.any?
+        errors.add(:grants, 'can only grant table permissions you have')
+      end
+    end
+
+    def check_schema_permissions
+      if errors[:grants].empty? && invalid_schemas_permissions.any?
+        errors.add(:grants, 'can only grant schema permissions you have')
+      end
+    end
+
+    def invalid_tables_permissions
+      databases = grants.find { |v| v[:type] == 'database' }
+      return [] unless databases.try(:[], :tables).present?
+
+      allowed = user.db_service.all_tables_granted_hashed
+
+      invalid = []
+      databases[:tables].each do |table|
+        if allowed[table[:schema]].nil? ||
+           allowed[table[:schema]][table[:name]].nil? ||
+           (table[:permissions] - allowed[table[:schema]][table[:name]]).any?
+          invalid << table
+        end
+      end
+      invalid
+    end
+
+    def invalid_schemas_permissions
+      databases = grants.find { |v| v[:type] == 'database' }
+      return [] unless databases.try(:[], :schemas).present?
+
+      allowed = user.db_service.all_schemas_granted_hashed
+
+      invalid = []
+      databases[:schemas].each do |schema|
+        invalid_schema = (schema[:permissions] - allowed[schema[:name]].to_a).any?
+        invalid << schema if invalid_schema
+      end
+      invalid
+    end
+
+    def create_db_config
+      begin
+        self.db_role = Carto::DB::Sanitize.sanitize_identifier("carto_role_#{SecureRandom.hex}")
+      end while self.class.exists?(user_id: user_id, db_role: db_role)
+      self.db_password = SecureRandom.hex(PASSWORD_LENGTH / 2) unless db_password
+    end
+
+    def setup_db_role
+      create_role
+      setup_table_permissions
+      setup_schema_permissions
+      grant_ownership_role_privileges
+    end
+
+    def setup_table_permissions
+      setup_permissions(table_permissions) do |tp|
+        Carto::TableAndFriends.apply(db_connection, tp.schema, tp.name) do |schema, table_name, qualified_name|
+          db_run("GRANT #{tp.permissions.join(', ')} ON TABLE #{qualified_name} TO \"#{db_role}\"")
+          sequences_for_table(schema, table_name).each do |seq|
+            db_run("GRANT USAGE, SELECT ON SEQUENCE #{seq} TO \"#{db_role}\"")
+          end
+        end
+      end
+
+      schemas_from_granted_tables.each { |s| grant_usage_privileges_for_schema(s) }
+    end
+
+    def setup_schema_permissions
+      setup_permissions(schema_permissions) do |sp|
+        db_run("GRANT #{sp.permissions.join(', ')} ON SCHEMA \"#{sp.name}\" TO \"#{db_role}\"")
+      end
+    end
+
+    def setup_permissions(permissions)
+      non_existent = []
+      errors = []
+      permissions.each do |api_key_permission|
+        next if api_key_permission.permissions.empty?
+        begin
+          # here we catch exceptions to show a proper error to the user request
+          # this is because we allow OAuth requests to include a `datasets` or `schemas` scope with
+          # tables or schema that may or may not exist
+          yield api_key_permission
+        rescue Carto::UnprocesableEntityError => e
+          raise e unless e.message =~ /does not exist/
+          non_existent << api_key_permission.name
+          errors << e.message
+        end
+      end
+
+      raise Carto::RelationDoesNotExistError.new(errors, non_existent) unless non_existent.empty?
+    end
+
+    def create_role
+      (role_creation_queries + role_permission_queries).each { |q| db_run(q) }
+    end
+
+    def drop_db_role
+      db_run("DROP OWNED BY \"#{db_role}\"")
+      db_run("DROP ROLE IF EXISTS \"#{db_role}\"")
+    end
+
+    def reassign_owner
+      db_run("REASSIGN OWNED BY \"#{db_role}\" TO \"#{user.database_username}\";")
+    end
+
+    def schemas_from_granted_tables
+      # assume table friends don't introduce new schemas
+      table_permissions.map(&:schema).uniq
+    end
+
+    def redis_key(token = self.token)
+      "#{REDIS_KEY_PREFIX}#{user.username}:#{token}"
+    end
+
+    def remove_from_redis(key = redis_key)
+      redis_client.del(key)
+    end
+
+    def sequences_for_table(schema, table)
+      db_run(%{
+        SELECT
+          n.nspname, c.relname
+        FROM
+          pg_depend d
+          JOIN pg_class c ON d.objid = c.oid
+          JOIN pg_namespace n ON c.relnamespace = n.oid
+        WHERE
+          d.refobjsubid > 0 AND
+          d.classid = 'pg_class'::regclass AND
+          c.relkind = 'S'::"char" AND
+          d.refobjid = (
+            QUOTE_IDENT(#{db_connection.quote(schema)}) ||
+            '.' ||
+            QUOTE_IDENT(#{db_connection.quote(table)})
+          )::regclass;
+      }).map { |r| "\"#{r['nspname']}\".\"#{r['relname']}\"" }
+    end
+
+    def db_run(query, connection = db_connection)
+      connection.execute(query)
+    rescue ActiveRecord::StatementInvalid => e
+      CartoDB::Logger.warning(message: 'Error running SQL command', exception: e)
+      return if e.message =~ /OWNED BY/ # role might not exist becuase it has been already dropped      
+      raise_unprocessable_entity_error(e)
+    end
+
+    def user_db_run(query)
+      db_run(query, user_db_connection)
+    end
+
+    def db_connection
+      @db_connection ||= user.in_database(as: :superuser)
+    end
+
+    def user_db_connection
+      @user_db_connection ||= user.in_database
+    end
+
+    def redis_hash_as_array
+      hash = ['user', user.username, 'type', type, 'database_role', db_role, 'database_password', db_password]
+      granted_apis.each { |api| hash += ["grants_#{api}", true] }
+      hash
+    end
+
+    def redis_client
+      $users_metadata
+    end
+
+    def grant_usage_privileges_for_schema(schema)
+      db_run("GRANT USAGE ON SCHEMA \"#{schema}\" TO \"#{db_role}\"")
+      db_run("GRANT ALL ON FUNCTION \"#{schema}\"._CDB_UserQuotaInBytes() TO \"#{db_role}\"")
+    end
+
+    def valid_master_key
+      errors.add(:name, "must be #{NAME_MASTER} for master keys") unless name == NAME_MASTER
+      unless grants == MASTER_API_KEY_GRANTS
+        errors.add(:grants, "must grant all apis")
+      end
+      errors.add(:token, "must match user model for master keys") unless token == user.api_key
+    end
+
+    def valid_default_public_key
+      errors.add(:name, "must be #{NAME_DEFAULT_PUBLIC} for default public keys") unless name == NAME_DEFAULT_PUBLIC
+      errors.add(:grants, "must grant all apis") unless grants == [GRANTS_ALL_APIS]
+      errors.add(:token, "must be #{TOKEN_DEFAULT_PUBLIC} for default public keys") unless token == TOKEN_DEFAULT_PUBLIC
+    end
+
+    def valid_user?
+      # This is not avalidation per-se, since we don't want to remove api keys when a user is disabled
+      !(user.locked? || regular? && !user.engine_enabled?)
+    end
+  end
+end

app/models/carto/api_key_permissions.rb

@@ -0,0 +1,45 @@
+module Carto
+  class ApiKeyPermissions
+    attr_reader :name, :permissions
+
+    def initialize(name:, permissions: [])
+      @name = name
+      @permissions = permissions
+    end
+
+    def merge!(permissions)
+      down_permissions = permissions.map(&:downcase)
+      @permissions += down_permissions.reject { |p| @permissions.include?(p) }
+    end
+
+    def write?
+      !(@permissions & write_permissions).empty?
+    end
+
+    def write_permissions; end
+  end
+
+  class TablePermissions < ApiKeyPermissions
+    WRITE_PERMISSIONS = ['insert', 'update', 'delete', 'truncate'].freeze
+
+    attr_reader :schema, :owner
+
+    def initialize(schema:, name:, owner: false, permissions: [])
+      super(name: name, permissions: permissions)
+      @schema = schema
+      @owner = owner
+    end
+
+    def write_permissions
+      WRITE_PERMISSIONS
+    end
+  end
+
+  class SchemaPermissions < ApiKeyPermissions
+    WRITE_PERMISSIONS = ['create'].freeze
+
+    def write_permissions
+      WRITE_PERMISSIONS
+    end
+  end
+end

app/models/carto/asset.rb

@@ -0,0 +1,77 @@
+require_dependency 'carto/assets/image_assets_service'
+require_dependency 'carto/assets/organization_image_assets_service'
+require_dependency 'carto/assets/kuviz_assets_service'
+
+module Carto
+  class Asset < ActiveRecord::Base
+    belongs_to :user, class_name: Carto::User
+    belongs_to :organization, class_name: Carto::Organization
+    belongs_to :visualization, class_name: Carto::Visualization
+
+    serialize :storage_info, CartoJsonSymbolizerSerializer
+    validates :storage_info, carto_json_symbolizer: true
+
+    validates :user,         presence: true, unless: -> { organization.present? || visualization.present? }
+    validates :organization, presence: true, unless: -> { user.present? || visualization.present? }
+    validates :visualization, presence: true, unless: -> { user.present? || organization.present? }
+    validates :storage_info, presence: true, if: -> { organization.present? || visualization.present? }
+    validates :public_url,   presence: true
+
+    validate :validate_storage_info, if: :storage_info
+
+    before_destroy :remove_asset_from_storage, if: :storage_info
+
+    def self.for_organization(organization:, resource:)
+      storage_info, url = OrganizationImageAssetsService.instance.upload(organization, resource)
+
+      new(organization: organization,
+          public_url: url,
+          storage_info: storage_info,
+          kind: 'organization_asset')
+    end
+
+    def self.for_visualization(visualization:, resource:)
+      storage_info, url = KuvizAssetsService.instance.upload(visualization, resource)
+
+      new(visualization: visualization,
+          public_url: url,
+          storage_info: storage_info,
+          kind: 'kuviz_asset')
+    end
+
+    def update_visualization_resource(resource)
+      storage_info, url = KuvizAssetsService.instance.upload(visualization, resource)
+      self.public_url = url
+      self.storage_info = storage_info
+      save
+    end
+
+    def absolute_public_url
+      uri = URI.parse(public_url)
+      (uri.absolute? ? uri : URI.join(base_domain, uri)).to_s
+    rescue URI::InvalidURIError
+      public_url
+    end
+
+    private
+
+    def validate_storage_info
+      schema = Definition.instance.load_from_file('lib/formats/asset/storage_info.json')
+      indifferent_storage_info = storage_info.with_indifferent_access
+
+      errs = JSON::Validator::fully_validate(schema, indifferent_storage_info)
+      errors.add(:storage_info, errs.join(', ')) if errs.any?
+    end
+
+    def remove_asset_from_storage
+      Carto::AssetsService.new.remove(storage_info)
+    end
+
+    def base_domain
+      if visualization.present?
+        return CartoDB.base_domain_from_name(visualization.user.subdomain)
+      end
+      CartoDB.base_domain_from_name(user ? user.subdomain : organization.name)
+    end
+  end
+end

app/models/carto/automatic_geocoding.rb

@@ -0,0 +1,9 @@
+require 'active_record'
+
+module Carto
+  class AutomaticGeocoding < ActiveRecord::Base
+    has_many :geocodings, -> { order(:created_at) }
+    belongs_to :table, class_name: Carto::UserTable, inverse_of: :automatic_geocodings, foreign_key: :id
+    belongs_to :user, class_name: Carto::User, inverse_of: :automatic_geocodings
+  end
+end

app/models/carto/carto_json_serializer.rb

@@ -0,0 +1,66 @@
+module Carto
+  # Rails 3 stores nil strings with JSON serialization as 'null', so we need to replace it with a valid value.
+  # See https://github.com/rails/rails/issues/13144
+  class CartoJsonSerializer
+    def self.dump(hash)
+      hash.try(&:to_json)
+    end
+
+    def self.load(value)
+      value.try(&:with_indifferent_access)
+    end
+  end
+
+  # CartoJsonSerializer is closer to Rails Json serializer, using with_indifferent_access
+  # for load. The problem with that is that although you can _access_ first
+  # level keys both with symbols and keys, other common operations, such as
+  # key comparison, fail, and it only applies to the first level.
+  class CartoJsonSymbolizerSerializer < CartoJsonSerializer
+    def self.load(value)
+      value && symbolize(value)
+    end
+
+    def self.symbolize(value)
+      if value.is_a?(Hash)
+        value.deep_symbolize_keys
+      elsif value.is_a?(Array)
+        value.map { |v| symbolize(v) }
+      else
+        value
+      end
+    end
+    private_class_method :symbolize
+  end
+end
+
+class CartoJsonSymbolizerValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    if value && !(value.is_a?(Hash) || value.is_a?(Array))
+      record.errors[attribute] << 'wrongly formatted (not a Hash or invalid JSON)'
+    end
+  end
+end
+
+class JsonSchemaValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    return unless value
+
+    value_for_json = prepare_for_json(value)
+
+    schema = Carto::Definition.instance.load_from_file("lib/formats/#{record.class.to_s.underscore}/#{attribute}.json")
+    errors = JSON::Validator::fully_validate(schema, value_for_json, strict: true)
+    record.errors[attribute] << errors.join(', ') if errors.any?
+  end
+
+  private
+
+  def prepare_for_json(value)
+    if value.is_a? Hash
+      value.with_indifferent_access
+    elsif value.respond_to? :map
+      value.map { |v| prepare_for_json(v) }
+    else
+      value
+    end
+  end
+end

app/models/carto/client_application.rb

@@ -0,0 +1,11 @@
+require 'active_record'
+
+module Carto
+  class ClientApplication < ActiveRecord::Base
+
+    belongs_to :user, class_name: Carto::User
+    has_many :oauth_tokens, class_name: Carto::OauthToken
+    has_many :access_tokens, -> { where(type: ['AccessToken']) }, class_name: Carto::OauthToken
+
+  end
+end

app/models/carto/connector_configuration.rb

@@ -0,0 +1,66 @@
+require 'carto/connector'
+
+class Carto::ConnectorConfiguration < ActiveRecord::Base
+  belongs_to :connector_provider, class_name: Carto::ConnectorProvider
+  belongs_to :user, class_name: Carto::User, inverse_of: :connector_configurations
+  belongs_to :organization, class_name: Carto::Organization, inverse_of: :connector_configurations
+
+  # A ConnectorConfiguration can belong to either user or an organization, but not both;
+  # it may also be a default configuration (no user or organization).
+  # Only one configuration may exist for any valid combination of user, organization and provider.
+  # So we have 3 kinds of configuration records:
+  # * default configuration for provider: user.nil? && organization.nil? FIXME: remove this? (only app_config defaults)
+  # * organization configuration: user.nil? && organization.present?
+  # * user configuration: organization.nil? && user.present?
+  validates :connector_provider_id, presence: true
+  validates :user_id, uniqueness: { scope: [:connector_provider_id, :organization_id] }, if: 'organization_id.nil?'
+  validates :organization_id, uniqueness: { scope: [:connector_provider_id, :user_id] }, if: 'user_id.nil?'
+  validate :not_user_and_organization_simultaneously
+
+  def not_user_and_organization_simultaneously
+    if user_id.present? && organization_id.present?
+      errors.add(:user_id, "can't assign to an organization simultaneously with a user")
+    end
+  end
+
+  # columns:
+  # enabled boolean
+  # max_rows integer
+
+  def self.default(provider)
+    # Look for default provider configration
+    config = where(user_id: nil, organization_id: nil, connector_provider_id: provider.id).first
+    if !config
+      # Create in memory record using app_config defaults
+      config = new(
+        connector_provider: provider,
+        enabled:  Cartodb.get_config(:connectors, provider.name, 'enabled') || false,
+        max_rows: Cartodb.get_config(:connectors, provider.name, 'max_rows')
+      )
+    end
+    config
+  end
+
+  def self.for_organization(organization, provider)
+    if provider
+      config = where(organization_id: organization.id, connector_provider_id: provider.id).first
+      if config.blank?
+        config = default(provider)
+      end
+      config
+    end
+  end
+
+  def self.for_user(user, provider)
+    if provider
+      config = where(user_id: user.id, connector_provider_id: provider.id).first
+      if config.blank? && user.organization_id.present?
+        config = for_organization(user.organization, provider)
+      end
+      if config.blank?
+        config = default(provider)
+      end
+      config
+    end
+  end
+end

app/models/carto/connector_provider.rb

@@ -0,0 +1,8 @@
+require 'carto/connector'
+
+class Carto::ConnectorProvider < ActiveRecord::Base
+  validates :name, presence: true
+  validates :name, uniqueness: true
+
+  has_many :connector_configurations
+end

app/models/carto/dashboard_search_result.rb

@@ -0,0 +1,12 @@
+module Carto
+  class DashboardSearchResult
+
+    attr_accessor :tags, :visualizations, :total_count
+
+    def initialize(tags: [], visualizations: [], total_count: 0)
+      @tags = tags
+      @visualizations = visualizations
+      @total_count = total_count
+    end
+  end
+end

app/models/carto/data_import.rb

@@ -0,0 +1,42 @@
+require 'active_record'
+
+module Carto
+  class DataImport < ActiveRecord::Base
+    include Carto::DataImportConstants
+
+    # INFO: hack to workaround `ActiveRecord::DangerousAttributeError: logger is defined by ActiveRecord`
+    class << self
+      def instance_method_already_implemented?(method_name)
+        return true if method_name == 'logger'
+        super
+      end
+    end
+
+    STATE_ENQUEUED  = 'enqueued'  # Default state for imports whose files are not yet at "import source"
+    STATE_PENDING   = 'pending'   # Default state for files already at "import source" (e.g. S3 bucket)
+    STATE_UNPACKING = 'unpacking'
+    STATE_IMPORTING = 'importing'
+    STATE_COMPLETE  = 'complete'
+    STATE_UPLOADING = 'uploading'
+    STATE_FAILURE   = 'failure'
+    STATE_STUCK     = 'stuck'
+
+    belongs_to :user, class_name: Carto::User
+    belongs_to :log, class_name: Carto::Log, foreign_key: :logger
+    has_many :external_data_imports, inverse_of: :data_import, class_name: Carto::ExternalDataImport
+    has_many :user_tables, class_name: Carto::UserTable
+
+    validate :validate_collision_strategy
+
+    def is_raster?
+      ::JSON.parse(self.stats).select{ |item| item['type'] == '.tif' }.length > 0
+    end
+
+    def final_state?
+      [STATE_COMPLETE, STATE_FAILURE, STATE_STUCK].include?(state)
+    end
+
+    private
+
+  end
+end

app/models/carto/data_import_constants.rb

@@ -0,0 +1,14 @@
+module Carto
+  module DataImportConstants
+    COLLISION_STRATEGY_SKIP = 'skip'.freeze
+    COLLISION_STRATEGY_OVERWRITE = 'overwrite'.freeze
+
+    VALID_COLLISION_STRATEGIES = [COLLISION_STRATEGY_SKIP, COLLISION_STRATEGY_OVERWRITE, nil].freeze
+
+    def validate_collision_strategy
+      unless VALID_COLLISION_STRATEGIES.include?(collision_strategy)
+        errors.add(:collision_strategy, "Invalid collision strategy")
+      end
+    end
+  end
+end

app/models/carto/external_data_import.rb

@@ -0,0 +1,15 @@
+require 'active_record'
+
+module Carto
+  class ExternalDataImport < ActiveRecord::Base
+    belongs_to :data_import, class_name: Carto::DataImport
+    belongs_to :external_source, class_name: Carto::ExternalSource
+    belongs_to :synchronization, class_name: Carto::Synchronization, dependent: :destroy
+
+    def self.by_user_id(user_id)
+      user_data_imports = Carto::DataImport.where(user_id: user_id)
+      Carto::ExternalDataImport.where(data_import_id: user_data_imports.select(:id)).all
+    end
+
+  end
+end

app/models/carto/external_source.rb

@@ -0,0 +1,34 @@
+require 'active_record'
+
+class Carto::ExternalSource < ActiveRecord::Base
+  # Seconds
+  REFRESH_INTERVAL = 30*24*60*60
+
+  has_many :external_data_imports, class_name: Carto::ExternalDataImport, dependent: :destroy
+  belongs_to :visualization, inverse_of: :external_source
+  validates :visualization, :import_url, :rows_counted, :size, presence: true
+
+  def update_data(import_url, geometry_types, rows_counted, size, username = nil)
+    self.import_url = import_url
+    # This check avoids a "false change" if the data is actually equal (AR bug on arrays)
+    self.geometry_types = geometry_types unless equal_pg_array?(self.geometry_types, geometry_types)
+    self.rows_counted = rows_counted
+    self.size = size
+    self.username = username
+    self
+  end
+
+  def importable_by?(user)
+    user.present? && visualization.user_id == user.id
+  end
+
+  private
+
+  # True if array (`['ST_MultiPolygon']`) contains the same elements than string, in PG notation (`{ST_MultiPolygon}`)
+  def equal_pg_array?(array, string)
+    array = [] if array.nil?
+    string = '{}' if string.nil?
+
+    "{#{array.join(',')}}" == string
+  end
+end

app/models/carto/feature_flag.rb

@@ -0,0 +1,13 @@
+module Carto
+  class FeatureFlag < ActiveRecord::Base
+
+    validates :name, presence: true
+
+    has_many :feature_flags_user, dependent: :destroy
+
+    def self.find_by_user(user)
+      FeatureFlag.where(restricted: false) + user.feature_flags.select(&:restricted)
+    end
+
+  end
+end

app/models/carto/feature_flags_user.rb

@@ -0,0 +1,10 @@
+module Carto
+
+  class FeatureFlagsUser < ActiveRecord::Base
+    
+    belongs_to :feature_flag, foreign_key: :feature_flag_id
+    belongs_to :user, foreign_key: :user_id
+
+  end
+
+end

app/models/carto/geocoding.rb

@@ -0,0 +1,50 @@
+require 'active_record'
+require_relative '../../../services/table-geocoder/lib/exceptions'
+
+module Carto
+  class Geocoding < ActiveRecord::Base
+
+    PUBLIC_ATTRIBUTES = [:id, :table_id, :table_name, :state, :kind, :country_code, :region_code, :formatter,
+                        :geocoder_type, :geometry_type, :error, :processed_rows, :cache_hits, :processable_rows,
+                        :real_rows, :price, :used_credits, :remaining_quota, :country_column, :region_column,
+                        :data_import_id, :error_code]
+
+    def self.processable_rows(table_service, force_all_rows=false)
+      dataset = table_service.owner.in_database.select.from(table_service.sequel_qualified_table_name)
+      if !force_all_rows && dataset.columns.include?(:cartodb_georef_status)
+        dataset = dataset.exclude(cartodb_georef_status: true)
+      end
+      dataset.count
+    end
+
+    belongs_to :user
+    belongs_to :automatic_geocoding
+
+    def public_values
+      Hash[PUBLIC_ATTRIBUTES.map{ |k| [k, (self.send(k) rescue self[k].to_s)] }]
+    end
+
+    def error
+      additional_info = Carto::GeocoderErrors.additional_info(error_code)
+      if additional_info
+        { title: additional_info.title, description: additional_info.what_about }
+      else
+        { title: 'Geocoding error', description: '' }
+      end
+    end
+
+    def price
+      return 0 unless used_credits.to_i > 0
+      (user.geocoding_block_price * used_credits) / Carto::User::GEOCODING_BLOCK_SIZE.to_f
+    end
+
+    def remaining_quota
+      user.remaining_geocoding_quota
+    end
+
+    # TODO: Properly migrate log to AR and remove this
+    def log
+      CartoDB::Log[log_id]
+    end
+  end
+end

app/models/carto/grantable.rb

@@ -0,0 +1,15 @@
+module Carto
+  class Grantable
+
+    attr_reader :id, :name, :type, :avatar_url
+
+    def initialize(attributes)
+      h = attributes.symbolize_keys
+      @id = h[:id]
+      @name = h[:name]
+      @type = h[:type]
+      @avatar_url = h[:avatar_url]
+    end
+
+  end
+end

app/models/carto/group.rb

@@ -0,0 +1,203 @@
+require 'active_record'
+require_dependency 'cartodb/errors'
+require_dependency 'carto/helpers/auth_token_generator'
+require_relative 'paged_model'
+
+module Carto
+
+  # Groups are created by the editor because of extension requests, so
+  # standard Rails operations (creation, destruction, etc) doesn't trigger
+  # extension management functions. In order to keep extension and database
+  # in sync, there're several methods that do trigger it:
+  # - create_group_with_extension
+  # - rename_group_with_extension
+  # - destroy_group_with_extension
+  # - add_users_with_extension
+  # - remove_users_with_extension
+  class Group < ActiveRecord::Base
+    include PagedModel
+    include AuthTokenGenerator
+
+    belongs_to :organization, class_name: Carto::Organization
+    has_many :users_group, dependent: :destroy, class_name: Carto::UsersGroup
+    has_many :users, through: :users_group
+
+    private_class_method :new
+
+    validates :name, :database_role, :organization, presence: true
+
+    # In order to avoid locks between CDB_Organization_Remove_Organization_Access_Permission and CDB_Group_DropGroup
+    # the "shared with" deletion must be performed outside the transaction, on deletion.
+    after_commit :destroy_shared_with
+
+    # Constructor for groups already existing in the database
+    def self.new_instance(database_name, name, database_role, display_name = name)
+      organization = Organization.find_by_database_name(database_name)
+
+      raise "Organization not found for database #{database_name}" unless organization
+
+      raise CartoDB::ModelAlreadyExistsError if Group.find_by_organization_id_and_name_and_database_role(organization.id, name, database_role)
+
+      new(name: name, database_role: database_role, display_name: display_name, organization: organization)
+    end
+
+    # Creation of brand-new group with the extension
+    def self.create_group_with_extension(organization, display_name)
+      name = valid_group_name(display_name)
+      organization.owner.in_database do |conn|
+        create_group_extension_query(conn, name)
+      end
+      # Extension triggers a request to the editor databases endpoint which actually creates the group
+      group = Carto::Group.find_by_organization_id_and_name(organization.id, name)
+      raise "Group was not created by the extension. Is it installed and configured?" if group.nil?
+
+      group.display_name = display_name
+      group.save
+      group
+    end
+
+    # Constructor for groups metadata, ignores database roles. Should be generally avoided.
+    def self.new_instance_without_validation(name:, database_role:, display_name: name, auth_token: nil)
+      new(
+        name: name,
+        display_name: display_name,
+        database_role: database_role,
+        auth_token: auth_token
+      )
+    end
+
+    def rename_group_with_extension(new_display_name)
+      raise CartoDB::ModelAlreadyExistsError if Group.find_by_organization_id_and_display_name(organization.id, new_display_name)
+
+      new_name = Carto::Group.valid_group_name(new_display_name)
+      organization.owner.in_database do |conn|
+        Carto::Group.rename_group_extension_query(conn, name, new_name)
+      end
+      reload
+      self.display_name = new_display_name
+      save
+    end
+
+    # INFO: public because it's called by Organization.
+    def destroy_group_with_extension
+      # INFO: currently only a superuser can destroy a group. See CartoDB/cartodb-postgresql#114
+      organization.owner.in_database(as: :superuser) do |conn|
+        Carto::Group.destroy_group_extension_query(conn, name)
+      end
+    end
+
+    def add_users_with_extension(users)
+      organization.owner.in_database do |conn|
+        Carto::Group.add_users_group_extension_query(conn, name, users.collect(&:username))
+      end
+      reload
+    end
+
+    def remove_users_with_extension(users)
+      organization.owner.in_database do |conn|
+        Carto::Group.remove_users_group_extension_query(conn, name, users.collect(&:username))
+      end
+      reload
+    end
+
+    def grant_db_permission(table, access)
+      table.owner.in_database do |conn|
+        case access
+        when CartoDB::Permission::ACCESS_NONE
+          Carto::Group.revoke_all(conn, name, table.database_schema, table.name)
+        when CartoDB::Permission::ACCESS_READONLY
+          Carto::Group.grant_read(conn, name, table.database_schema, table.name)
+        when CartoDB::Permission::ACCESS_READWRITE
+          Carto::Group.grant_write(conn, name, table.database_schema, table.name)
+        else
+          raise "Unknown access: #{access}"
+        end
+      end
+    end
+
+    def rename(new_name, new_database_role)
+      self.name = new_name
+      self.database_role = new_database_role
+    end
+
+    def add_user(username)
+      user = Carto::User.find_by_username(username)
+
+      raise "User #{username} not found" unless user
+
+      raise CartoDB::ModelAlreadyExistsError unless users_group.where(user_id: user.id, group_id: id).first.nil?
+
+      user_group = Carto::UsersGroup.new(user: user, group: self)
+      users_group << user_group
+      user_group
+    end
+
+    def remove_user(username)
+      user = Carto::User.find_by_username(username)
+
+      raise "User #{username} not found" unless user
+
+      if users.include?(user)
+        users.destroy(user)
+        user
+      end
+    end
+
+    def database_name
+      organization.database_name
+    end
+
+    private
+
+    def destroy_shared_with
+      if transaction_include_any_action?([:destroy])
+        Carto::SharedEntity.where(recipient_id: id).each do |se|
+          viz = Carto::Visualization.find(se.entity_id)
+          permission = viz.permission
+          permission.remove_group_permission(self)
+          permission.save
+        end
+      end
+    end
+
+    # TODO: PG Format("%I", strvar); ?
+    def self.valid_group_name(display_name)
+      name = display_name.squish
+      name = "g_#{name}" unless name[/^[a-zA-Z_]{1}/]
+      name.gsub(/[^a-zA-Z0-9_ ]/, '_').gsub(/_{2,}/, '_')
+    end
+
+    def self.create_group_extension_query(conn, name)
+      conn.execute(%{ select cartodb.CDB_Group_CreateGroup('#{name}') })
+    end
+
+    def self.rename_group_extension_query(conn, name, new_name)
+      conn.execute(%{ select cartodb.CDB_Group_RenameGroup('#{name}', '#{new_name}') })
+    end
+
+    def self.destroy_group_extension_query(conn, name)
+      conn.execute(%{ select cartodb.CDB_Group_DropGroup('#{name}') })
+    end
+
+    def self.add_users_group_extension_query(conn, name, usernames)
+      conn.execute(%{ select cartodb.CDB_Group_AddUsers('#{name}', ARRAY['#{usernames.join("','")}']) })
+    end
+
+    def self.remove_users_group_extension_query(conn, name, usernames)
+      conn.execute(%{ select cartodb.CDB_Group_RemoveUsers('#{name}', ARRAY['#{usernames.join("','")}']) })
+    end
+
+    def self.revoke_all(conn, group_name, table_database_schema, table_name)
+      conn.execute(%{ select cartodb._CDB_Group_Table_RevokeAll('#{group_name}', '#{table_database_schema}', '#{table_name}', false) })
+    end
+
+    def self.grant_read(conn, group_name, table_database_schema, table_name)
+      conn.execute(%{ select cartodb._CDB_Group_Table_GrantRead('#{group_name}', '#{table_database_schema}', '#{table_name}', false) })
+    end
+
+    def self.grant_write(conn, group_name, table_database_schema, table_name)
+      conn.execute(%{ select cartodb._CDB_Group_Table_GrantReadWrite('#{group_name}', '#{table_database_schema}', '#{table_name}', false) })
+    end
+
+  end
+end

app/models/carto/helpers/auth_token_generator.rb

@@ -0,0 +1,19 @@
+module Carto
+  module AuthTokenGenerator
+    def get_auth_token
+      auth_token || generate_and_save_auth_token
+    end
+
+    private
+
+    def generate_and_save_auth_token
+      self.auth_token = generate_auth_token
+      save
+      auth_token
+    end
+
+    def generate_auth_token
+      SecureRandom.urlsafe_base64(nil, false)
+    end
+  end
+end

app/models/carto/helpers/batch_queries_statement_timeout.rb

@@ -0,0 +1,30 @@
+module Carto
+  module BatchQueriesStatementTimeout
+    def batch_queries_statement_timeout
+      if !defined?(@batch_queries_statement_timeout)
+        timeout_str = $users_metadata.HMGET(batch_limits_key, 'timeout').first
+        @batch_queries_statement_timeout = timeout_str.nil? ? nil : timeout_str.to_i
+      end
+      @batch_queries_statement_timeout
+    end
+
+    def batch_queries_statement_timeout=(timeout_ms)
+      unless !timeout_ms.present? || timeout_ms.to_i > 0
+        raise 'batch_queries_statement_timeout must be nil or an integer greater than 0'
+      end
+      if !timeout_ms.present?
+        # NOTE: nil is cast to the empty string in redis (""), which can be cast to 0 in
+        # the SQL API. In order to prevent undesired behavior, just delete this key.
+        @batch_queries_statement_timeout = nil
+        $users_metadata.HDEL batch_limits_key, 'timeout'
+      else
+        @batch_queries_statement_timeout = timeout_ms.to_i
+        $users_metadata.HMSET batch_limits_key, 'timeout', @batch_queries_statement_timeout
+      end
+    end
+
+    def batch_limits_key
+      "limits:batch:#{username}"
+    end
+  end
+end

app/models/carto/helpers/billing_cycle.rb

@@ -0,0 +1,27 @@
+require 'active_support/time'
+
+module Carto
+  module BillingCycle
+    def last_billing_cycle
+      day = period_end_date.day
+      date = day > Date.today.day ? (Date.today - 1.month) : Date.today
+      begin
+        Date.new(date.year, date.month, day)
+      rescue ArgumentError
+        day = day - 1
+        retry
+      end
+    end
+
+    def next_billing_cycle
+      day = period_end_date.day
+      date = day > Date.today.day ? Date.today : Date.today + 1.month
+      begin
+        Date.new(date.year, date.month, day)
+      rescue ArgumentError
+        day = day - 1
+        retry
+      end
+    end
+  end
+end

app/models/carto/helpers/has_connector_configuration.rb

@@ -0,0 +1,8 @@
+module Carto
+  module HasConnectorConfiguration
+    def connector_configuration(provider_name)
+      provider = ConnectorProvider.find_by_name(provider_name)
+      ConnectorConfiguration.for_user(self, provider)
+    end
+  end
+end

app/models/carto/helpers/url_validator.rb

@@ -0,0 +1,10 @@
+class UrlValidator < ActiveModel::EachValidator
+  def validate_each(record, attribute, value)
+    record.errors[attribute] << (options[:message] || "must be a valid URL") unless url_valid?(value)
+  end
+
+  def url_valid?(url)
+    url = URI.parse(url) rescue false
+    url.kind_of?(URI::HTTP) || url.kind_of?(URI::HTTPS)
+  end
+end

app/models/carto/invitation.rb

@@ -0,0 +1,98 @@
+require 'active_record'
+require 'cartodb-common'
+require_dependency 'cartodb/errors'
+require_dependency 'carto/user_authenticator'
+
+module Carto
+  class Invitation < ActiveRecord::Base
+
+    # Because of an activerecord-postgresql-array bug that makes array
+    # insertions unusable we can't set _users_emails mandatory on construction,
+    # so we create a creator enforcing desired behaviour.
+    # This will be fixed when we upgrade Ruby and Rails
+    # validates :users_emails, :welcome_text, presence: true
+    validates :inviter_user, :organization, :welcome_text, presence: true
+    validates :users_emails, email: true
+    validate :users_emails_not_taken
+
+    belongs_to :inviter_user, class_name: Carto::User
+    belongs_to :organization
+
+    private_class_method :new
+
+    def self.create_new(inviter_user, users_emails, welcome_text, viewer)
+      raise CartoDB::InvalidUser.new("Only admins can create invitations") unless inviter_user.organization_admin?
+
+      # ActiveRecord validation for all values
+      invitation = new(inviter_user: inviter_user,
+                       organization: inviter_user.organization,
+                       users_emails: users_emails,
+                       welcome_text: welcome_text,
+                       viewer: viewer)
+      return invitation unless invitation.valid?
+
+      # Two-step creation workarounding array bug
+      invitation = new(inviter_user: inviter_user,
+                       organization: inviter_user.organization,
+                       welcome_text: welcome_text,
+                       viewer: viewer)
+
+      invitation.seed = Carto::Common::EncryptionService.make_token
+      if invitation.save
+        invitation.reload
+        invitation.users_emails = users_emails
+        invitation.used_emails = []
+        invitation.save
+
+        ::Resque.enqueue(::Resque::OrganizationJobs::Mail::Invitation, invitation.id)
+      end
+      invitation
+    end
+
+    def self.query_with_valid_email(email)
+      Carto::Invitation.where('? = ANY(users_emails)', email)
+    end
+
+    def self.query_with_unused_email(email)
+      query_with_valid_email(email).where('? != ALL(used_emails)', email)
+    end
+
+    def token(email)
+      Carto::Common::EncryptionService.encrypt(sha_class: Digest::SHA256, password: email, salt: seed)
+    end
+
+    def use(email, token)
+      # reload and used_emails assignment is needed because otherwise
+      # activerecord-postgresql-array won't update the invitations
+      reload
+      if users_emails.include?(email) && verify_token(token, email)
+        if used_emails.include?(email)
+          raise AlreadyUsedInvitationError.new("#{email} has already used the invitation")
+        else
+          self.used_emails = used_emails.push(email)
+          save
+          true
+        end
+      else
+        false
+      end
+    end
+
+    private
+
+    def verify_token(token, email)
+      Carto::Common::EncryptionService.verify(password: email, secure_password: token, salt: seed)
+    end
+
+    def users_emails_not_taken
+      return unless users_emails
+
+      users_emails.each do |email|
+        errors[:users_emails] << "Existing user for #{email}" if Carto::User.find_by_email(email)
+      end
+    end
+
+  end
+end
+
+class AlreadyUsedInvitationError < StandardError; end

app/models/carto/layer.rb

@@ -0,0 +1,372 @@
+require 'active_record'
+require 'active_record/connection_adapters/postgresql/oid/json'
+require_relative './carto_json_serializer'
+require_dependency 'carto/table_utils'
+require_dependency 'carto/query_rewriter'
+
+module Carto
+  module LayerTableDependencies
+    private
+
+    def affected_tables
+      return [] unless maps.first.present? && options.present?
+      node_id = options.symbolize_keys[:source]
+      if node_id.present?
+        visualization_id = map.visualization.id
+        node = AnalysisNode.find_by_natural_id(visualization_id, node_id)
+        return [] unless node
+        dependencies = node.source_descendants.map do |source_node|
+          tables_by_query = tables_from_query(source_node.params[:query])
+          table_name = source_node.options[:table_name]
+          tables_by_name = table_name ? tables_from_names([table_name], user) : []
+
+          tables_by_query.present? ? tables_by_query : tables_by_name
+        end
+        dependencies.flatten.compact.uniq
+      else
+        tables_by_query = tables_from_query_option
+        dependencies = tables_by_query.present? ? tables_by_query : tables_from_table_name_option
+        dependencies.compact.uniq
+      end
+    end
+
+    def tables_from_query_option
+      tables_from_query(query)
+    end
+
+    def tables_from_query(query)
+      query.present? ? tables_from_names(affected_table_names(query), user) : []
+    rescue => e
+      # INFO: this covers changes that CartoDB can't track, so we must handle it gracefully.
+      # For example, if layer SQL contains wrong SQL (uses a table that doesn't exist, or uses an invalid operator).
+      # This warning level is checked in tests to ensure that embed view does not need user DB connection,
+      # so we need to keep it (or change the tests accordingly)
+      CartoDB::Logger.warning(message: 'Could not retrieve tables from query', exception: e, user: user, layer: self)
+      []
+    end
+
+    def tables_from_table_name_option
+      return [] if options.empty?
+      sym_options = options.symbolize_keys
+      user_name = sym_options[:user_name]
+      table_name = sym_options[:table_name]
+      schema_prefix = user_name.present? && table_name.present? && !table_name.include?('.') ? %{"#{user_name}".} : ''
+      tables_from_names(["#{schema_prefix}#{table_name}"], user)
+    end
+  end
+
+  class Layer < ActiveRecord::Base
+    include Carto::TableUtils
+    include LayerTableDependencies
+    include Carto::QueryRewriter
+
+    attribute :options, ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Json.new
+    attribute :infowindow, ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Json.new
+    attribute :tooltip, ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Json.new
+
+    serialize :options, CartoJsonSerializer
+    serialize :infowindow, CartoJsonSerializer
+    serialize :tooltip, CartoJsonSerializer
+
+    has_many :layers_maps, dependent: :destroy
+    has_many :maps, through: :layers_maps, after_add: :after_added_to_map
+
+    has_many :layers_user, dependent: :destroy
+    has_many :users, through: :layers_user, after_add: :set_default_order
+
+    has_many :layers_user_tables, dependent: :destroy
+    has_many :user_tables, through: :layers_user_tables, class_name: Carto::UserTable
+
+    has_many :widgets, -> { order(:order) }, class_name: Carto::Widget
+    has_many :legends, -> { order(:created_at) }, class_name: Carto::Legend, dependent: :destroy
+
+    has_many :layer_node_styles
+
+    before_destroy :ensure_not_viewer
+    before_save :lock_user_tables
+    after_save :invalidate_maps, :update_layer_node_style
+    after_save :register_table_dependencies, if: :data_layer?
+
+    ALLOWED_KINDS = %w{carto tiled background gmapsbase torque wms}.freeze
+    validates :kind, inclusion: { in: ALLOWED_KINDS }
+    validate :validate_not_viewer
+
+    TEMPLATES_MAP = {
+      'table/views/infowindow_light' =>               'infowindow_light',
+      'table/views/infowindow_dark' =>                'infowindow_dark',
+      'table/views/infowindow_light_header_blue' =>   'infowindow_light_header_blue',
+      'table/views/infowindow_light_header_yellow' => 'infowindow_light_header_yellow',
+      'table/views/infowindow_light_header_orange' => 'infowindow_light_header_orange',
+      'table/views/infowindow_light_header_green' =>  'infowindow_light_header_green',
+      'table/views/infowindow_header_with_image' =>   'infowindow_header_with_image'
+    }.freeze
+
+    def set_default_order(parent)
+      # Reload maps upon adding this layer to a map (AR doesn't do this automatically)
+      maps.reload if persisted?
+
+      return unless order.nil?
+      max_order = parent.layers.map(&:order).compact.max || -1
+      self.order = max_order + 1
+      save if persisted?
+    end
+
+    # Sequel model compatibility (for TableBlender)
+    # TODO: Remove this after `::UserTable` deletion, and inline into TableBlender
+    def add_map(map)
+      map.layers << self
+    end
+
+    def user_tables_readable_by(user)
+      user_tables.select { |ut| ut.readable_by?(user) }
+    end
+
+    def data_readable_by?(user)
+      user_tables.all? { |ut| ut.readable_by?(user) }
+    end
+
+    def legend
+      @legend ||= options['legend']
+    end
+
+    def qualified_table_name(schema_owner_user = nil)
+      table_name = options['table_name']
+      if table_name.present? && table_name.include?('.')
+        table_name
+      else
+        schema_prefix = schema_owner_user.nil? ? '' : "#{schema_owner_user.sql_safe_database_schema}."
+        "#{schema_prefix}#{safe_table_name_quoting(options['table_name'])}"
+      end
+    end
+
+    # INFO: for vizjson v3 this is not used, see VizJSON3LayerPresenter#to_vizjson_v3
+    def infowindow_template_path
+      if infowindow.present? && infowindow['template_name'].present?
+        template_name = TEMPLATES_MAP.fetch(infowindow['template_name'], infowindow['template_name'])
+        Rails.root.join("lib/assets/javascripts/cartodb/table/views/infowindow/templates/#{template_name}.jst.mustache")
+      end
+    end
+
+    # INFO: for vizjson v3 this is not used, see VizJSON3LayerPresenter#to_vizjson_v3
+    def tooltip_template_path
+      if tooltip.present? && tooltip['template_name'].present?
+        template_name = TEMPLATES_MAP.fetch(tooltip['template_name'], tooltip['template_name'])
+        Rails.root.join("lib/assets/javascripts/cartodb/table/views/tooltip/templates/#{template_name}.jst.mustache")
+      end
+    end
+
+    def basemap?
+      gmapsbase? || tiled?
+    end
+
+    def base_layer?
+      tiled? || background? || gmapsbase? || wms?
+    end
+
+    def torque?
+      kind == 'torque'
+    end
+
+    def data_layer?
+      !base_layer?
+    end
+
+    def user_layer?
+      tiled? || background? || gmapsbase? || wms?
+    end
+
+    def named_map_layer?
+      tiled? || background? || gmapsbase? || wms? || carto?
+    end
+
+    def carto?
+      kind == 'carto'
+    end
+
+    def tiled?
+      kind == 'tiled'
+    end
+
+    def background?
+      kind == 'background'
+    end
+
+    def gmapsbase?
+      kind == 'gmapsbase'
+    end
+
+    def wms?
+      kind == 'wms'
+    end
+
+    def supports_labels_layer?
+      basemap? && options["labels"] && options["labels"]["urlTemplate"]
+    end
+
+    def map
+      maps[0]
+    end
+
+    def visualization
+      map.visualization if map
+    end
+
+    def user
+      @user ||= map.nil? ? nil : map.user
+    end
+
+    def default_query(user = nil)
+      sym_options = options.symbolize_keys
+      query = sym_options[:query]
+
+      if query.present?
+        query
+      else
+        user_username = user.nil? ? nil : user.username
+        user_name = sym_options[:user_name] || user_username
+        table_name = sym_options[:table_name]
+        qualify = (user && user.organization_user?) || user_username != user_name
+
+        if table_name.present? && !table_name.include?('.') && user_name.present? && qualify
+          "SELECT * FROM #{safe_table_name_quoting(user_name)}.#{safe_table_name_quoting(table_name)}"
+        else
+          "SELECT * FROM #{qualified_table_name}"
+        end
+      end
+    end
+
+    def register_table_dependencies
+      if data_layer?
+        if persisted?
+          user_tables.reload
+          maps.reload
+        end
+        self.user_tables = affected_tables
+      end
+    end
+
+    def fix_layer_user_information(old_username, new_user, renamed_tables)
+      new_username = new_user.username
+
+      if options.key?(:user_name)
+        old_username = options[:user_name] || old_username
+        options[:user_name] = new_username
+      end
+
+      if options.key?(:table_name)
+        old_table_name = options[:table_name]
+        options[:table_name] = renamed_tables.fetch(old_table_name, old_table_name)
+      end
+
+      # query_history is not modified as a safety measure for cases where this naive replacement doesn't work
+      query = options[:query]
+      options[:query] = rewrite_query(query, old_username, new_user, renamed_tables) if query.present?
+    end
+
+    def force_notify_change
+      map.force_notify_map_change if map
+    end
+
+    def custom?
+      CUSTOM_CATEGORIES.include?(category)
+    end
+
+    def category
+      options && options['category']
+    end
+
+    def rename_table(current_table_name, new_table_name)
+      return self unless data_layer?
+      target_keys = %w{table_name tile_style query}
+
+      targets = options.select { |key, _| target_keys.include?(key) }
+      renamed = targets.map do |key, value|
+        [key, rename_in(value, current_table_name, new_table_name)]
+      end
+
+      self.options = options.merge(renamed.to_h)
+      self
+    end
+
+    def uses_private_tables?
+      user_tables.any?(&:private?)
+    end
+
+    def after_added_to_map(map)
+      # This avoids unnecessary operations for in-memory logic. Example: Mapcap recreation. See #12473.
+      return unless map.persisted?
+
+      set_default_order(map)
+      register_table_dependencies
+    end
+
+    def depends_on?(user_table)
+      layers_user_tables.map(&:user_table_id).include?(user_table.id)
+    end
+
+    def source_id
+      options.symbolize_keys[:source]
+    end
+
+    def qualify_for_organization(owner_username)
+      options['query'] = qualify_query(query, options['table_name'], owner_username) if query
+    end
+
+    private
+
+    # The table dependencies will only be updated after the layer. However, when deleting them, they need to be deleted
+    # before the model. This can cause deadlocks with simultaneous request to update and delete the model.
+    # This request a explicit lock to PostgreSQL so the tables are always accessed in the same order. #11443
+    def lock_user_tables
+      user_tables.lock.all if persisted?
+    end
+
+    def rename_in(target, anchor, substitution)
+      return if target.blank?
+      regex = /(\A|\W+)(#{anchor})(\W+|\z)/
+      target.gsub(regex) { |match| match.gsub(anchor, substitution) }
+    end
+
+    CUSTOM_CATEGORIES = %w{Custom NASA TileJSON Mapbox WMS}.freeze
+
+    def tables_from_names(table_names, user)
+      ::Table.get_all_user_tables_by_names(table_names, user)
+    end
+
+    def affected_table_names(query)
+      return [] unless query.present?
+
+      query_tables = user.in_database.execute("SELECT unnest(CDB_QueryTablesText(#{user.in_database.quote(query)}))")
+      query_tables.column_values(0).uniq
+    end
+
+    def query
+      options.symbolize_keys[:query]
+    end
+
+    def invalidate_maps
+      maps.each(&:notify_map_change)
+    end
+
+    def ensure_not_viewer
+      raise CartoDB::InvalidMember.new(user: "Viewer users can't destroy layers") if user && user.viewer
+    end
+
+    def validate_not_viewer
+      errors.add(:maps, "Viewer users can't edit layers") if maps.any? { |m| m.user && m.user.viewer }
+    end
+
+    def update_layer_node_style
+      style = current_layer_node_style
+      if style
+        style.update_from_layer(self)
+        style.save
+      end
+    end
+
+    def current_layer_node_style
+      return nil unless source_id
+      layer_node_styles.where(source_id: source_id).first || LayerNodeStyle.new(layer: self, source_id: source_id)
+    end
+  end
+end

app/models/carto/layer_node_style.rb

@@ -0,0 +1,23 @@
+require_dependency 'carto/carto_json_serializer'
+
+module Carto
+  class LayerNodeStyle < ActiveRecord::Base
+    belongs_to :layer
+
+    serialize :options, CartoJsonSymbolizerSerializer
+    serialize :infowindow, CartoJsonSymbolizerSerializer
+    serialize :tooltip, CartoJsonSymbolizerSerializer
+
+    OPTIONS_TO_COPY = [:sql_wrap, :style_properties, :tile_style].freeze
+
+    def self.from_visualization_and_source(visualization, source_id)
+      Carto::LayerNodeStyle.where(layer_id: visualization.data_layers.map(&:id), source_id: source_id)
+    end
+
+    def update_from_layer(layer)
+      self.infowindow = layer.infowindow || {}
+      self.tooltip = layer.tooltip || {}
+      self.options = layer.options.symbolize_keys.slice(*OPTIONS_TO_COPY)
+    end
+  end
+end

app/models/carto/layers_map.rb

@@ -0,0 +1,37 @@
+require 'active_record'
+require_dependency 'carto/visualization_backup_service'
+
+module Carto
+  class LayersMap < ActiveRecord::Base
+    include Carto::VisualizationBackupService
+
+    belongs_to :layer, class_name: Carto::Layer
+    belongs_to :map, class_name: Carto::Map
+
+    before_destroy :backup_visualization
+    after_destroy :invalidate_map, :destroy_layer
+
+    private
+
+    def invalidate_map
+      map.notify_map_change if map
+    end
+
+    def destroy_layer
+      # Destroy the actual layer when removing it from the map (as layers cannot be in multiple maps)
+      # This is a workaround for a bug in Rails 4 (https://github.com/rails/rails/issues/13609)
+      # A better way to do this is with `dependent: :destroy` on the association
+      return if @layer_destroyed || layer.destroyed?
+      @layer_destroyed = true
+      layer.destroy
+    end
+
+    def backup_visualization
+      return if !map.visualization || map.visualization.destroyed? || @layer_destroyed || layer.destroyed?
+      create_visualization_backup(
+        visualization: map.visualization,
+        category: Carto::VisualizationBackup::CATEGORY_LAYER
+      )
+    end
+  end
+end

app/models/carto/layers_user.rb

@@ -0,0 +1,6 @@
+module Carto
+  class LayersUser < ActiveRecord::Base
+    belongs_to :layer
+    belongs_to :user
+  end
+end

app/models/carto/layers_user_table.rb

@@ -0,0 +1,10 @@
+require 'active_record'
+
+module Carto
+  class LayersUserTable < ActiveRecord::Base
+
+    belongs_to :layer, class_name: Carto::Layer
+    belongs_to :user_table, class_name: Carto::UserTable
+
+  end
+end

app/models/carto/ldap/configuration.rb

@@ -0,0 +1,205 @@
+# See http://www.rubydoc.info/gems/net-ldap/0.11
+require 'net/ldap'
+
+class Carto::Ldap::Configuration < ActiveRecord::Base
+
+  # Not encrypted
+  ENCRYPTION_NONE = nil
+  # Encrypted from start
+  ENCRYPTION_SIMPLE_TLS = 'simple_tls'
+  # Upgrade to encrypted once connected
+  ENCRYPTION_START_TLS = 'start_tls'
+
+  ENCRYPTION_SSL_VERSION_DEFAULT = nil
+  ENCRYPTION_SSL_VERSION_TLSV1_1 = 'TLSv1_1'
+
+  DOMAIN_BASES_SEPARATOR = '||'
+
+  self.table_name = 'ldap_configurations'
+
+  belongs_to :organization, class_name: Carto::Organization
+
+  # @param Uuid id  (Self-generated)
+  # @param Uuid organization_id
+  # @param String host LDAP host or ip address
+  # @param Int port LDAP port e.g. 389, 636 (LDAPS)
+  # @param String encryption (Optional) Encryption type to use. Empty means standard/simple Auth
+  # @param String ca_file UNUSED FOR NOW -  (Optional) Certificate file path for start_tls encryption.
+  #                       Example: "/etc/cafile.pem"
+  # @param String ssl_version For start_tls_encryption. Example: "TLSv1_1"
+  # @param String connection_user Full CN for "search connections" to LDAP: `CN=admin, DC=cartodb, DC=COM`
+  # @param String connection_password Password for "search connections" to LDAP
+  # @param String user_id_field Which LDAP entry field represents the user id. e.g. `sAMAccountName`, `uid`
+  # @param String username_field Which LDAP entry field represents the username that will be mapped to cartodb.
+  #                              For now, same as user_id_field
+  # @param String email_field Which LDAP entry field represents the email
+  # @param String domain_bases List of DCs conforming the path.
+  #                            Serialized, e.g. "['a','b']", due to Rails 3 or PG gem issue handling `PG text[]` fields
+  # @param String additional_search_filter Additional filter to add (with &) to the search query if present
+  # @param String user_object_class Name of the attribute where the sers are maped in LDAP
+  # @param String group_object_class Name of the attribute where the groups are maped in LDAP
+  # @param DateTime created_at (Self-generated)
+  # @param DateTime updated_at (Self-generated)
+
+  attr_readonly :user_id_field
+
+  validates :organization, :host, :port, :connection_user, :connection_password, :user_id_field, :username_field,
+            :email_field, :user_object_class, :group_object_class, presence: true
+
+  validates :ca_file, length: { minimum: 0, allow_nil: true }
+
+  validates :encryption,  inclusion: { in: [ENCRYPTION_SIMPLE_TLS, ENCRYPTION_START_TLS], allow_nil: true }
+  validates :ssl_version, inclusion: { in: [ENCRYPTION_SSL_VERSION_TLSV1_1], allow_nil: true }
+  validate :domain_bases_not_empty
+
+  def domain_bases_list
+    domain_bases.split(DOMAIN_BASES_SEPARATOR) if domain_bases
+  end
+
+  def domain_bases_list=(list)
+    self.domain_bases = list.join(DOMAIN_BASES_SEPARATOR)
+  end
+
+  # Returns matching Carto::Ldap::Entry or false if credentials are wrong
+  # @param String username. No full CN, just the username, e.g. 'administrator1'
+  # @param String password
+  def authenticate(username, password)
+    return false if username.blank? || password.blank?
+    ldap_connection = Net::LDAP.new(connect_timeout: CONNECTION_TIMEOUT)
+    ldap_connection.host = self.host
+    ldap_connection.port = self.port
+    configure_encryption(ldap_connection)
+
+    ldap_connection.auth self.connection_user, self.connection_password
+
+    valid_ldap_entry = nil
+    domain_bases_list.find do |domain|
+      valid_ldap_entry = ldap_connection.bind_as(
+        base: domain,
+        filter: search_filter(username),
+        password: password
+      )
+    end
+    @last_authentication_result = ldap_connection.get_operation_result
+    return false unless valid_ldap_entry
+
+    Carto::Ldap::Entry.new(valid_ldap_entry.first, self)
+  rescue Net::LDAP::Error => e
+    CartoDB::Logger.error(exception: e, message: 'Error authenticating against LDAP', username: username)
+  end
+
+  # INFO: Resets connection if already made
+  def test_connection
+    result = ldap_connection(true).bind
+    if result
+      { success: true, connection: result }
+    else
+      { success: false, error: last_operation_result.to_hash }
+    end
+  rescue => exception
+    { success: false, error: { message: exception.message } }
+  end
+
+  def users(objectClass = self.user_object_class)
+    search_in_domain_bases(Net::LDAP::Filter.eq('objectClass', objectClass))
+  end
+
+  def groups(objectClass = self.group_object_class)
+    search_in_domain_bases(Net::LDAP::Filter.eq('objectClass', objectClass))
+  end
+
+  def last_authentication_result
+    @last_authentication_result.nil? ? nil : Carto::Ldap::OperationResult.new(
+      @last_authentication_result.code, @last_authentication_result.error_message,
+      @last_authentication_result.matched_dn, @last_authentication_result.message)
+  end
+
+  def last_operation_result
+    ldap_result = ldap_connection.get_operation_result
+    Carto::Ldap::OperationResult.new(ldap_result.code, ldap_result.error_message, ldap_result.matched_dn,
+      ldap_result.message)
+  end
+
+  private
+
+  CONNECTION_TIMEOUT = 8
+
+  def search_filter(username)
+    user_id_filter = "(#{user_id_field}=#{username})"
+    if additional_search_filter.present?
+      "(&#{user_id_filter}#{additional_search_filter})"
+    else
+      user_id_filter
+    end
+  end
+
+  def domain_bases_not_empty
+    errors.add(:domain_bases, "No domain bases set") unless domain_bases.present?
+    errors.add(:domain_bases_list, "Domain bases list empty") unless domain_bases_list.present?
+  end
+
+  def search_in_domain_bases(filter)
+    domain_bases_list.map { |domain|
+      search(domain, filter)
+    }.flatten.compact
+  end
+
+  # @param String base DC to search at
+  # @Param Net::LDAP::Filter filter (Optional)
+  def search(base, filter = nil)
+    if filter
+      ldap_connection.search(base: base, filter: filter)
+    else
+      ldap_connection.search(base: base)
+    end
+  end
+
+  # Performs connection always with the search connection user
+  def ldap_connection(reset = false)
+    @conn = nil if reset
+    @conn ||= connect
+  end
+
+  # Connect, by default with the search connection user
+  # @param String user full CN, like `CN=test_user, CN=developers, DC=cartodb, DC=COM`
+  # @param String password Connection password
+  # @throws InvalidConfigurationEncryptionError
+  def connect(user = self.connection_user, password = self.connection_password)
+    ldap = Net::LDAP.new(connect_timeout: CONNECTION_TIMEOUT)
+    ldap.host = self.host
+    ldap.port = self.port
+    configure_encryption(ldap)
+    # implicity this does basic/simple auth if no encryption added above
+    ldap.auth(user, password)
+    ldap
+  end
+
+  def configure_encryption(ldap)
+    tls_options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS
+
+    case self.encryption
+    when ENCRYPTION_NONE
+      return
+    when ENCRYPTION_START_TLS
+      tls_options.merge!(:ca_file => self.ca_file) if self.ca_file
+    when ENCRYPTION_SIMPLE_TLS
+      # No special value needed
+    else
+      raise InvalidConfigurationEncryptionError.new(self.encryption)
+    end
+
+    tls_options.merge!(:verify_mode => OpenSSL::SSL::VERIFY_NONE)
+
+    # Default value is "SSLv23" at the gem
+    tls_options.merge!(:ssl_version => self.ssl_version) if self.ssl_version
+
+    ldap.encryption(method: self.encryption.to_sym, tls_options: tls_options)
+  end
+
+end
+
+class InvalidConfigurationEncryptionError < StandardError
+  def initialize(incorrect_encryption_value)
+    super("Invalid encryption value supplied: #{incorrect_encryption_value}. Valid values: [nil, '', '']")
+  end
+end

app/models/carto/ldap/entry.rb

@@ -0,0 +1,38 @@
+# See http://www.rubydoc.info/gems/net-ldap/0.11
+require 'net/ldap'
+
+class Carto::Ldap::Entry
+
+  def initialize(ldap_entry, ldap_configuration)
+    @entry = ldap_entry
+    @configuration = ldap_configuration
+  end
+
+  def user_id
+    extract_field(@configuration.user_id_field)
+  end
+
+  def cartodb_user_id
+    Carto::Ldap::Manager.sanitize_for_cartodb(user_id)
+  end
+
+  def username
+    extract_field(@configuration.username_field)
+  end
+
+  def email
+    extract_field(@configuration.email_field)
+  end
+
+  def configuration
+    @configuration
+  end
+
+  private
+
+  def extract_field(field)
+    value = @entry[field]
+    value.nil? ? nil : value.first
+  end
+
+end

app/models/carto/ldap/manager.rb

@@ -0,0 +1,64 @@
+# See http://www.rubydoc.info/gems/net-ldap/0.11
+require 'net/ldap'
+
+module Carto
+  module Ldap
+
+    class Manager
+
+      # @throws LDAPUserNotPresentAtCartoDBError
+      def authenticate(username, password)
+        @last_authentication_result = nil
+        user = nil
+        ldap_entry = nil
+
+        Carto::Ldap::Configuration.all.each { |ldap|
+          ldap_entry ||= ldap.authenticate(username, password)
+          @last_authentication_result = ldap.last_authentication_result
+        }
+
+        if ldap_entry
+          user = ::User.where({
+              username: ldap_entry.cartodb_user_id,
+              organization_id: ldap_entry.configuration.organization_id
+            }).first
+
+          if user.nil?
+            raise LDAPUserNotPresentAtCartoDBError.new(ldap_entry.cartodb_user_id,
+              ldap_entry.configuration.organization_id, username, ldap_entry.email)
+          end
+        end
+
+        user
+      end
+
+      def configuration_present?
+        Carto::Ldap::Configuration.first != nil
+      end
+
+      def last_authentication_result
+        @last_authentication_result
+      end
+
+      def self.sanitize_for_cartodb(ldap_value)
+        ldap_value.to_s.downcase.gsub(/[^a-z0-9\-]/,'')
+      end
+    end
+
+
+    class LDAPUserNotPresentAtCartoDBError < StandardError
+
+      def initialize(cartodb_username, cartodb_organization_id, ldap_username, ldap_email='')
+        @ldap_username = ldap_username
+        @ldap_email = ldap_email
+        @cartodb_username = cartodb_username
+        @organization_id = cartodb_organization_id
+        super("'#{ldap_username}' not found at CARTO (username:'#{cartodb_username}', organization id:'#{cartodb_organization_id}')")
+      end
+
+      attr_reader :ldap_email, :ldap_username, :cartodb_username, :organization_id
+
+    end
+
+  end
+end

app/models/carto/ldap/operation_result.rb

@@ -0,0 +1,37 @@
+# See http://www.rubydoc.info/gems/net-ldap/0.11
+require 'net/ldap'
+
+module Carto
+  module Ldap
+    class OperationResult
+
+      CODE_UNKNOWN = -1
+
+      def initialize(code = CODE_UNKNOWN, error_message = "", matched_dn = "", message = "")
+        @code = code ||
+        @error_message = error_message
+        @matched_dn = matched_dn
+        @message = message
+      end
+
+      attr_reader :code, :message, :error_message, :matched_dn
+
+      def to_hash
+        {
+          code:           code,
+          message:        message,
+          error_message:  error_message,
+          matched_dn:     matched_dn
+        }
+      end
+
+      def success?
+        @code == 0
+      end
+
+      def failure?
+        !success?
+      end
+    end
+  end
+end

app/models/carto/legend.rb

@@ -0,0 +1,99 @@
+require_relative './carto_json_serializer'
+require_relative '../../controllers/carto/api/legend_presenter'
+require_dependency 'carto/legend_definition_validator'
+
+module Carto
+  class Legend < ActiveRecord::Base
+    self.inheritance_column = :_type
+
+    belongs_to :layer, class_name: Carto::Layer
+
+    VALID_LEGEND_TYPES = %w(category bubble choropleth custom custom_choropleth torque).freeze
+    LEGEND_TYPES_PER_ATTRIBUTE = {
+      color: %w(category choropleth custom custom_choropleth torque),
+      size: %w(bubble)
+    }.freeze
+
+    serialize :definition, ::Carto::CartoJsonSymbolizerSerializer
+    serialize :conf, ::Carto::CartoJsonSymbolizerSerializer
+
+    validates :definition, carto_json_symbolizer: true
+    serialize :conf, ::Carto::CartoJsonSymbolizerSerializer
+    validates :type, :layer, presence: true
+    validates :type, inclusion: { in: VALID_LEGEND_TYPES }, allow_nil: true
+
+    validate :on_data_layer,
+             :under_max_legends_per_layer,
+             :under_max_legends_per_layer_and_type,
+             :validate_definition_schema,
+             :validate_conf_schema
+
+    before_validation :ensure_definition, :ensure_conf
+
+    after_save :force_notify_layer_change
+    after_destroy :force_notify_layer_change
+
+    private
+
+    def ensure_definition
+      self.definition ||= Hash.new
+    end
+
+    def ensure_conf
+      self.conf ||= Hash.new
+    end
+
+    def on_data_layer
+      if layer && !layer.data_layer?
+        errors.add(:layer, "'#{layer.kind}' layers can't have legends")
+      end
+    end
+
+    MAX_LEGENDS_PER_LAYER = 2
+
+    def under_max_legends_per_layer
+      if layer
+        other_legends = layer.legends.select { |legend| legend.id != id }
+
+        unless other_legends.count < MAX_LEGENDS_PER_LAYER
+          errors.add(:layer, 'Maximum number of legends per layer reached')
+        end
+      end
+    end
+
+    def under_max_legends_per_layer_and_type
+      return unless type
+      LEGEND_TYPES_PER_ATTRIBUTE.each do |attribute, legend_types|
+        next unless legend_types.include?(type)
+
+        other_legends_present = layer.legends.any? { |legend| legend.id != id && legend_types.include?(legend.type) }
+        errors.add(:layer, "Only one #{attribute} legend per layer allowed") if other_legends_present
+      end
+    end
+
+    def validate_definition_schema
+      validator = Carto::LegendDefinitionValidator.new(type, definition)
+      definition_errors = validator.errors
+
+      if definition_errors.any?
+        errors.add(:definition, definition_errors.join(', '))
+      end
+    end
+
+    def validate_conf_schema
+      schema = Carto::Definition.instance
+                                .load_from_file('lib/formats/legends/conf.json')
+
+      parsed_conf = conf.try(:is_a?, Hash) ? conf.with_indifferent_access : conf
+      conf_errors = JSON::Validator.fully_validate(schema, parsed_conf)
+
+      if conf_errors.any?
+        errors.add(:conf, conf_errors.join(', '))
+      end
+    end
+
+    def force_notify_layer_change
+      layer.force_notify_change
+    end
+  end
+end

app/models/carto/license.rb

@@ -0,0 +1,68 @@
+module Carto
+
+  class License
+
+    MIT_LICENSE = :mit
+    GPLV2_LICENSE = :gplv2
+    GPLV3_LICENSE = :gplv3
+    APACHE_LICENSE = :apache
+    PDDL_LICENSE = :pddl
+    CC_0 = :cc_0
+    CC_BY_LICENSE = :cc_by
+    CC_BY_NC_LICENSE = :cc_by_nc
+    CC_BY_SA_LICENSE = :cc_by_sa
+    CC_BY_ND_LICENSE = :cc_by_nd
+    CC_BY_NC_SA_LICENSE = :cc_by_nc_sa
+    CC_BY_NC_ND_LICENSE = :cc_by_nc_nd
+    PUBLIC_DOMAIN_LICENSE = :public_domain
+    OPEN_GOVERNMENT_LICENSE = :open_government
+
+    LICENSE_VALUES = {
+      MIT_LICENSE => { id: MIT_LICENSE, name: 'MIT' },
+      APACHE_LICENSE => { id: APACHE_LICENSE, name: 'Apache license' },
+      GPLV3_LICENSE => { id: GPLV3_LICENSE, name: 'GPLv3 license' },
+      GPLV2_LICENSE => { id: GPLV2_LICENSE, name: 'GPLv2 license' },
+      PDDL_LICENSE => { id: PDDL_LICENSE, name: 'Public Domain Dedication and License' },
+      CC_0 => { id: CC_0, name: 'CC0: CC Public domain' },
+      CC_BY_LICENSE => { id: CC_BY_LICENSE, name: 'CC BY: Attribution alone' },
+      CC_BY_NC_LICENSE => { id: CC_BY_NC_LICENSE, name: 'CC BY-NC: Attribution + Noncommercial' },
+      CC_BY_SA_LICENSE => { id: CC_BY_SA_LICENSE, name: 'CC BY-SA: Attribution + ShareAlike' },
+      CC_BY_ND_LICENSE => { id: CC_BY_ND_LICENSE, name: 'CC BY-ND: Attribution + NoDerivatives' },
+      CC_BY_NC_SA_LICENSE => { id: CC_BY_NC_SA_LICENSE, name: 'CC BY-NC-SA: Attribution + Noncommercial + ShareAlike' },
+      CC_BY_NC_ND_LICENSE => { id: CC_BY_NC_ND_LICENSE, name: 'CC BY-NC-ND: Attribution + Noncommercial + NoDerivatives' },
+      PUBLIC_DOMAIN_LICENSE => { id: PUBLIC_DOMAIN_LICENSE, name: 'Public domain' },
+      OPEN_GOVERNMENT_LICENSE => { id: OPEN_GOVERNMENT_LICENSE, name: 'Open Government License' }
+    }
+
+    attr_reader :id, :name
+
+    private_class_method :new
+
+    def self.find(license_id)
+      if !license_id.nil?  && !license_id.empty? && LICENSE_VALUES.has_key?(license_id)
+        data = LICENSE_VALUES[license_id]
+        license = new(data[:id], data[:name])
+        license
+      else
+        return nil
+      end
+    end
+
+    def self.all
+      values = []
+      LICENSE_VALUES.each do |k, v|
+        values << find(k)
+      end
+      values
+    end
+
+    private
+
+    attr_writer :id, :name
+
+    def initialize(id, name)
+      self.id = id
+      self.name = name
+    end
+  end
+end

app/models/carto/license/presenter.rb

@@ -0,0 +1,15 @@
+module Carto
+  class LicensePresenter
+    def initialize(license)
+      @license = license
+    end
+
+    def to_poro
+      {
+        id:        @license.id,
+        name:      @license.name
+      }
+    end
+  end
+end
+

app/models/carto/like.rb

@@ -0,0 +1,5 @@
+require 'active_record'
+
+class Carto::Like < ActiveRecord::Base
+  # INFO: old code expects `actor` to be the uuid, not the entity
+end

app/models/carto/log.rb

@@ -0,0 +1,72 @@
+class Carto::Log < ActiveRecord::Base
+
+  has_one :data_import, class_name: Carto::DataImport, foreign_key: :logger
+
+  MAX_ENTRY_LENGTH = 256
+
+  ENTRY_POSTFIX = "\n"
+  ENTRY_FORMAT = "%s: %s#{ENTRY_POSTFIX}"
+
+  TYPE_USER_CREATION = 'user_creation'.freeze
+
+  # INFO: disable ActiveRecord inheritance column
+  self.inheritance_column = :_type
+
+  def self.new_user_creation
+    log = Carto::Log.new
+    log.entries = 'New user creation'
+    log.type = TYPE_USER_CREATION
+    log
+  end
+
+  def append(line, truncate = true, timestamp = Time.now.utc)
+    line.slice!(MAX_ENTRY_LENGTH..-1) if truncate
+
+    entry = ENTRY_FORMAT % [ timestamp, line.slice(0..MAX_ENTRY_LENGTH) ]
+    self.entries = "#{self.entries}#{entry}"
+    self.save
+  end
+
+  def append_exception(line, exception:, truncate: true, timestamp: Time.now.utc)
+    append("#{line}: #{exception_to_string(exception)}", truncate, timestamp)
+  end
+
+  def store
+    self.save
+  end
+
+  def logger
+    @logger ||= LogWriter.new(self)
+  end
+
+  private
+
+  def exception_to_string(error)
+    error.inspect + "\n" + error.backtrace.join("\n") + "\n"
+  end
+
+  # A logger implementation that logs to this Carto::Log
+  class LogWriter < ::Logger
+    SAVE_BLOCK = 10 # Save the changes to DB every X lines
+
+    def initialize(log)
+      @log = log
+      @stored_entries = 0
+    end
+
+    def add(_severity, _progname = nil, message = nil)
+      if message.present?
+        @log.entries += message.to_s + "\n"
+        @stored_entries += 1
+        if @stored_entries >= SAVE_BLOCK
+          @stored_entries = 0
+          @log.save
+        end
+      end
+    end
+
+    def close
+      @log.save
+    end
+  end
+end

app/models/carto/map.rb

@@ -0,0 +1,230 @@
+require 'active_record'
+
+require_relative './carto_json_serializer'
+require_dependency 'common/map_common'
+require_dependency 'carto/bounding_box_utils'
+
+class Carto::Map < ActiveRecord::Base
+  include Carto::MapBoundaries
+
+  has_many :layers_maps, dependent: :destroy
+  has_many :layers, -> { order(:order) }, class_name: 'Carto::Layer',
+                                          through: :layers_maps,
+                                          after_add: Proc.new { |map, layer| layer.after_added_to_map(map) }
+
+  has_many :base_layers, -> { order(:order) }, class_name: 'Carto::Layer',
+                                               through: :layers_maps,
+                                               source: :layer
+
+  # autosave must be explicitly disabled due to https://github.com/rails/rails/issues/9336
+  has_one :user_table, class_name: Carto::UserTable, inverse_of: :map, dependent: :destroy, autosave: false
+
+  belongs_to :user
+
+  # Autosave disabled because this caused the `inverse_of` to break (visualization.map != self) until `reload`
+  # Fixed in Rails 5 https://github.com/rails/rails/pull/23197
+  has_one :visualization, class_name: Carto::Visualization, inverse_of: :map, autosave: false
+
+  # bounding_box_sw, bounding_box_new and center should probably be JSON serialized fields
+  # However, many callers expect to get an string (and do the JSON deserialization themselves), mainly in presenters
+  # So for now, we are just treating them as strings (see the .to_s in the constant below), but this could be improved
+  DEFAULT_OPTIONS = {
+    zoom:            3,
+    bounding_box_sw: [Carto::BoundingBoxUtils::DEFAULT_BOUNDS[:miny],
+                      Carto::BoundingBoxUtils::DEFAULT_BOUNDS[:minx]].to_s,
+    bounding_box_ne: [Carto::BoundingBoxUtils::DEFAULT_BOUNDS[:maxy],
+                      Carto::BoundingBoxUtils::DEFAULT_BOUNDS[:maxx]].to_s,
+    provider:        'leaflet',
+    center:          [30, 0].to_s
+  }.freeze
+
+  serialize :options, ::Carto::CartoJsonSerializer
+  validates :options, carto_json_symbolizer: true
+
+  validate :validate_options
+
+  after_initialize :ensure_options
+  after_save :notify_map_change
+  after_save :save_table # Manual save, since autosave is disabled
+
+  def data_layers
+    layers.select(&:data_layer?)
+  end
+
+  def carto_layers
+    layers.select(&:carto?)
+  end
+
+  def user_layers
+    layers.select(&:user_layer?)
+  end
+
+  def torque_layers
+    layers.select(&:torque?)
+  end
+
+  def other_layers
+    layers.reject(&:carto?)
+          .reject(&:tiled?)
+          .reject(&:background?)
+          .reject(&:gmapsbase?)
+          .reject(&:wms?)
+  end
+
+  def named_map_layers
+    layers.select(&:named_map_layer?)
+  end
+
+  def viz_updated_at
+    latest_mapcap = visualization.latest_mapcap
+
+    latest_mapcap ? latest_mapcap.created_at : get_the_last_time_tiles_have_changed_to_render_it_in_vizjsons
+  end
+
+  def self.provider_for_baselayer_kind(kind)
+    kind == 'tiled' ? 'leaflet' : 'googlemaps'
+  end
+
+  # (lat,lon) points on all map data
+  def center_data
+    (center.nil? || center == '') ? DEFAULT_OPTIONS[:center] : center.gsub(/\[|\]|\s*/, '').split(',')
+  end
+
+  def view_bounds_data
+    if view_bounds_sw.nil? || view_bounds_sw == ''
+      bbox_sw = DEFAULT_OPTIONS[:bounding_box_sw]
+    else
+      bbox_sw = view_bounds_sw.gsub(/\[|\]|\s*/, '').split(',').map(&:to_f)
+    end
+    if view_bounds_ne.nil? || view_bounds_ne == ''
+      bbox_ne = DEFAULT_OPTIONS[:bounding_box_ne]
+    else
+      bbox_ne = view_bounds_ne.gsub(/\[|\]|\s*/, '').split(',').map(&:to_f)
+    end
+
+    {
+      # LowerCorner longitude, in decimal degrees
+      west:  bbox_sw[1],
+      # LowerCorner latitude, in decimal degrees
+      south: bbox_sw[0],
+      # UpperCorner longitude, in decimal degrees
+      east:  bbox_ne[1],
+      # UpperCorner latitude, in decimal degrees
+      north: bbox_ne[0]
+    }
+  end
+
+  def writable_by_user?(user)
+    visualization.writable_by?(user)
+  end
+
+  def contains_layer?(layer)
+    return false unless layer
+
+    layers_maps.map(&:layer_id).include?(layer.id)
+  end
+
+  def notify_map_change
+    visualization.try(:invalidate_after_commit)
+  end
+  alias :force_notify_map_change :notify_map_change
+
+  def update_dataset_dependencies
+    data_layers.each(&:register_table_dependencies)
+  end
+
+  def dashboard_menu=(value)
+    options[:dashboard_menu] = value
+  end
+
+  def dashboard_menu
+    options[:dashboard_menu]
+  end
+
+  def layer_selector=(value)
+    options[:layer_selector] = value
+  end
+
+  def layer_selector
+    options[:layer_selector]
+  end
+
+  def admits_layer?(layer)
+    return admits_more_torque_layers? if layer.torque?
+    return admits_more_data_layers? if layer.data_layer?
+    return admits_more_base_layers?(layer) if layer.base_layer?
+  end
+
+  def can_add_layer?(user, layer)
+    return true if layer.base_layer?
+    return false if user.max_layers && user.max_layers <= data_layers.count
+
+    visualization.writable_by?(user)
+  end
+
+  def process_privacy_in(layer)
+    if layer.uses_private_tables? && visualization.can_be_private?
+      visualization.privacy = Carto::Visualization::PRIVACY_PRIVATE
+      visualization.save
+    end
+  end
+
+  private
+
+  def save_table
+    if user_table && !user_table.persisted?
+      user_table.map = self
+      user_table.save!
+    end
+  end
+
+  def admits_more_data_layers?
+    !visualization.canonical? || data_layers.empty?
+  end
+
+  def admits_more_torque_layers?
+    !visualization.canonical? || torque_layers.empty?
+  end
+
+  def admits_more_base_layers?(layer)
+    # no basemap layer, always allow
+    return true if user_layers.empty?
+    # have basemap? then allow only if comes on top (for labels)
+    layer.order >= layers.last.order
+  end
+
+  def ensure_options
+    self.zoom ||= DEFAULT_OPTIONS[:zoom]
+    self.bounding_box_sw ||= DEFAULT_OPTIONS[:bounding_box_sw]
+    self.bounding_box_ne ||= DEFAULT_OPTIONS[:bounding_box_ne]
+    self.center ||= DEFAULT_OPTIONS[:center]
+    self.provider ||= DEFAULT_OPTIONS[:provider]
+
+    self.options ||= {}
+    options[:dashboard_menu] = true if options[:dashboard_menu].nil?
+    options[:layer_selector] = false if options[:layer_selector].nil?
+    options[:legends] = legends if options[:legends].nil?
+    options[:scrollwheel] = scrollwheel if options[:scrollwheel].nil?
+
+    options
+  end
+
+  def validate_options
+    location = "#{Rails.root}/lib/formats/map/options.json"
+    schema = Carto::Definition.instance.load_from_file(location)
+
+    options_wia = options.with_indifferent_access
+    json_errors = JSON::Validator.fully_validate(schema, options_wia)
+    errors.add(:options, json_errors.join(', ')) if json_errors.any?
+  end
+
+  def get_the_last_time_tiles_have_changed_to_render_it_in_vizjsons
+    from_table = user_table.service.data_last_modified if user_table
+
+    [from_table, data_layers.map(&:updated_at)].flatten.compact.max
+  end
+
+  def table_name
+    user_table.try(:name)
+  end
+end

app/models/carto/mapcap.rb

@@ -0,0 +1,75 @@
+require_dependency 'carto/visualizations_export_service_2'
+require_relative './carto_json_serializer'
+require_dependency 'carto/named_maps/api'
+
+module Carto
+  class Mapcap < ActiveRecord::Base
+    include Carto::VisualizationsExportService2Importer
+    include Carto::VisualizationsExportService2Exporter
+
+    belongs_to :visualization, class_name: Carto::Visualization, foreign_key: 'visualization_id'
+
+    serialize :ids_json, ::Carto::CartoJsonSymbolizerSerializer
+    serialize :export_json, ::Carto::CartoJsonSymbolizerSerializer
+
+    after_save :notify_map_change, :update_named_map
+    after_destroy :notify_map_change
+
+    before_validation :lazy_export_json, :lazy_ids_json
+
+    validates :ids_json, carto_json_symbolizer: true
+    validates :export_json, carto_json_symbolizer: true
+
+    def regenerate_visualization
+      regenerated_visualization = build_visualization_from_hash_export(lazy_export_json)
+
+      regenerated_visualization.user = regenerated_visualization.map.user = visualization.user
+      regenerated_visualization.permission = visualization.permission
+
+      regenerated_visualization.populate_ids(lazy_ids_json)
+      set_tree_as_readonly!(regenerated_visualization)
+
+      regenerated_visualization
+    end
+
+    def self.latest_for_visualization(visualization_id)
+      where(visualization_id: visualization_id).order('created_at DESC')
+    end
+
+    private
+
+    def lazy_export_json
+      self.export_json ||= export_visualization_json_hash(visualization_id, visualization.user, with_mapcaps: false)
+    end
+
+    def lazy_ids_json
+      self.ids_json ||= visualization.ids_json
+    end
+
+    def notify_map_change
+      visualization.map.force_notify_map_change
+    end
+
+    def update_named_map
+      Carto::NamedMaps::Api.new(regenerate_visualization).upsert
+    end
+
+    def set_tree_as_readonly!(entity)
+      set_entity_as_readonly(entity)
+      Carto::Visualization.reflections.each_key do |dep_name|
+        dependency = entity.public_send(dep_name)
+        if dependency.is_a? ActiveRecord::Associations::CollectionProxy
+          dependency.each { |e| set_entity_as_readonly(e) }
+        else
+          set_entity_as_readonly(dependency)
+        end
+      end
+    end
+
+    def set_entity_as_readonly(entity)
+      return unless entity.public_methods.include?(:readonly!)
+      return if entity.readonly?
+      entity.readonly!
+    end
+  end
+end

app/models/carto/mobile_app.rb

@@ -0,0 +1,30 @@
+class Carto::MobileApp
+  include ActiveModel::Validations
+  include ActiveModel::Conversion
+  extend ActiveModel::Naming
+
+  APP_PLATFORMS = %w(android ios xamarin-android xamarin-ios windows-phone).freeze
+  APP_TYPES = %w(open private).freeze
+
+  validates :name,          presence: true
+  validates :icon_url,      presence: true
+  validates :platform,      inclusion: { in: APP_PLATFORMS }
+  validates :app_id,        presence: true
+  validates :app_type,      inclusion: { in: APP_TYPES }
+
+  attr_accessor :id, :name, :description, :icon_url, :platform, :app_id, :app_type, :license_key, :monthly_users
+
+  def initialize(attributes = {})
+    attributes.each do |name, value|
+      send("#{name}=", value)
+    end
+  end
+
+  def persisted?
+    id.present?
+  end
+
+  def data(current_user, fetch_mobile_platforms: false, fetch_app_types: false)
+    Carto::Api::MobileAppPresenter.new(self, current_user, fetch_mobile_platforms, fetch_app_types).data
+  end
+end

app/models/carto/notification.rb

@@ -0,0 +1,50 @@
+require_dependency 'carto/notifications_markdown_renderer'
+
+module Carto
+  class Notification < ActiveRecord::Base
+    MAX_BODY_LENGTH = 140
+
+    # Update CartoGearsAPI::Notifications::Notification when adding constants here
+    ICON_ALERT = 'alert'.freeze
+    ICON_SUCCESS = 'success'.freeze
+    ICONS = [ICON_SUCCESS, ICON_ALERT].freeze
+
+    RECIPIENT_ALL = 'all'.freeze
+    RECIPIENTS = [RECIPIENT_ALL, 'builders'.freeze, 'viewers'.freeze].freeze
+
+    belongs_to :organization, inverse_of: :notifications
+    has_many :received_notifications, inverse_of: :notification
+
+    validates :icon, presence: true, inclusion: { in: ICONS }
+    validates :recipients, inclusion: { in: [nil] + RECIPIENTS }
+    validates :recipients, presence: true, if: :organization
+    validates :body, presence: true
+    validate  :valid_markdown
+
+    after_create :send_to_organization_members
+
+    private
+
+    def send_to_organization_members
+      return unless organization
+      # This avoids resending an already sent notification. Example: importing notifications
+      return unless received_notifications.empty?
+      users = if recipients == 'builders'
+                organization.builder_users
+              elsif recipients == 'viewers'
+                organization.viewer_users
+              else
+                organization.users
+              end
+      users.each { |u| received_notifications.create!(user: u, received_at: created_at) }
+    end
+
+    def valid_markdown
+      return unless body.present?
+      text = Redcarpet::Markdown.new(NotificationsMarkdownRenderer).render(body).strip
+      errors.add(:body, "cannot be longer than #{MAX_BODY_LENGTH} characters") if text.length > MAX_BODY_LENGTH
+    rescue RuntimeError => e
+      errors.add(:body, e.message)
+    end
+  end
+end

app/models/carto/oauth_access_token.rb

@@ -0,0 +1,57 @@
+require_dependency 'carto/oauth_provider/errors'
+require_dependency 'carto/oauth_provider/scopes/scopes'
+
+module Carto
+  class OauthAccessToken < ActiveRecord::Base
+    include OauthProvider::Scopes
+
+    ACCESS_TOKEN_EXPIRATION_TIME = 1.hour
+
+    belongs_to :oauth_app_user, inverse_of: :oauth_access_tokens
+    belongs_to :api_key, inverse_of: :oauth_access_token, dependent: :destroy
+
+    validates :oauth_app_user, presence: true
+
+    validates :scopes, scopes: true
+
+    before_create :create_api_key, unless: :skip_api_key_creation
+    after_create :rename_api_key, unless: :skip_api_key_creation
+
+    scope :expired, -> { where('created_at < ?', Time.now - ACCESS_TOKEN_EXPIRATION_TIME) }
+
+    attr_accessor :skip_api_key_creation
+
+    def expires_in
+      created_at + ACCESS_TOKEN_EXPIRATION_TIME - Time.now
+    end
+
+    def user
+      oauth_app_user.user
+    end
+
+    def ownership_role_name
+      oauth_app_user.ownership_role_name
+    end
+
+    private
+
+    def create_api_key
+      grants = [{ type: 'apis', apis: [] }]
+      scopes.each do |s|
+        scope = OauthProvider::Scopes.build(s)
+        scope.add_to_api_key_grants(grants, user)
+      end
+
+      self.api_key = oauth_app_user.user.api_keys.create_oauth_key!(
+        name: "oauth_authorization #{SecureRandom.uuid}",
+        grants: grants,
+        ownership_role_name: ownership_role_name
+      )
+    end
+
+    def rename_api_key
+      # Rename after creation so we have the ID
+      api_key.update!(name: "oauth_authorization #{id}")
+    end
+  end
+end

app/models/carto/oauth_app.rb

@@ -0,0 +1,116 @@
+require_dependency 'carto/helpers/url_validator'
+
+module Carto
+  class OauthApp < ActiveRecord::Base
+    # Multiple of 3 for pretty base64
+    CLIENT_ID_RANDOM_BYTES = 9
+    CLIENT_SECRET_RANDOM_BYTES = 18
+
+    belongs_to :user, inverse_of: :oauth_apps
+    has_many :oauth_app_users, inverse_of: :oauth_app, dependent: :destroy
+    has_many :oauth_app_organizations, inverse_of: :oauth_app, dependent: :destroy
+
+    validates :user, presence: true, if: -> { sync_with_central? || !central_enabled? }
+    validates :name, presence: true
+    validates :website_url, presence: true, url: true
+    validates :client_id, presence: true
+    validates :client_secret, presence: true
+    validates :redirect_uris, presence: true
+    validates :icon_url, url: true, if: -> { icon_url.present? }
+    validates :oauth_app_organizations, absence: true, unless: :restricted?
+    validate :validate_uris
+
+    before_validation :ensure_keys_generated
+
+    after_create :create_central, if: :sync_with_central?
+    after_update :update_central, if: :sync_with_central?
+    after_destroy :delete_central, if: :sync_with_central?
+
+    before_destroy :collect_users, unless: :avoid_send_notification, prepend: true
+    after_destroy :send_app_removal_notification, unless: :avoid_send_notification
+
+    ALLOWED_SYNC_ATTRIBUTES = %i[id name client_id client_secret redirect_uris
+      icon_url restricted description website_url].freeze
+
+    attr_accessor :avoid_sync_central, :avoid_send_notification
+
+    # this should be exactly the same method as in Central
+    # mostly used for testing the Superadmin API
+    def api_attributes
+      attributes.symbolize_keys.slice(*ALLOWED_SYNC_ATTRIBUTES).merge(user_id: user.id)
+    end
+
+    def regenerate_client_secret!
+      self.client_secret = SecureRandom.urlsafe_base64(CLIENT_SECRET_RANDOM_BYTES)
+      save!
+    end
+
+    private
+
+    def collect_users
+      @user_ids = oauth_app_users.collect { |u| u.user.id }
+    end
+
+    def send_app_removal_notification
+      return if @user_ids.empty?
+      notification = Carto::Notification.create!(body: notification_body, icon: Notification::ICON_ALERT)
+      ::Resque.enqueue(::Resque::UserJobs::Notifications::Send, @user_ids, notification.id)
+    rescue StandardError => e
+      CartoDB::Logger.warning(
+        message: "Couldn't notify users about oauth_app '#{name}' deletion",
+        notification_id: notification&.id,
+        exception: e)
+        raise e
+    end
+
+    def notification_body
+      "The app #{name} has signed off. You may find more information in their [website](#{website_url})."
+    end
+
+    def ensure_keys_generated
+      self.client_id ||= SecureRandom.urlsafe_base64(CLIENT_ID_RANDOM_BYTES)
+      self.client_secret ||= SecureRandom.urlsafe_base64(CLIENT_SECRET_RANDOM_BYTES)
+    end
+
+    def validate_uris
+      redirect_uris && redirect_uris.each { |uri| validate_uri(uri) }
+    end
+
+    def validate_uri(redirect_uri)
+      uri = URI.parse(redirect_uri)
+      errors.add(:redirect_uris, "must be absolute") unless uri.absolute?
+      errors.add(:redirect_uris, "must be https") unless uri.scheme == 'https'
+      errors.add(:redirect_uris, "must not contain a fragment") unless uri.fragment.nil?
+    rescue URI::InvalidURIError
+      errors.add(:redirect_uris, "must be valid")
+    end
+
+    def create_central
+      cartodb_central_client.create_oauth_app(user.username, sync_attributes)
+    end
+
+    def update_central
+      cartodb_central_client.update_oauth_app(user.username, id, sync_attributes)
+    end
+
+    def delete_central
+      cartodb_central_client.delete_oauth_app(user.username, id)
+    end
+
+    def cartodb_central_client
+      @cartodb_central_client ||= Cartodb::Central.new
+    end
+
+    def sync_with_central?
+      central_enabled? && !avoid_sync_central
+    end
+
+    def central_enabled?
+      Cartodb::Central.sync_data_with_cartodb_central?
+    end
+
+    def sync_attributes(attrs = attributes)
+      attrs.symbolize_keys.slice(*ALLOWED_SYNC_ATTRIBUTES)
+    end
+  end
+end

app/models/carto/oauth_app_organization.rb

@@ -0,0 +1,14 @@
+module Carto
+  class OauthAppOrganization < ActiveRecord::Base
+    belongs_to :organization, inverse_of: :oauth_app_organizations
+    belongs_to :oauth_app, inverse_of: :oauth_app_organizations
+
+    validates :organization, presence: true, uniqueness: { scope: :oauth_app }
+    validates :oauth_app, presence: true
+    validates :seats, presence: true, numericality: { only_integer: true, greater_than: 0 }
+
+    def open_seats?
+      oauth_app.oauth_app_users.joins(:user).where(users: { organization_id: organization.id }).count < seats
+    end
+  end
+end

app/models/carto/oauth_app_user.rb

@@ -0,0 +1,253 @@
+require_dependency 'carto/oauth_provider/scopes/scopes'
+require_dependency 'carto/oauth_provider/errors'
+
+module Carto
+  class OauthAppUser < ActiveRecord::Base
+    include OauthProvider::Scopes
+
+    belongs_to :user, inverse_of: :oauth_app_users
+    belongs_to :oauth_app, inverse_of: :oauth_app_users
+    belongs_to :api_key, inverse_of: :oauth_app_user
+    has_many :oauth_authorization_codes, inverse_of: :oauth_app_user, dependent: :destroy
+    has_many :oauth_access_tokens, inverse_of: :oauth_app_user, dependent: :destroy
+    has_many :oauth_refresh_tokens, inverse_of: :oauth_app_user, dependent: :destroy
+
+    validates :user, presence: true, uniqueness: { scope: :oauth_app }
+    validates :oauth_app, presence: true
+    validates :scopes, scopes: true
+    validate  :validate_user_authorizable, on: :create
+
+    after_create :create_roles, :enable_schema_triggers, :ensure_role_grants, unless: :skip_role_setup
+    before_update :grant_dataset_role_privileges
+    after_destroy :reassign_owners, :drop_roles, :disable_schema_triggers, unless: :skip_role_setup
+
+    attr_accessor :skip_role_setup
+
+    def authorized?(requested_scopes)
+      OauthProvider::Scopes.subtract_scopes(requested_scopes, all_scopes, user.database_schema).empty?
+    end
+
+    def upgrade!(requested_scopes)
+      update!(scopes: all_scopes | requested_scopes)
+    end
+
+    def all_scopes
+      no_dataset_scopes + dataset_scopes
+    end
+
+    def revoke_permissions(table, revoked_permissions)
+      db_connection = user.in_database(as: :superuser)
+      Carto::TableAndFriends.apply(db_connection, table.database_schema, table.name) do |_s, _t, qualified_name|
+        query = %{
+          REVOKE #{revoked_permissions.join(', ')}
+          ON TABLE #{qualified_name}
+          FROM \"#{dataset_role_name}\"
+        }
+        db_connection.execute(query)
+        sequences_for_table(qualified_name).each do |seq|
+          db_connection.execute("REVOKE ALL ON SEQUENCE #{seq} FROM \"#{dataset_role_name}\"")
+        end
+      end
+    end
+
+    def create_roles
+      create_dataset_role
+      create_ownership_role
+    end
+
+    def create_dataset_role
+      db_run(create_dataset_role_query)
+    end
+
+    def create_ownership_role
+      db_run(create_ownership_role_query)
+    end
+
+    def create_dataset_role_query
+      "CREATE ROLE \"#{dataset_role_name}\" CREATEROLE"
+    end
+
+    def create_ownership_role_query
+      "CREATE ROLE \"#{ownership_role_name}\""
+    end
+
+    def ensure_role_grants
+      grant_dataset_role_privileges
+      grant_ownership_role_privileges
+    end
+
+    def grant_ownership_role_privileges
+      db_run("GRANT \"#{ownership_role_name}\" TO \"#{user.database_username}\"")
+    end
+
+    def grant_dataset_role_privileges
+      validate_scopes
+
+      invalid_scopes = []
+      DatasetsScope.valid_scopes(scopes).each do |scope|
+        dataset_scope = DatasetsScope.new(scope)
+
+        schema = dataset_scope.schema || user.database_schema
+        schema_table = Carto::TableAndFriends.qualified_table_name(schema, dataset_scope.table)
+
+        table_query = %{
+          GRANT #{dataset_scope.permission.join(',')}
+          ON TABLE #{schema_table}
+          TO \"#{dataset_role_name}\" WITH GRANT OPTION;
+        }
+
+        schema_query = %{
+          GRANT USAGE ON SCHEMA \"#{schema}\"
+          TO \"#{dataset_role_name}\";
+        }
+
+        begin
+          user.in_database(as: :superuser).execute(table_query)
+          user.in_database(as: :superuser).execute(schema_query)
+          sequences_for_table(schema_table).each do |seq|
+            seq_query = "GRANT USAGE, SELECT ON SEQUENCE #{seq} TO \"#{dataset_role_name}\""
+            user.in_database(as: :superuser).execute(seq_query)
+          end
+        rescue ActiveRecord::StatementInvalid => e
+          invalid_scopes << scope
+          CartoDB::Logger.error(message: 'Error granting permissions to dataset role', exception: e)
+        end
+      end
+
+      raise OauthProvider::Errors::InvalidScope.new(invalid_scopes) if invalid_scopes.any?
+    end
+
+    def ownership_role_name
+      "carto_oauth_app_o_#{id}"
+    end
+
+    def exists_ownership_role?
+      user.db_service.exists_role?(ownership_role_name)
+    end
+
+    private
+
+    def validate_user_authorizable
+      return unless oauth_app.try(:restricted?)
+
+      organization = user.organization
+      return errors.add(:user, 'is not part of an organization') unless user.organization
+
+      org_authorization = oauth_app.oauth_app_organizations.where(organization: organization).first
+      unless org_authorization
+        return errors.add(:user, 'is part of an organization which is not allowed access to this application')
+      end
+
+      errors.add(:user, 'does not have an available seat to use this application') unless org_authorization.open_seats?
+    end
+
+    def no_dataset_scopes
+      DatasetsScope.non_dataset_scopes(scopes)
+    end
+
+    def dataset_scopes
+      begin
+        results = user.db_service.all_tables_granted(dataset_role_name)
+      rescue ActiveRecord::StatementInvalid => e
+        CartoDB::Logger.error(message: 'Error getting scopes', exception: e)
+        raise OauthProvider::Errors::ServerError.new
+      end
+
+      scopes = []
+      results.each do |row|
+        permission = DatasetsScope.permission_from_db_to_scope(row['permission'])
+        unless permission.nil?
+          schema = row['schema'] != user.database_schema ? "#{row['schema']}." : ''
+          scopes << "datasets:#{permission}:#{schema}#{row['t']}"
+        end
+      end
+      scopes
+    end
+
+    def reassign_owners
+      roles = [dataset_role_name, ownership_role_name]
+      queries = roles.map do |role|
+        "REASSIGN OWNED BY \"#{role}\" TO \"#{user.database_username}\";"
+      end
+      db_run(queries.join, error_title: 'Error reassigning owners')
+    end
+
+    def drop_roles
+      roles = [dataset_role_name, ownership_role_name]
+      queries = roles.map do |role|
+        %{
+          DROP OWNED BY "#{role}";
+          DROP ROLE IF EXISTS "#{role}";
+        }
+      end
+      db_run(queries.join, error_title: 'Error dropping roles')
+    end
+
+    def enable_schema_triggers
+      return if user.organization_user? && oauth_users_in_organization > 1
+      user.db_service.create_oauth_reassign_ownership_event_trigger
+    rescue StandardError => e
+      CartoDB::Logger.error(
+        message:    "Error enabling schema trigger",
+        exception:  e,
+        user:       self
+      )
+    end
+
+    def disable_schema_triggers
+      return if user.organization_user? && oauth_users_in_organization >= 1
+      user.db_service.drop_oauth_reassign_ownership_event_trigger
+    rescue StandardError => e
+      CartoDB::Logger.error(
+        message:    "Error disabling schema trigger",
+        exception:  e,
+        user:       self
+      )
+    end
+
+    def validate_scopes
+      invalid_scopes = OauthProvider::Scopes.invalid_scopes_and_tables(scopes, user)
+      raise OauthProvider::Errors::InvalidScope.new(invalid_scopes) if invalid_scopes.present?
+    end
+
+    def sequences_for_table(schema_table)
+      query = %{
+        SELECT
+          n.nspname, quote_ident(c.relname) as relname
+        FROM
+          pg_depend d
+          JOIN pg_class c ON d.objid = c.oid
+          JOIN pg_namespace n ON c.relnamespace = n.oid
+        WHERE
+          d.refobjsubid > 0 AND
+          d.classid = 'pg_class'::regclass AND
+          c.relkind = 'S'::"char" AND
+          d.refobjid = ('#{schema_table}')::regclass
+      }
+
+      user.in_database.execute(query).map { |r| "\"#{r['nspname']}\".#{r['relname']}" }
+    end
+
+    def dataset_role_name
+      "carto_oauth_app_#{id}"
+    end
+
+    def db_run(query, connection = db_connection, error_title: 'Error running SQL command')
+      connection.execute(query)
+    rescue ActiveRecord::StatementInvalid => e
+      CartoDB::Logger.error(message: error_title, exception: e)
+      return if e.message =~ /OWNED BY/ # role might not exist becuase it has been already dropped      
+      raise OauthProvider::Errors::ServerError.new("#{error_title}: #{e.message}")
+    end
+
+    def db_connection
+      @db_connection ||= user.in_database(as: :superuser)
+    end
+
+    def oauth_users_in_organization
+      return 0 unless user.organization_user?
+
+      Carto::OauthAppUser.joins(:user).where('organization_id = ?', user.organization_id).count
+    end
+  end
+end

app/models/carto/oauth_authorization_code.rb

@@ -0,0 +1,49 @@
+require_dependency 'carto/oauth_provider/errors'
+require_dependency 'carto/oauth_provider/scopes/scopes'
+
+module Carto
+  class OauthAuthorizationCode < ActiveRecord::Base
+    include OauthProvider::Scopes
+    # Multiple of 3 for pretty base64
+    CODE_RANDOM_BYTES = 12
+
+    CODE_EXPIRATION_TIME = 1.minute
+
+    belongs_to :oauth_app_user, inverse_of: :oauth_authorization_codes
+
+    validates :oauth_app_user, presence: true
+    validates :code, presence: true
+    validates :scopes, scopes: true
+
+    before_validation :ensure_code_generated
+
+    scope :expired, -> { where('created_at < ?', Time.now - CODE_EXPIRATION_TIME) }
+
+    def exchange!
+      raise OauthProvider::Errors::InvalidGrant.new if expired?
+
+      ActiveRecord::Base.transaction do
+        destroy!
+        access_token = oauth_app_user.oauth_access_tokens.create!(scopes: scopes)
+        if scopes.include?(SCOPE_OFFLINE)
+          refresh_token = oauth_app_user.oauth_refresh_tokens.create!(scopes: scopes)
+        end
+        [access_token, refresh_token]
+      end
+    end
+
+    def oauth_app
+      oauth_app_user.oauth_app
+    end
+
+    private
+
+    def ensure_code_generated
+      self.code ||= SecureRandom.urlsafe_base64(CODE_RANDOM_BYTES)
+    end
+
+    def expired?
+      created_at < Time.now - CODE_EXPIRATION_TIME
+    end
+  end
+end

app/models/carto/oauth_refresh_token.rb

@@ -0,0 +1,68 @@
+require_dependency 'carto/oauth_provider/errors'
+require_dependency 'carto/oauth_provider/scopes/scopes'
+
+module Carto
+  class OauthRefreshToken < ActiveRecord::Base
+    include OauthProvider::Scopes
+
+    # Multiple of 3 for pretty base64
+    TOKEN_RANDOM_BYTES = 15
+    REFRESH_TOKEN_EXPIRATION_TIME = 6.months
+    MAX_TOKENS_PER_OAUTH_APP_USER = 50
+
+    belongs_to :oauth_app_user, inverse_of: :oauth_refresh_tokens
+
+    validates :oauth_app_user, presence: true
+    validates :scopes, scopes: true
+    validate  :check_offline_scope
+
+    before_create :remove_oldest_ones, unless: :skip_token_regeneration
+    before_create :regenerate_token, unless: :skip_token_regeneration
+
+    scope :expired, -> { where('updated_at < ?', Time.now - REFRESH_TOKEN_EXPIRATION_TIME) }
+
+    attr_accessor :skip_token_regeneration
+
+    def exchange!(requested_scopes: scopes)
+      raise OauthProvider::Errors::InvalidGrant.new if expired?
+      invalid_scopes = Carto::OauthProvider::Scopes.subtract_scopes(requested_scopes, scopes, oauth_app_user.user.database_schema)
+      raise OauthProvider::Errors::InvalidScope.new(invalid_scopes) if invalid_scopes.any?
+
+      ActiveRecord::Base.transaction do
+        regenerate_token
+        save!
+        [oauth_app_user.oauth_access_tokens.create!(scopes: requested_scopes), self]
+      end
+    end
+
+    def oauth_app
+      oauth_app_user.oauth_app
+    end
+
+    private
+
+    def remove_oldest_ones
+      oldest_tokens = OauthRefreshToken.where(oauth_app_user: oauth_app_user)
+                                       .order(updated_at: :desc)
+                                       .offset(MAX_TOKENS_PER_OAUTH_APP_USER - 1)
+
+      unless oldest_tokens.count.zero?
+        CartoDB::Logger.debug(message: "Removing #{oldest_tokens.count} refresh tokens",
+                              oauth_app_user: oauth_app_user)
+        oldest_tokens.find_each(&:destroy!)
+      end
+    end
+
+    def expired?
+      updated_at < Time.now - REFRESH_TOKEN_EXPIRATION_TIME
+    end
+
+    def check_offline_scope
+      errors.add(:scopes, "must contain `#{SCOPE_OFFLINE}`") unless scopes.include?(SCOPE_OFFLINE)
+    end
+
+    def regenerate_token
+      self.token = SecureRandom.urlsafe_base64(TOKEN_RANDOM_BYTES)
+    end
+  end
+end

app/models/carto/oauth_token.rb

@@ -0,0 +1,10 @@
+require 'active_record'
+
+module Carto
+  class OauthToken < ActiveRecord::Base
+
+    belongs_to :user, class_name: Carto::User
+    belongs_to :client_application, class_name: Carto::ClientApplication
+
+  end
+end

app/models/carto/organization.rb

@@ -0,0 +1,176 @@
+require 'active_record'
+require_relative '../../helpers/data_services_metrics_helper'
+require_dependency 'carto/helpers/auth_token_generator'
+require_dependency 'carto/carto_json_serializer'
+require_dependency 'common/organization_common'
+
+module Carto
+  class Organization < ActiveRecord::Base
+    include DataServicesMetricsHelper
+    include AuthTokenGenerator
+    include Carto::OrganizationSoftLimits
+
+    serialize :auth_saml_configuration, CartoJsonSymbolizerSerializer
+    before_validation :ensure_auth_saml_configuration
+    validates :auth_saml_configuration, carto_json_symbolizer: true
+
+    has_many :users, -> { order(:username) }, inverse_of: :organization
+    belongs_to :owner, class_name: Carto::User, inverse_of: :owned_organization
+    has_many :groups, -> { order(:display_name) }, inverse_of: :organization
+    has_many :assets, class_name: Carto::Asset, dependent: :destroy, inverse_of: :organization
+    has_many :notifications, -> { order('created_at DESC') }, dependent: :destroy
+    has_many :connector_configurations, inverse_of: :organization, dependent: :destroy
+    has_many :oauth_app_organizations, inverse_of: :oauth_app, dependent: :destroy
+
+    before_destroy :destroy_groups_with_extension
+
+    def self.find_by_database_name(database_name)
+      Carto::Organization.
+        joins('INNER JOIN users ON organizations.owner_id = users.id').
+        where('users.database_name = ?', database_name).first
+    end
+
+    def get_geocoding_calls(options = {})
+      require_organization_owner_presence!
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      get_organization_geocoding_data(self, date_from, date_to)
+    end
+
+    def period_end_date
+      owner && owner.period_end_date
+    end
+
+    def get_here_isolines_calls(options = {})
+      require_organization_owner_presence!
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      get_organization_here_isolines_data(self, date_from, date_to)
+    end
+
+    def get_mapzen_routing_calls(options = {})
+      require_organization_owner_presence!
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      get_organization_mapzen_routing_data(self, date_from, date_to)
+    end
+
+    def get_obs_snapshot_calls(options = {})
+      require_organization_owner_presence!
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      get_organization_obs_snapshot_data(self, date_from, date_to)
+    end
+
+    def get_obs_general_calls(options = {})
+      require_organization_owner_presence!
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      get_organization_obs_general_data(self, date_from, date_to)
+    end
+
+    def twitter_imports_count(options = {})
+      require_organization_owner_presence!
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : owner.last_billing_cycle)
+      Carto::SearchTweet.twitter_imports_count(users.joins(:search_tweets), date_from, date_to)
+    end
+
+    def owner?(user)
+      owner_id == user.id
+    end
+
+    def remaining_geocoding_quota(options = {})
+      remaining = geocoding_quota.to_i - get_geocoding_calls(options)
+      (remaining > 0 ? remaining : 0)
+    end
+
+    def remaining_here_isolines_quota(options = {})
+      remaining = here_isolines_quota.to_i - get_here_isolines_calls(options)
+      (remaining > 0 ? remaining : 0)
+    end
+
+    def remaining_mapzen_routing_quota(options = {})
+      remaining = mapzen_routing_quota.to_i - get_mapzen_routing_calls(options)
+      (remaining > 0 ? remaining : 0)
+    end
+
+    def remaining_obs_snapshot_quota(options = {})
+      remaining = obs_snapshot_quota.to_i - get_obs_snapshot_calls(options)
+      (remaining > 0 ? remaining : 0)
+    end
+
+    def remaining_obs_general_quota(options = {})
+      remaining = obs_general_quota.to_i - get_obs_general_calls(options)
+      (remaining > 0 ? remaining : 0)
+    end
+
+    def signup_page_enabled
+      whitelisted_email_domains.present? && auth_enabled?
+    end
+
+    def auth_enabled?
+      auth_username_password_enabled || auth_google_enabled || auth_github_enabled || auth_saml_enabled?
+    end
+
+    def database_name
+      owner ? owner.database_name : nil
+    end
+
+    def create_group(display_name)
+      Carto::Group.create_group_with_extension(self, display_name)
+    end
+
+    def name_to_display
+      display_name.nil? ? name : display_name
+    end
+
+    def assigned_quota
+      self.users.sum(:quota_in_bytes).to_i
+    end
+
+    def unassigned_quota
+      self.quota_in_bytes - assigned_quota
+    end
+
+    def require_organization_owner_presence!
+      if owner.nil?
+        raise ::Organization::OrganizationWithoutOwner.new(self)
+      end
+    end
+
+    def auth_saml_enabled?
+      auth_saml_configuration.present?
+    end
+
+    def builder_users
+      users.reject(&:viewer)
+    end
+
+    def viewer_users
+      users.select(&:viewer)
+    end
+
+    def admin?(user)
+      user.belongs_to_organization?(self) && user.organization_admin?
+    end
+
+    def non_owner_users
+      users.reject { |u| owner && u.id == owner.id }
+    end
+
+    private
+
+    def ensure_auth_saml_configuration
+      self.auth_saml_configuration ||= Hash.new
+    end
+
+    def destroy_groups_with_extension
+      return unless groups
+
+      groups.map { |g| g.destroy_group_with_extension }
+
+      reload
+    end
+  end
+end

app/models/carto/overlay.rb

@@ -0,0 +1,75 @@
+require 'active_record'
+
+module Carto
+  class Overlay < ActiveRecord::Base
+    # INFO: disable ActiveRecord inheritance column
+    self.inheritance_column = :_type
+
+    belongs_to :visualization
+
+    serialize :options, JSON
+
+    validates :type, presence: true
+    validate :unique_overlay_not_duplicated
+    validate :validate_user_not_viewer
+
+    before_destroy :validate_user_not_viewer
+
+    after_save :invalidate_cache
+    after_destroy :invalidate_cache
+
+    # There can be at most one of this types per visualization
+    UNIQUE_TYPES = [
+      'header', 'search', 'layer_selector', 'share', 'zoom', 'logo', 'loader', 'fullscreen'
+    ].freeze
+
+    BUILDER_COMPATIBLE_TYPES = ['search', 'layer_selector', 'share', 'fullscreen', 'loader', 'logo', 'zoom'].freeze
+    scope :builder_incompatible, -> { where("type NOT IN ('#{BUILDER_COMPATIBLE_TYPES.join("','")}')") }
+
+    def hide
+      options['display'] = false
+      self
+    end
+
+    def show
+      options['display'] = true
+      self
+    end
+
+    def hidden?
+      !options['display']
+    end
+
+    private
+
+    def unique_overlay_not_duplicated
+      # This check probably belongs in visualization. See #6919
+      if UNIQUE_TYPES.include?(type)
+        other_overlay = Carto::Overlay.where(visualization_id: visualization_id, type: type)
+        other_overlay = other_overlay.where('id != ?', id) unless new_record?
+
+        unless other_overlay.first.nil?
+          errors.add(:base, "Unique overlay of type #{type} already exists")
+        end
+      end
+    end
+
+    def validate_user_not_viewer
+      # TODO: `visualization` check is needed because the creation of default overlays for visualization
+      # is yet done from the old models (Member). Member assigns the id (because it can't assign itself, as
+      # it's not a `Carto::Visualization`), but since that happens in a transaction managed by Sequel
+      # we can't get the `Carto::Visualization` here. When `Member` is gone the `visualization` check can
+      # be removed, as an `Overlay` must have a `Visualization`.
+      if visualization && visualization.user.viewer
+        errors.add(:visualization, "Viewer users can't edit overlays")
+        return false
+      end
+    end
+
+    def invalidate_cache
+      # Using `send` to avoid making it public. A future refactor should make this call unnecessary
+      # Visualization might not exist during creation, as the overlays are created before the visualization
+      visualization.try(:invalidate_after_commit)
+    end
+  end
+end

app/models/carto/paged_model.rb

@@ -0,0 +1,15 @@
+module Carto
+  module PagedModel
+
+    def paged(page = 1, per_page = 20, order = nil)
+      self.paged_association(self, page, per_page, order)
+    end
+
+    def self.paged_association(association, page = 1, per_page = 20, order = nil)
+      paged = association.offset((page - 1) * per_page).limit(per_page)
+      paged = paged.order(order) unless order.nil?
+      paged
+    end
+
+  end
+end

app/models/carto/permission.rb

@@ -0,0 +1,530 @@
+require 'active_record'
+require_dependency 'cartodb/errors'
+
+class Carto::Permission < ActiveRecord::Base
+  DEFAULT_ACL_VALUE = [].freeze
+
+  ACCESS_READONLY   = 'r'.freeze
+  ACCESS_READWRITE  = 'rw'.freeze
+  ACCESS_NONE       = 'n'.freeze
+  SORTED_ACCESSES   = [ACCESS_READWRITE, ACCESS_READONLY, ACCESS_NONE].freeze
+
+  TYPE_USER         = 'user'.freeze
+  TYPE_ORGANIZATION = 'org'.freeze
+  TYPE_GROUP        = 'group'.freeze
+
+  belongs_to :owner, -> { select(Carto::User::DEFAULT_SELECT) }, class_name: Carto::User
+  has_one :visualization, inverse_of: :permission, class_name: Carto::Visualization, foreign_key: :permission_id
+
+  validate :not_w_permission_to_viewers
+
+  after_update :update_changes
+  after_destroy :update_changes_deletion
+
+  def acl
+    JSON.parse(access_control_list, symbolize_names: true)
+  end
+
+  def user_has_read_permission?(user)
+    owner?(user) || permitted?(user, ACCESS_READONLY)
+  end
+
+  def user_has_write_permission?(user)
+    owner?(user) || permitted?(user, ACCESS_READWRITE)
+  end
+
+  # Explicitly remove columns from AR schema
+  DELETED_COLUMNS = ['entity_id', 'entity_type'].freeze
+  def self.columns
+    super.reject { |c| DELETED_COLUMNS.include?(c.name) }
+  end
+
+  def entities_with_read_permission
+    entities = acl.map do |entry|
+      entity_id = entry[:id]
+      case entry[:type]
+      when TYPE_USER
+        Carto::User.where(id: entity_id).first
+      when TYPE_ORGANIZATION
+        Carto::Organization.where(id: entity_id).first
+      when TYPE_GROUP
+        Carto::Group.where(id: entity_id).first
+      end
+    end
+    entities.compact.uniq
+  end
+
+  def entity_type
+    ENTITY_TYPE_VISUALIZATION
+  end
+
+  def entity_id
+    visualization.try(:id)
+  end
+
+  def is_owner?(subject)
+    owner_id == subject.id
+  end
+
+  # Format:
+  # [
+  #   {
+  #     type:         string,
+  #     entity:
+  #     {
+  #       id:         uuid,
+  #       username:   string,
+  #       avatar_url: string,   (optional)
+  #     },
+  #     access:       string
+  #   }
+  # ]
+  #
+  # type is from TYPE_xxxxx constants
+  # access is from ACCESS_xxxxx constants
+  #
+  # @param value Array
+  # @throws CartoDB::PermissionError
+  def acl=(value)
+    incoming_acl = value.nil? ? DEFAULT_ACL_VALUE : value
+    raise CartoDB::PermissionError.new('ACL is not an array') unless incoming_acl.is_a? Array
+    incoming_acl.map do |item|
+      unless item.is_a?(Hash) && acl_has_required_fields?(item) && acl_has_valid_access?(item)
+        raise CartoDB::PermissionError.new('Wrong ACL entry format')
+      end
+    end
+
+    acl_items = incoming_acl.map do |item|
+      {
+        type:   item[:type],
+        id:     item[:entity][:id],
+        access: item[:access]
+      }
+    end
+
+    cleaned_acl = acl_items.select { |i| i[:id] } # Cleaning, see #5668
+
+    if @old_acl.nil?
+      @old_acl = acl
+    end
+
+    self.access_control_list = ::JSON.dump(cleaned_acl)
+  end
+
+  def owner=(value)
+    self.owner_id = value.id
+    self.owner_username = value.username
+  end
+
+  def permission_for_user(user)
+    return ACCESS_READWRITE if owner?(user)
+
+    accesses = acl_entries_for_user(user).map do |entry|
+      entry[:access]
+    end
+
+    higher_access(accesses)
+  end
+
+  def permitted?(user, permission_type)
+    SORTED_ACCESSES.index(permission_for_user(user)) <= SORTED_ACCESSES.index(permission_type)
+  end
+
+  def entity
+    visualization
+  end
+
+  def notify_permissions_change(permissions_changes)
+    permissions_changes.each do |c, v|
+      # At the moment we just check users permissions
+      if c == TYPE_USER
+        v.each do |affected_id, perm|
+          # Perm is an array. For the moment just one type of permission can
+          # be applied to a type of object. But with an array this is open
+          # to more than one permission change at a time
+          perm.each do |p|
+            if visualization.derived?
+              if p['action'] == 'grant'
+                # At this moment just inform as read grant
+                if p['type'].include?('r')
+                  ::Resque.enqueue(::Resque::UserJobs::Mail::ShareVisualization, entity.id, affected_id)
+                end
+              elsif p['action'] == 'revoke'
+                if p['type'].include?('r')
+                  ::Resque.enqueue(::Resque::UserJobs::Mail::UnshareVisualization,
+                                   entity.name, owner_username, affected_id)
+                end
+              end
+            elsif visualization.canonical?
+              if p['action'] == 'grant'
+                # At this moment just inform as read grant
+                if p['type'].include?('r')
+                  ::Resque.enqueue(::Resque::UserJobs::Mail::ShareTable, entity.id, affected_id)
+                end
+              elsif p['action'] == 'revoke'
+                if p['type'].include?('r')
+                  ::Resque.enqueue(::Resque::UserJobs::Mail::UnshareTable, entity.name, owner_username, affected_id)
+                end
+              end
+            end
+          end
+        end
+      end
+    end
+  rescue => e
+    CartoDB::Logger.error(message: "Problem sending notification mail", exception: e)
+  end
+
+  def set_user_permission(subject, access)
+    set_subject_permission(subject.id, access, TYPE_USER)
+  end
+
+  def set_group_permission(group, access)
+    # You only want to switch it off when request is a permission request coming from database
+    @update_db_group_permission = false
+
+    granted_access = granted_access_for_group(group)
+    if granted_access == access
+      raise ModelAlreadyExistsError.new("Group #{group.name} already has #{access} access")
+    elsif granted_access == ACCESS_NONE
+      set_subject_permission(group.id, access, TYPE_GROUP)
+    else
+      # Remove group entry from acl in order to add new (or none if ACCESS_NONE)
+      new_acl = inputable_acl.reject { |entry| entry[:entity][:id] == group.id }
+
+      unless access == ACCESS_NONE
+        acl_entry = {
+          type: TYPE_GROUP,
+          entity: {
+            id: group.id,
+            name: group.name
+          },
+          access: access
+        }
+        new_acl << acl_entry
+      end
+
+      self.acl = new_acl
+    end
+  end
+
+  def remove_group_permission(group)
+    # You only want to switch it off when request is a permission request coming from database
+    @update_db_group_permission = false
+
+    set_group_permission(group, ACCESS_NONE)
+  end
+
+  def remove_user_permission(user)
+    granted_access = granted_access_for_user(user)
+    if granted_access != ACCESS_NONE
+      self.acl = inputable_acl.select { |entry| entry[:entity][:id] != user.id }
+    end
+  end
+
+  # acl write method expects entries to have entity, although they're not
+  # stored.
+  # TODO: fix this, since this is coupled to representation.
+  def inputable_acl
+    acl.map do |entry|
+      {
+        type: entry[:type],
+        entity: {
+          id: entry[:id],
+          avatar_url: '',
+          username: '',
+          name: ''
+        },
+        access: entry[:access]
+      }
+    end
+  end
+
+  private
+
+  def real_entity_type
+    entity.type
+  end
+
+  def not_w_permission_to_viewers
+    viewer_writers = users_with_permissions(ACCESS_READWRITE).select(&:viewer)
+    unless viewer_writers.empty?
+      errors.add(:access_control_list, "grants write to viewers: #{viewer_writers.map(&:username).join(',')}")
+    end
+  end
+
+  def users_with_permissions(access)
+    user_ids = relevant_user_acl_entries(acl).select { |e| access == e[:access] }.map { |e| e[:id] }
+    user_ids.empty? ? [] : Carto::User.where(id: user_ids).all
+  end
+
+  ENTITY_TYPE_VISUALIZATION = 'vis'.freeze
+
+  ALLOWED_ENTITY_KEYS = [:id, :username, :name, :avatar_url].freeze
+
+  def owner?(user)
+    owner_id == user.id
+  end
+
+  def higher_access(accesses)
+    return ACCESS_NONE if accesses.empty?
+    index = SORTED_ACCESSES.index do |access|
+      accesses.include?(access)
+    end
+    SORTED_ACCESSES[index]
+  end
+
+  def acl_entries_for_user(user)
+    acl.select do |entry|
+      (
+        acl_entry_is_for_user_id?(entry, user.id) ||
+        acl_entry_is_for_organization_id(entry, user.organization_id) ||
+        (!user.groups.nil? && acl_entry_is_for_a_user_group(entry, user.groups.map(&:id)))
+      )
+    end
+  end
+
+  def acl_entry_is_for_user_id?(entry, user_id)
+    entry[:type] == TYPE_USER && entry[:id] == user_id
+  end
+
+  def acl_entry_is_for_organization_id(entry, organization_id)
+    entry[:type] == TYPE_ORGANIZATION && entry[:id] == organization_id
+  end
+
+  def acl_entry_is_for_a_user_group(entry, group_ids)
+    entry[:type] == TYPE_GROUP && group_ids.include?(entry[:id])
+  end
+
+  def acl_has_required_fields?(acl_item)
+    acl_item[:entity].present? && acl_item[:type].present? && acl_item[:access].present? &&
+      acl_has_valid_entity_field?(acl_item)
+  end
+
+  def acl_has_valid_entity_field?(acl_item)
+    acl_item[:entity].keys - ALLOWED_ENTITY_KEYS == []
+  end
+
+  def acl_has_valid_access?(acl_item)
+    valid_access = [ACCESS_READONLY, ACCESS_NONE]
+    if entity.table?
+      valid_access << ACCESS_READWRITE
+    end
+    valid_access.include?(acl_item[:access])
+  end
+
+  def update_changes
+    if !@old_acl.nil?
+      notify_permissions_change(CartoDB::Permission.compare_new_acl(@old_acl, acl))
+      Carto::DbPermissionService.shared_entities_revokes(@old_acl, acl, entity.table) if entity.table?
+    end
+    update_shared_entities
+    # Notify change, caches should be invalidated
+    entity.table.update_cdb_tablemetadata if entity && entity.table
+  end
+
+  def update_changes_deletion
+    # Hack. I need to set the new acl as empty so all the old acls are
+    # considered revokes
+    # We need to pass the current acl as old_acl and the new_acl as something
+    # empty to recreate a revoke by deletion
+    notify_permissions_change(CartoDB::Permission.compare_new_acl(acl, []))
+  end
+
+  def update_shared_entities
+    e = entity
+    # First clean previous sharings
+    destroy_shared_entities
+    revoke_previous_permissions(e)
+
+    # Create user entities for the new ACL
+    users = relevant_user_acl_entries(acl)
+    # Avoid entries without recipient id. See #5668.
+    users.select { |u| u[:id] }.each do |user|
+      shared_entity = Carto::SharedEntity.create(
+        recipient_id:   user[:id],
+        recipient_type: Carto::SharedEntity::RECIPIENT_TYPE_USER,
+        entity_id:      entity.id,
+        entity_type:    Carto::SharedEntity::ENTITY_TYPE_VISUALIZATION
+      )
+
+      if e.table?
+        grant_db_permission(e, user[:access], shared_entity)
+      end
+    end
+
+    org = relevant_org_acl_entry(acl)
+    if org
+      shared_entity = Carto::SharedEntity.create(
+        recipient_id:   org[:id],
+        recipient_type: Carto::SharedEntity::RECIPIENT_TYPE_ORGANIZATION,
+        entity_id:      entity.id,
+        entity_type:    Carto::SharedEntity::ENTITY_TYPE_VISUALIZATION
+      )
+
+      if e.table?
+        grant_db_permission(e, org[:access], shared_entity)
+      end
+    end
+
+    groups = relevant_groups_acl_entries(acl)
+    groups.each do |group|
+      Carto::SharedEntity.create(
+        recipient_id:   group[:id],
+        recipient_type: Carto::SharedEntity::RECIPIENT_TYPE_GROUP,
+        entity_id:      entity.id,
+        entity_type:    Carto::SharedEntity::ENTITY_TYPE_VISUALIZATION
+      )
+
+      # You only want to switch it off when request is a permission request coming from database
+      if e.table? && @update_db_group_permission != false
+        Carto::Group.find(group[:id]).grant_db_permission(e.table, group[:access])
+      end
+    end
+
+    e.invalidate_after_commit
+  end
+
+  def destroy_shared_entities
+    Carto::SharedEntity.where(entity_id: entity.id).each(&:destroy)
+  end
+
+  def revoke_previous_permissions(entity)
+    users = relevant_user_acl_entries(@old_acl.nil? ? [] : @old_acl)
+    org = relevant_org_acl_entry(@old_acl.nil? ? [] : @old_acl)
+    groups = relevant_groups_acl_entries(@old_acl.nil? ? [] : @old_acl)
+    case entity.class.name
+    when Carto::Visualization.to_s
+      if entity.table
+        if org
+          entity.table.remove_organization_access
+        end
+        users.each do |user|
+          # Cleaning, see #5668
+          u = Carto::User.find(user[:id])
+          entity.table.remove_access(u) if u
+        end
+        # update_db_group_permission check is needed to avoid updating db requests
+        if @update_db_group_permission != false
+          groups.each do |group|
+            Carto::Group.find(group[:id]).grant_db_permission(entity.table, ACCESS_NONE)
+          end
+        end
+        check_related_visualizations(entity.table)
+      end
+    else
+      raise CartoDB::PermissionError.new("Unsupported entity type trying to grant permission: #{entity.class.name}")
+    end
+  end
+
+  def relevant_user_acl_entries(acl_list)
+    relevant_acl_entries(acl_list, TYPE_USER)
+  end
+
+  def relevant_org_acl_entry(acl_list)
+    relevant_acl_entries(acl_list, TYPE_ORGANIZATION).first
+  end
+
+  def relevant_groups_acl_entries(acl_list)
+    relevant_acl_entries(acl_list, TYPE_GROUP)
+  end
+
+  def relevant_acl_entries(acl_list, type)
+    acl_list.select { |entry| entry[:type] == type && entry[:access] != ACCESS_NONE }.map do |entry|
+      { id: entry[:id], access: entry[:access] }
+    end
+  end
+
+  def check_related_visualizations(table)
+    fully_dependent_visualizations = table.fully_dependent_visualizations.to_a
+    partially_dependent_visualizations = table.partially_dependent_visualizations.to_a
+    table_visualization = table.table_visualization
+    partially_dependent_visualizations.each do |visualization|
+      # check permissions, if the owner does not have permissions
+      # to see the table the layers using this table are removed
+      perm = visualization.permission
+      unless table_visualization.has_permission?(perm.owner, ACCESS_READONLY)
+        visualization.unlink_from(table)
+      end
+    end
+
+    fully_dependent_visualizations.each do |visualization|
+      # check permissions, if the owner does not have permissions
+      # to see the table the visualization is removed
+      perm = visualization.permission
+      unless table_visualization.has_permission?(perm.owner, ACCESS_READONLY)
+        visualization.delete
+      end
+    end
+  end
+
+  def grant_db_permission(entity, access, shared_entity)
+    if shared_entity.recipient_type == Carto::SharedEntity::RECIPIENT_TYPE_ORGANIZATION
+      permission_strategy = CartoDB::OrganizationPermission.new
+    else
+      u = Carto::User.find(shared_entity[:recipient_id])
+      permission_strategy = CartoDB::UserPermission.new(u)
+    end
+
+    case entity.class.name
+    when Carto::Visualization.to_s
+      # assert database permissions for non canonical tables are assigned
+      # its canonical vis
+      if not entity.table
+        raise CartoDB::PermissionError.new('Trying to change permissions to a table without ownership')
+      end
+      table = entity.table
+
+      # check ownership
+      if not owner_id == entity.permission.owner_id
+        raise CartoDB::PermissionError.new('Trying to change permissions to a table without ownership')
+      end
+      # give permission
+      if access == ACCESS_READONLY
+        permission_strategy.add_read_permission(table)
+      elsif access == ACCESS_READWRITE
+        permission_strategy.add_read_write_permission(table)
+      end
+    else
+      raise CartoDB::PermissionError.new('Unsupported entity type trying to grant permission')
+    end
+  end
+
+  def set_subject_permission(subject_id, access, type)
+    new_acl = inputable_acl
+
+    new_acl << {
+      type: type,
+      entity: {
+        id: subject_id,
+        avatar_url: '',
+        username: '',
+        name: ''
+      },
+      access: access
+    }
+
+    self.acl = new_acl
+  end
+
+  def granted_access_for_user(user)
+    granted_access_for_entry_type(TYPE_USER, user)
+  end
+
+  def granted_access_for_group(group)
+    granted_access_for_entry_type(TYPE_GROUP, group)
+  end
+
+  def granted_access_for_entry_type(type, entity)
+    permission = nil
+
+    acl.map do |entry|
+      if entry[:type] == type && entry[:id] == entity.id
+        permission = entry[:access]
+      end
+    end
+    permission = ACCESS_NONE if permission.nil?
+    permission
+  end
+
+end

app/models/carto/rate_limit.rb

@@ -0,0 +1,87 @@
+module Carto
+  class RateLimit < ActiveRecord::Base
+
+    RATE_LIMIT_ATTRIBUTES = [:maps_anonymous,
+                             :maps_static,
+                             :maps_static_named,
+                             :maps_dataview,
+                             :maps_dataview_search,
+                             :maps_analysis,
+                             :maps_tile,
+                             :maps_attributes,
+                             :maps_named_list,
+                             :maps_named_create,
+                             :maps_named_get,
+                             :maps_named,
+                             :maps_named_update,
+                             :maps_named_delete,
+                             :maps_named_tiles,
+                             :maps_analysis_catalog,
+                             :sql_query,
+                             :sql_query_format,
+                             :sql_job_create,
+                             :sql_job_get,
+                             :sql_job_delete,
+                             :sql_copy_from,
+                             :sql_copy_to].freeze
+
+    RATE_LIMIT_ATTRIBUTES.each { |attr| serialize attr, RateLimitValues }
+    RATE_LIMIT_ATTRIBUTES.each { |attr| validates attr, presence: true }
+
+    # FIXME remove this after syncing rate_limits
+    RATE_LIMIT_DEFAULTS = {
+      sql_copy_from: [1, 1, 60],
+      sql_copy_to: [1, 1, 60]
+    }.freeze
+    ###
+
+    def self.from_api_attributes(attributes)
+      rate_limit = RateLimit.new
+      rate_limit.rate_limit_attributes(attributes).each { |k, v| rate_limit[k] = RateLimitValues.new(v) }
+      rate_limit
+    end
+
+    def to_redis
+      result = {}
+      RATE_LIMIT_ATTRIBUTES.each do |key|
+        result[key.to_s.sub('_', ':')] = self[key].to_redis_array
+      end
+      result
+    end
+
+    def save_to_redis(user)
+      raise ActiveRecord::RecordInvalid.new(self) unless valid?
+      to_redis.each do |key, value|
+        $limits_metadata.DEL "limits:rate:store:#{user.username}:#{key}"
+        $limits_metadata.RPUSH "limits:rate:store:#{user.username}:#{key}", value
+      end
+    end
+
+    def api_attributes
+      RATE_LIMIT_ATTRIBUTES.map { |attr| [attr.to_sym, self[attr].to_array] }.to_h
+    end
+
+    def destroy_completely(user)
+      destroy
+      delete_from_redis(user)
+    end
+
+    def delete_from_redis(user)
+      to_redis.each_key do |key|
+        $limits_metadata.DEL "limits:rate:store:#{user.username}:#{key}"
+      end
+    end
+
+    def rate_limit_attributes(attrs = attributes)
+      attrs.with_indifferent_access.slice(*Carto::RateLimit::RATE_LIMIT_ATTRIBUTES)
+           .reverse_merge!(RATE_LIMIT_DEFAULTS) # FIXME remove this after syncing rate_limits
+    end
+
+    def ==(rate_limit)
+      super(rate_limit) ||
+        rate_limit_attributes.lazy.zip(rate_limit.rate_limit_attributes).all? do |x, y|
+          x == y
+        end
+    end
+  end
+end

app/models/carto/rate_limit_values.rb

@@ -0,0 +1,81 @@
+module Carto
+  class RateLimitValues
+    extend Forwardable
+    delegate [:<<,
+              :[],
+              :[]=,
+              :all,
+              :clear,
+              :each,
+              :empty?,
+              :flat_map,
+              :first,
+              :length,
+              :pop,
+              :present?,
+              :push,
+              :second] => :@rate_limits
+
+    VALUES_PER_RATE_LIMIT = 3
+
+    def initialize(values)
+      @rate_limits = []
+      values = convert_from_db_array(values)
+
+      values.each_slice(VALUES_PER_RATE_LIMIT) do |slice|
+        push(RateLimitValue.new(slice))
+      end
+    end
+
+    def to_array
+      flat_map(&:to_array)
+    end
+
+    def ==(other)
+      other.class == self.class && to_array == other.try(:to_array)
+    end
+
+    def to_redis_array
+      RateLimitValues.dump(self)
+    end
+
+    def self.dump(rate_limit_values)
+      return [] if rate_limit_values.nil?
+
+      rate_limit_values.flat_map(&:to_array)
+    end
+
+    def self.load(values)
+      RateLimitValues.new(values)
+    end
+
+    private
+
+    def convert_from_db_array(values)
+      return [] if values.nil? || values.empty?
+      return values.delete('{}').split(',') if values.is_a? String
+      values
+    end
+  end
+
+  class RateLimitValue
+    attr_accessor :max_burst, :count_per_period, :period
+
+    def initialize(values)
+      @values = values
+      self.max_burst, self.count_per_period, self.period = values.map(&:to_i) if valid?
+    end
+
+    def valid?
+      values_per_rate_limit = Carto::RateLimitValues::VALUES_PER_RATE_LIMIT
+      if !@values || @values.length < values_per_rate_limit && @values.length % values_per_rate_limit != 0
+        raise 'Error: Number of rate limits needs to be multiple of three'
+      end
+      true
+    end
+
+    def to_array
+      [max_burst, count_per_period, period]
+    end
+  end
+end

app/models/carto/received_notification.rb

@@ -0,0 +1,11 @@
+module Carto
+  class ReceivedNotification < ActiveRecord::Base
+    # autosave must be explicitly disabled due to https://github.com/rails/rails/issues/9336
+    # but we probably should not autosave from a ternary table anyway
+    belongs_to :user, inverse_of: :received_notifications, autosave: false
+    belongs_to :notification, inverse_of: :received_notifications, autosave: false
+    scope :unread, -> { where(read_at: nil).order('received_at DESC') }
+
+    delegate :icon, :body, to: :notification
+  end
+end

app/models/carto/search_tweet.rb

@@ -0,0 +1,28 @@
+require 'active_record'
+
+module Carto
+  class SearchTweet < ActiveRecord::Base
+
+    belongs_to :user, inverse_of: :search_tweets, class_name: 'Carto::User'
+    belongs_to :user_table, class_name: 'Carto::UserTable'
+    belongs_to :data_import, class_name: 'Carto::DataImport'
+
+    def self.twitter_imports_count(query, date_from, date_to)
+      query
+        .where('search_tweets.state' => ::SearchTweet::STATE_COMPLETE)
+        .where('search_tweets.created_at >= ? AND search_tweets.created_at <= ?', date_from, date_to + 1.days)
+        .sum("search_tweets.retrieved_items").to_i
+    end
+
+    def self.twitter_imports_count_by_date(query, date_from, date_to)
+      query
+        .where('search_tweets.state' => ::SearchTweet::STATE_COMPLETE)
+        .where('search_tweets.created_at >= ? AND search_tweets.created_at <= ?', date_from, date_to + 1.days)
+        .group("date_trunc('day', search_tweets.created_at)")
+        .select("date_trunc('day', search_tweets.created_at) as date, SUM(search_tweets.retrieved_items) as count")
+        .all
+        .map { |t| { t.date.to_date => t.count.to_i } }
+        .reduce({}, &:merge)
+    end
+  end
+end

app/models/carto/shared_entity.rb

@@ -0,0 +1,32 @@
+require 'active_record'
+require_relative '../shared_entity'
+
+module Carto
+  class SharedEntity < ActiveRecord::Base
+    ENTITY_TYPE_VISUALIZATION = 'vis'.freeze
+
+    RECIPIENT_TYPE_USER         = 'user'.freeze
+    RECIPIENT_TYPE_ORGANIZATION = 'org'.freeze
+    RECIPIENT_TYPE_GROUP        = 'group'.freeze
+
+    scope :shared_visualizations, -> { where(entity_type: ENTITY_TYPE_VISUALIZATION) }
+
+    belongs_to :visualization, foreign_key: :entity_id, inverse_of: :shared_entities
+
+    validates :recipient_id, :recipient_type, :entity_id, :entity_type, presence: true
+    validates :recipient_id, uniqueness: { scope: :entity_id, message: 'is already taken' }
+    validate :supported_type
+
+    attr_accessible :entity_id, :recipient_id, :entity_type, :recipient_type
+
+    def self.columns
+      super.reject { |c| c.name == "id" }
+    end
+
+    private
+
+    def supported_type
+      errors.add(:entity_type, 'unsupported type') unless self.entity_type == ENTITY_TYPE_VISUALIZATION
+    end
+  end
+end

app/models/carto/snapshot.rb

@@ -0,0 +1,15 @@
+require 'json'
+require_relative './carto_json_serializer'
+
+module Carto
+  class Snapshot < ActiveRecord::Base
+    belongs_to :visualization, class_name: Carto::Visualization
+    belongs_to :user, class_name: Carto::User
+
+    default_scope { order('created_at DESC') }
+
+    serialize :state, ::Carto::CartoJsonSymbolizerSerializer
+    validates :state, carto_json_symbolizer: true
+    validates :visualization, :user, presence: true
+  end
+end

app/models/carto/state.rb

@@ -0,0 +1,21 @@
+require 'json'
+require_relative './carto_json_serializer'
+
+class Carto::State < ActiveRecord::Base
+  belongs_to :visualization, class_name: Carto::Visualization
+
+  serialize :json, ::Carto::CartoJsonSymbolizerSerializer
+  validates :json, carto_json_symbolizer: true
+
+  after_initialize :ensure_json
+
+  def self.columns
+    super.reject { |c| c.name == 'user_id' }
+  end
+
+  private
+
+  def ensure_json
+    self.json ||= Hash.new
+  end
+end

app/models/carto/synchronization.rb

@@ -0,0 +1,39 @@
+require 'active_record'
+
+class Carto::Synchronization < ActiveRecord::Base
+
+  belongs_to :user
+  belongs_to :visualization, class_name: Carto::Visualization
+  belongs_to :log, class_name: Carto::Log
+
+  has_many :external_data_imports
+
+  STATE_CREATED   = 'created'
+  # Already at resque, waiting for slot
+  STATE_QUEUED   = 'queued'
+  # Actually syncing
+  STATE_SYNCING   = 'syncing'
+  STATE_SUCCESS   = 'success'
+  STATE_FAILURE   = 'failure'
+
+
+  def authorize?(user)
+    user.id == user_id && (!!user.sync_tables_enabled || from_external_source?)
+  end
+
+  def to_json(*args)
+    attributes.merge({from_external_source: from_external_source?}).to_json(*args)
+  end
+
+  def to_hash
+    attributes.merge({from_external_source: from_external_source?}).to_hash
+  end
+
+  def success?
+    state == STATE_SUCCESS
+  end
+
+  def from_external_source?
+    ::ExternalDataImport.where(synchronization_id: self.id).first != nil
+  end
+end

app/models/carto/synchronization_oauth.rb

@@ -0,0 +1,18 @@
+require 'active_record'
+
+module Carto
+  class SynchronizationOauth < ActiveRecord::Base
+
+    belongs_to :user, class_name: 'Carto::User', inverse_of: :synchronization_oauths
+
+    # TODO: partial duplication with DataImportsService (this one hasn't redis_storage)
+    def get_service_datasource
+      datasource = CartoDB::Datasources::DatasourcesFactory.get_datasource(service, user, {
+        http_timeout: ::DataImport.http_timeout_for(user)
+      })
+      datasource.token = token unless datasource.nil?
+      datasource
+    end
+
+  end
+end

app/models/carto/tag.rb

@@ -0,0 +1,10 @@
+require 'active_record'
+
+module Carto
+  class Tag < ActiveRecord::Base
+
+    belongs_to :user, class_name: Carto::User
+    belongs_to :user_table, class_name: Carto::UserTable, foreign_key: :table_id, primary_key: :table_id
+
+  end
+end

app/models/carto/template.rb

@@ -0,0 +1,75 @@
+require 'active_record'
+
+module Carto
+  class Template < ActiveRecord::Base
+
+    # INFO: On delete org will wipe out all templates
+    belongs_to :organization
+    # INFO: On delete of source visualization will wipe out all templates
+    belongs_to :visualization, primary_key: :source_visualization_id
+
+    validates :organization_id, presence: true
+    validates :source_visualization_id, presence: true
+    validates :title, presence: true
+
+    validate :source_visualization_referential_integrity
+    validate :required_tables_should_be_qualified
+    validate :required_tables_referential_integrity
+
+    # INFO: Needed as rails 3 + activerecord-postgres-array gem is still buggy
+    def required_tables
+      self.required_tables_list.split(',')
+    end
+
+    # INFO: Needed as rails 3 + activerecord-postgres-array gem is still buggy
+    def required_tables=(list=[])
+      self.required_tables_list = list.join(',')
+    end
+
+    def relates_to_table?(table)
+      required_tables.include?(table.qualified_table_name.gsub('"',''))
+    end
+
+    private
+
+    def source_visualization_referential_integrity
+      return if errors.keys.include?(:source_visualization_id)
+
+      visualization = Carto::Visualization.where(id: self.source_visualization_id).first
+      errors.add(:source_visualization_id, "Source visualization not found") if visualization.nil?
+
+      if visualization.user.organization_id != self.organization_id
+        errors.add(:source_visualization_id, "Source visualization not found")
+      end
+    end
+
+    def required_tables_should_be_qualified
+      wrong_table_names = required_tables.select { |table_name|
+          !::Table.is_qualified_name_valid?(table_name)
+        }
+      errors.add(:required_tables, "Invalid names: #{wrong_table_names.join(', ')}") if wrong_table_names.length > 0
+    end
+
+    def required_tables_referential_integrity
+      # If already have invalid names, don't run this validation
+      return if errors.keys.include?(:required_tables)
+
+      wrong_tables = required_tables.select { |qualified_name|
+          schema, table_name = qualified_name.split('.')
+          begin
+            user = Carto::User.where({ database_schema: schema, organization_id: self.organization_id}).first
+            Carto::VisualizationQueryBuilder.new
+                                            .with_name(table_name)
+                                            .with_user_id(user.id)
+                                            .build
+                                            .count == 0
+          rescue
+            true
+          end
+        }
+
+      errors.add(:required_tables, "Invalid tables: #{wrong_tables.join(', ')}") if wrong_tables.length > 0
+    end
+
+  end
+end

app/models/carto/user.rb

@@ -0,0 +1,819 @@
+require 'active_record'
+require 'cartodb-common'
+require_relative 'user_service'
+require_relative 'user_db_service'
+require_relative 'synchronization_oauth'
+require_relative '../../helpers/data_services_metrics_helper'
+require_dependency 'carto/helpers/auth_token_generator'
+require_dependency 'carto/helpers/has_connector_configuration'
+require_dependency 'carto/helpers/batch_queries_statement_timeout'
+require_dependency 'carto/helpers/billing_cycle'
+
+# TODO: This probably has to be moved as the service of the proper User Model
+class Carto::User < ActiveRecord::Base
+  extend Forwardable
+  include DataServicesMetricsHelper
+  include Carto::AuthTokenGenerator
+  include Carto::HasConnectorConfiguration
+  include Carto::BatchQueriesStatementTimeout
+  include Carto::BillingCycle
+
+  GEOCODING_BLOCK_SIZE = 1000
+  HERE_ISOLINES_BLOCK_SIZE = 1000
+  OBS_SNAPSHOT_BLOCK_SIZE = 1000
+  OBS_GENERAL_BLOCK_SIZE = 1000
+  MAPZEN_ROUTING_BLOCK_SIZE = 1000
+
+  STATE_ACTIVE = 'active'.freeze
+  STATE_LOCKED = 'locked'.freeze
+
+  # Make sure the following date is after Jan 29, 2015,
+  # which is the date where a message to accept the Terms and
+  # conditions and the Privacy policy was included in the Signup page.
+  # See https://github.com/CartoDB/cartodb-central/commit/3627da19f071c8fdd1604ddc03fb21ab8a6dff9f
+  FULLSTORY_ENABLED_MIN_DATE = Date.new(2017, 1, 1)
+  FULLSTORY_SUPPORTED_PLANS = ['FREE', 'PERSONAL30', 'Individual'].freeze
+
+  MAGELLAN_TRIAL_DAYS = 15
+  PERSONAL30_TRIAL_DAYS = 30
+  INDIVIDUAL_TRIAL_DAYS = 14
+  TRIAL_PLANS = ['personal30', 'individual'].freeze
+
+  # INFO: select filter is done for security and performance reasons. Add new columns if needed.
+  DEFAULT_SELECT = "users.email, users.username, users.admin, users.organization_id, users.id, users.avatar_url," \
+                   "users.api_key, users.database_schema, users.database_name, users.name, users.location," \
+                   "users.disqus_shortname, users.account_type, users.twitter_username, users.google_maps_key, " \
+                   "users.viewer, users.quota_in_bytes, users.database_host, users.crypted_password, " \
+                   "users.builder_enabled, users.private_tables_enabled, users.private_maps_enabled, " \
+                   "users.org_admin, users.last_name, users.google_maps_private_key, users.website, " \
+                   "users.description, users.available_for_hire, users.frontend_version, users.asset_host, "\
+                   "users.no_map_logo, users.industry, users.company, users.phone, users.job_role, "\
+                   "users.public_map_quota, users.maintenance_mode, users.company_employees, users.use_case".freeze
+
+  has_many :tables, class_name: Carto::UserTable, inverse_of: :user
+  has_many :visualizations, inverse_of: :user
+  has_many :maps, inverse_of: :user
+  has_many :layers_user
+  has_many :layers, through: :layers_user, after_add: Proc.new { |user, layer| layer.set_default_order(user) }
+
+  belongs_to :organization, inverse_of: :users
+  belongs_to :rate_limit
+  has_one :owned_organization, class_name: Carto::Organization, inverse_of: :owner, foreign_key: :owner_id
+  has_one :static_notifications, class_name: Carto::UserNotification, inverse_of: :user
+
+  has_many :feature_flags_user, dependent: :destroy, foreign_key: :user_id, inverse_of: :user
+  has_many :feature_flags, through: :feature_flags_user
+  has_many :assets, inverse_of: :user
+  has_many :data_imports, inverse_of: :user
+  has_many :geocodings, inverse_of: :user
+  has_many :synchronization_oauths, class_name: Carto::SynchronizationOauth, inverse_of: :user, dependent: :destroy
+  has_many :search_tweets, inverse_of: :user
+  has_many :synchronizations, inverse_of: :user
+  has_many :tags, inverse_of: :user
+  has_many :permissions, inverse_of: :owner, foreign_key: :owner_id
+  has_many :connector_configurations, inverse_of: :user, dependent: :destroy
+
+  has_many :client_applications, class_name: Carto::ClientApplication
+  has_many :oauth_tokens, class_name: Carto::OauthToken
+
+  has_many :users_group, dependent: :destroy, class_name: Carto::UsersGroup
+  has_many :groups, through: :users_group
+
+  has_many :received_notifications, inverse_of: :user
+
+  has_many :api_keys, inverse_of: :user
+  has_many :user_multifactor_auths, inverse_of: :user, class_name: Carto::UserMultifactorAuth
+
+  has_many :oauth_apps, inverse_of: :user, dependent: :destroy
+  has_many :oauth_app_users, inverse_of: :user, dependent: :destroy
+  has_many :granted_oauth_apps, through: :oauth_app_users, class_name: Carto::OauthApp, source: 'oauth_app'
+
+  delegate [
+    :database_username, :database_password, :in_database,
+    :db_size_in_bytes, :get_api_calls, :table_count, :public_visualization_count, :all_visualization_count,
+    :visualization_count, :owned_visualization_count, :twitter_imports_count,
+    :link_privacy_visualization_count, :password_privacy_visualization_count, :public_privacy_visualization_count
+  ] => :service
+
+  attr_reader :password
+
+  # TODO: From sequel, can be removed once finished
+  alias_method :maps_dataset, :maps
+  alias_method :layers_dataset, :layers
+  alias_method :assets_dataset, :assets
+  alias_method :data_imports_dataset, :data_imports
+  alias_method :geocodings_dataset, :geocodings
+
+  before_create :set_database_host
+  before_create :generate_api_key
+
+  after_save { reset_password_rate_limit if crypted_password_changed? }
+
+  after_destroy { rate_limit.destroy_completely(self) if rate_limit }
+  after_destroy :invalidate_varnish_cache
+
+  LOGIN_NOT_RATE_LIMITED = -1
+
+  MULTIFACTOR_AUTHENTICATION_ENABLED = 'enabled'.freeze
+  MULTIFACTOR_AUTHENTICATION_DISABLED = 'disabled'.freeze
+  MULTIFACTOR_AUTHENTICATION_NEEDS_SETUP = 'setup'.freeze
+
+  include ::VarnishCacheHandler
+
+  # Auto creates notifications on first access
+  def static_notifications_with_creation
+    static_notifications_without_creation || build_static_notifications(user: self, notifications: {})
+  end
+  alias_method_chain :static_notifications, :creation
+
+  def name_or_username
+    name.present? || last_name.present? ? [name, last_name].select(&:present?).join(' ') : username
+  end
+
+  def password_validator
+    if organization.try(:strong_passwords_enabled)
+      Carto::PasswordValidator.new(Carto::StrongPasswordStrategy.new)
+    else
+      Carto::PasswordValidator.new(Carto::StandardPasswordStrategy.new)
+    end
+  end
+
+  def password=(value)
+    return if !value.nil? && password_validator.validate(value, value, self).any?
+
+    @password = value
+    self.crypted_password = Carto::Common::EncryptionService.encrypt(password: value,
+                                                                     secret: Cartodb.config[:password_secret])
+  end
+
+  def reset_password_rate_limit
+    $users_metadata.DEL rate_limit_password_key if password_rate_limit_configured?
+  end
+
+  def rate_limit_password_key
+    "limits:password:#{username}"
+  end
+
+  def password_login_attempt
+    return LOGIN_NOT_RATE_LIMITED unless password_rate_limit_configured?
+
+    rate_limit = $users_metadata.call('CL.THROTTLE', rate_limit_password_key, @max_burst, @count, @period)
+
+    # it returns the number of seconds until the user should retry
+    # -1 means the action was allowed
+    # see https://github.com/brandur/redis-cell#response
+    rate_limit[3]
+  end
+
+  def password_confirmation=(password_confirmation)
+    # TODO: Implement
+  end
+
+  def default_avatar
+    "cartodb.s3.amazonaws.com/static/public_dashboard_default_avatar.png"
+  end
+
+  def feature_flag_names
+    @feature_flag_names ||= (feature_flags_user.map do |ff|
+                               ff.feature_flag.name
+                             end +
+                            FeatureFlag.where(restricted: false).map(&:name)).uniq.sort
+  end
+
+  # TODO: Revisit methods below to delegate to the service, many look like not proper of the model itself
+
+  def service
+    @service ||= Carto::UserService.new(self)
+  end
+
+  def db_service
+    @db_service ||= Carto::UserDBService.new(self)
+  end
+
+  #                             +--------+---------+------+
+  #       valid_privacy logic   | Public | Private | Link |
+  #   +-------------------------+--------+---------+------+
+  #   | private_tables_enabled  |    T   |    T    |   T  |
+  #   | !private_tables_enabled |    T   |    F    |   F  |
+  #   +-------------------------+--------+---------+------+
+  #
+  def valid_privacy?(privacy)
+    private_tables_enabled || privacy == Carto::UserTable::PRIVACY_PUBLIC
+  end
+
+  def default_dataset_privacy
+    Carto::UserTable::PRIVACY_VALUES_TO_TEXTS[default_table_privacy]
+  end
+
+  def default_table_privacy
+    private_tables_enabled ? Carto::UserTable::PRIVACY_PRIVATE : Carto::UserTable::PRIVACY_PUBLIC
+  end
+
+  # @return String public user url, which is also the base url for a given user
+  def public_url(subdomain_override = nil, protocol_override = nil)
+    base_subdomain = subdomain_override.nil? ? subdomain : subdomain_override
+    CartoDB.base_url(base_subdomain, CartoDB.organization_username(self), protocol_override)
+  end
+
+  def subdomain
+    if CartoDB.subdomainless_urls?
+      username
+    else
+      organization.nil? ? username : organization.name
+    end
+  end
+
+  def feature_flags_list
+    @feature_flag_names ||= (feature_flags_user
+                                 .map { |ff| ff.feature_flag.name } + FeatureFlag.where(restricted: false)
+                                                                                 .map(&:name)).uniq.sort
+  end
+
+  def has_feature_flag?(feature_flag_name)
+    feature_flags_list.present? && feature_flags_list.include?(feature_flag_name)
+  end
+
+  def has_organization?
+    !organization_id.nil?
+  end
+
+  def avatar
+    avatar_url.nil? ? "//#{default_avatar}" : avatar_url
+  end
+
+  def remove_logo?
+    has_organization? ? organization.no_map_logo? : no_map_logo?
+  end
+
+  def sql_safe_database_schema
+    database_schema.include?('-') ? "\"#{database_schema}\"" : database_schema
+  end
+
+  def database_public_username
+    database_schema == CartoDB::DEFAULT_DB_SCHEMA ? CartoDB::PUBLIC_DB_USER : "cartodb_publicuser_#{id}"
+  end
+
+  # returns google maps api key. If the user is in an organization and
+  # that organization has api key it's used
+  def google_maps_api_key
+    organization.try(:google_maps_key).blank? ? google_maps_key : organization.google_maps_key
+  end
+
+  def twitter_datasource_enabled
+    (read_attribute(:twitter_datasource_enabled) || organization.try(&:twitter_datasource_enabled)) && twitter_configured?
+  end
+
+  def twitter_configured?
+    # DatasourcesFactory.config_for takes configuration from organization if user is an organization user
+    CartoDB::Datasources::DatasourcesFactory.customized_config?(Search::Twitter::DATASOURCE_NAME, self)
+  end
+
+  # TODO: this is the correct name for what's stored in the model, refactor changing that name
+  alias_method :google_maps_query_string, :google_maps_api_key
+
+  # Returns the google maps private key. If the user is in an organization and
+  # that organization has a private key, the org's private key is returned.
+  def google_maps_private_key
+    if organization.try(:google_maps_private_key).blank?
+      read_attribute(:google_maps_private_key)
+    else
+      organization.google_maps_private_key
+    end
+  end
+
+  def google_maps_geocoder_enabled?
+    google_maps_private_key.present? && google_maps_client_id.present?
+  end
+
+  def google_maps_client_id
+    Rack::Utils.parse_nested_query(google_maps_query_string)['client'] if google_maps_query_string
+  end
+
+  # returns a list of basemaps enabled for the user
+  def basemaps
+    (Cartodb.config[:basemaps] || []).select { |group| group != 'GMaps' || google_maps_enabled? }
+  end
+
+  def google_maps_enabled?
+    google_maps_query_string.present?
+  end
+
+  # return the default basemap based on the default setting. If default attribute is not set, first basemaps is returned
+  # it only takes into account basemaps enabled for that user
+  def default_basemap
+    default = if google_maps_enabled? && basemaps['GMaps'].present?
+                ['GMaps', basemaps['GMaps']]
+              else
+                basemaps.find { |_, group_basemaps| group_basemaps.find { |_, attr| attr['default'] } }
+              end
+    default ||= basemaps.first
+    # return only the attributes
+    default[1].first[1]
+  end
+
+  def remaining_geocoding_quota(options = {})
+    remaining = if organization.present?
+                  organization.remaining_geocoding_quota(options)
+                else
+                  geocoding_quota - get_geocoding_calls(options)
+                end
+    (remaining > 0 ? remaining : 0)
+  end
+
+  def remaining_here_isolines_quota(options = {})
+    remaining = if organization.present?
+                  organization.remaining_here_isolines_quota(options)
+                else
+                  here_isolines_quota - get_here_isolines_calls(options)
+                end
+    (remaining > 0 ? remaining : 0)
+  end
+
+  def remaining_obs_snapshot_quota(options = {})
+    remaining = if organization.present?
+                  organization.remaining_obs_snapshot_quota(options)
+                else
+                  obs_snapshot_quota - get_obs_snapshot_calls(options)
+                end
+    (remaining > 0 ? remaining : 0)
+  end
+
+  def remaining_obs_general_quota(options = {})
+    remaining = if organization.present?
+                  organization.remaining_obs_general_quota(options)
+                else
+                  obs_general_quota - get_obs_general_calls(options)
+                end
+    (remaining > 0 ? remaining : 0)
+  end
+
+  def remaining_mapzen_routing_quota(options = {})
+    remaining = if organization.present?
+                  organization.remaining_mapzen_routing_quota(options)
+                else
+                  mapzen_routing_quota.to_i - get_mapzen_routing_calls(options)
+                end
+    (remaining > 0 ? remaining : 0)
+  end
+
+  def oauth_for_service(service)
+    synchronization_oauths.where(service: service).first
+  end
+
+  # INFO: don't use, use CartoDB::OAuths#add instead
+  def add_oauth(service, token)
+    # INFO: this should be the right way, but there's a problem with pgbouncer:
+    # ActiveRecord::StatementInvalid: PG::Error: ERROR:  prepared statement "a1" does not exist
+    # synchronization_oauths.create(
+    #    service:  service,
+    #    token:    token
+    # )
+    # INFO: even this fails eventually, th the same error. See https://github.com/CartoDB/cartodb/issues/4003
+    synchronization_oauth = Carto::SynchronizationOauth.new(
+      user_id: id,
+      service: service,
+      token: token
+    )
+    synchronization_oauth.save
+    synchronization_oauths.append(synchronization_oauth)
+    synchronization_oauth
+  end
+
+  def get_geocoding_calls(options = {})
+    date_from, date_to, orgwise = ds_metrics_parameters_from_options(options)
+    get_user_geocoding_data(self, date_from, date_to, orgwise)
+  end
+
+  def get_here_isolines_calls(options = {})
+    date_from, date_to, orgwise = ds_metrics_parameters_from_options(options)
+    get_user_here_isolines_data(self, date_from, date_to, orgwise)
+  end
+
+  def get_obs_snapshot_calls(options = {})
+    date_from, date_to, orgwise = ds_metrics_parameters_from_options(options)
+    get_user_obs_snapshot_data(self, date_from, date_to, orgwise)
+  end
+
+  def get_obs_general_calls(options = {})
+    date_from, date_to, orgwise = ds_metrics_parameters_from_options(options)
+    get_user_obs_general_data(self, date_from, date_to, orgwise)
+  end
+
+  def get_mapzen_routing_calls(options = {})
+    date_from, date_to, orgwise = ds_metrics_parameters_from_options(options)
+    get_user_mapzen_routing_data(self, date_from, date_to, orgwise)
+  end
+
+  # TODO: Remove unused param `use_total`
+  def remaining_quota(_use_total = false, db_size = service.db_size_in_bytes)
+    return nil unless db_size
+
+    quota_in_bytes - db_size
+  end
+
+  # can be nil table quotas
+  def remaining_table_quota
+    if table_quota.present?
+      remaining = table_quota - service.table_count
+      remaining < 0 ? 0 : remaining
+    end
+  end
+
+  def organization_user?
+    organization.present?
+  end
+
+  def belongs_to_organization?(organization)
+    organization_user? && !organization.nil? && organization_id == organization.id
+  end
+
+  def soft_geocoding_limit?
+    Carto::AccountType.new.soft_geocoding_limit?(self)
+  end
+  alias_method :soft_geocoding_limit, :soft_geocoding_limit?
+
+  def hard_geocoding_limit?
+    !soft_geocoding_limit?
+  end
+  alias_method :hard_geocoding_limit, :hard_geocoding_limit?
+
+  def soft_here_isolines_limit?
+    Carto::AccountType.new.soft_here_isolines_limit?(self)
+  end
+  alias_method :soft_here_isolines_limit, :soft_here_isolines_limit?
+
+  def hard_here_isolines_limit?
+    !soft_here_isolines_limit?
+  end
+  alias_method :hard_here_isolines_limit, :hard_here_isolines_limit?
+
+  def soft_obs_snapshot_limit?
+    Carto::AccountType.new.soft_obs_snapshot_limit?(self)
+  end
+  alias_method :soft_obs_snapshot_limit, :soft_obs_snapshot_limit?
+
+  def hard_obs_snapshot_limit?
+    !soft_obs_snapshot_limit?
+  end
+  alias_method :hard_obs_snapshot_limit, :hard_obs_snapshot_limit?
+
+  def soft_obs_general_limit?
+    Carto::AccountType.new.soft_obs_general_limit?(self)
+  end
+  alias_method :soft_obs_general_limit, :soft_obs_general_limit?
+
+  def hard_obs_general_limit?
+    !soft_obs_general_limit?
+  end
+  alias_method :hard_obs_general_limit, :hard_obs_general_limit?
+
+  def soft_twitter_datasource_limit?
+    soft_twitter_datasource_limit == true
+  end
+
+  def hard_twitter_datasource_limit?
+    !soft_twitter_datasource_limit?
+  end
+  alias_method :hard_twitter_datasource_limit, :hard_twitter_datasource_limit?
+
+  def soft_mapzen_routing_limit?
+    Carto::AccountType.new.soft_mapzen_routing_limit?(self)
+  end
+  alias_method :soft_mapzen_routing_limit, :soft_mapzen_routing_limit?
+
+  def hard_mapzen_routing_limit?
+    !soft_mapzen_routing_limit?
+  end
+  alias_method :hard_mapzen_routing_limit, :hard_mapzen_routing_limit?
+  def trial_ends_at
+    if account_type.to_s.casecmp('magellan').zero? && upgraded_at && upgraded_at + 15.days > Date.today
+      upgraded_at + MAGELLAN_TRIAL_DAYS.days
+    elsif account_type.to_s.casecmp('personal30').zero?
+      created_at + PERSONAL30_TRIAL_DAYS.days
+    elsif account_type.to_s.casecmp('individual').zero?
+      created_at + INDIVIDUAL_TRIAL_DAYS.days
+    end
+  end
+
+  def remaining_days_deletion
+    return nil unless state == STATE_LOCKED
+    begin
+      deletion_date = Cartodb::Central.new.get_user(username).fetch('scheduled_deletion_date', nil)
+      return nil unless deletion_date
+      (deletion_date.to_date - Date.today).to_i
+    rescue StandardError => e
+      CartoDB::Logger.warning(exception: e, message: 'Something went wrong calculating the number of remaining days for account deletion')
+      return nil
+    end
+  end
+
+  def viewable_by?(viewer)
+    id == viewer.id || organization.try(:admin?, viewer)
+  end
+
+  def editable_by?(user)
+    id == user.id || user.belongs_to_organization?(organization) && (user.organization_owner? || !organization_admin?)
+  end
+
+  # Some operations, such as user deletion, won't ask for password confirmation if password is not set (because of Google sign in, for example)
+  def needs_password_confirmation?
+    (!oauth_signin? || !last_password_change_date.nil?) &&
+      !created_with_http_authentication? &&
+      !organization.try(:auth_saml_enabled?)
+  end
+
+  def validate_old_password(old_password)
+    return true unless needs_password_confirmation?
+
+    Carto::Common::EncryptionService.verify(password: old_password, secure_password: crypted_password,
+                                            secret: Cartodb.config[:password_secret])
+  end
+
+  def valid_password_confirmation(password)
+    valid = validate_old_password(password)
+    errors.add(:password, 'Confirmation password sent does not match your current password') unless valid
+    valid
+  end
+
+  def valid_password?(key, value, confirmation_value)
+    password_validator.validate(value, confirmation_value, self).each { |e| errors.add(key, e) }
+    validate_password_not_in_use(nil, value, key)
+
+    errors[key].empty?
+  end
+
+  def validate_password_not_in_use(old_password = nil, new_password = nil, key = :new_password)
+    if password_in_use?(old_password, new_password)
+      errors.add(key, 'New password cannot be the same as old password')
+    end
+    errors[key].empty?
+  end
+
+  def password_in_use?(old_password = nil, new_password = nil)
+    return false if new_record?
+    return old_password == new_password if old_password
+
+    Carto::Common::EncryptionService.verify(password: new_password, secure_password: crypted_password_was,
+                                            secret: Cartodb.config[:password_secret])
+  end
+
+  alias_method :should_display_old_password?, :needs_password_confirmation?
+  alias_method :password_set?, :needs_password_confirmation?
+
+  def oauth_signin?
+    google_sign_in || github_user_id.present?
+  end
+
+  def created_with_http_authentication?
+    Carto::UserCreation.http_authentication.find_by_user_id(id).present?
+  end
+
+  def organization_owner?
+    organization && organization.owner_id == id
+  end
+
+  def organization_admin?
+    organization_user? && (organization_owner? || org_admin)
+  end
+
+  def mobile_sdk_enabled?
+    mobile_max_open_users > 0 || mobile_max_private_users > 0
+  end
+
+  def get_auth_tokens
+    tokens = [get_auth_token]
+
+    if has_organization?
+      tokens << organization.get_auth_token
+      tokens += groups.map(&:get_auth_token)
+    end
+
+    tokens
+  end
+
+  def get_auth_token
+    # Circumvent DEFAULT_SELECT, didn't add auth_token there for sercurity (presenters, etc)
+    auth_token = Carto::User.select(:auth_token).find(id).auth_token
+
+    auth_token || generate_and_save_auth_token
+  end
+
+  def notifications_for_category(category)
+    static_notifications.notifications[category] || {}
+  end
+
+  def builder_enabled?
+    if has_organization? && builder_enabled.nil?
+      organization.builder_enabled
+    else
+      !!builder_enabled
+    end
+  end
+
+  def engine_enabled?
+    if has_organization? && engine_enabled.nil?
+      organization.engine_enabled
+    else
+      !!engine_enabled
+    end
+  end
+
+  def new_visualizations_version
+    builder_enabled? ? 3 : 2
+  end
+
+  def can_change_email?
+    (!google_sign_in || last_password_change_date.present?) && !Carto::Ldap::Manager.new.configuration_present?
+  end
+
+  def can_change_password?
+    !Carto::Ldap::Manager.new.configuration_present?
+  end
+
+  def view_dashboard
+    update_column(:dashboard_viewed_at, Time.now)
+  end
+
+  # Special url that goes to Central if active (for old dashboard only)
+  def account_url(request_protocol)
+    request_protocol + CartoDB.account_host + CartoDB.account_path + '/' + username if CartoDB.account_host
+  end
+
+  # Special url that goes to Central if active
+  def plan_url(request_protocol)
+    account_url(request_protocol) + '/plan'
+  end
+
+  def relevant_frontend_version
+    frontend_version || CartoDB::Application.frontend_version
+  end
+
+  def cant_be_deleted_reason
+    if organization_owner?
+      "You can't delete your account because you are admin of an organization"
+    elsif Carto::UserCreation.http_authentication.where(user_id: id).first.present?
+      "You can't delete your account because you are using HTTP Header Authentication"
+    end
+  end
+
+  # Gets the list of OAuth accounts the user has (currently only used for synchronization)
+  # @return CartoDB::OAuths
+  def oauths
+    @oauths ||= CartoDB::OAuths.new(self)
+  end
+
+  def get_oauth_services
+    datasources = CartoDB::Datasources::DatasourcesFactory.get_all_oauth_datasources
+    array = []
+
+    datasources.each do |serv|
+      obj ||= Hash.new
+
+      title = ::User::OAUTH_SERVICE_TITLES.fetch(serv, serv)
+      revoke_url = ::User::OAUTH_SERVICE_REVOKE_URLS.fetch(serv, nil)
+      enabled = case serv
+                when 'gdrive'
+                  Cartodb.config[:oauth][serv]['client_id'].present?
+                when 'box'
+                  Cartodb.config[:oauth][serv]['client_id'].present?
+                when 'dropbox'
+                  Cartodb.config[:oauth]['dropbox']['app_key'].present?
+                when 'mailchimp'
+                  Cartodb.config[:oauth]['mailchimp']['app_key'].present? && has_feature_flag?('mailchimp_import')
+                when 'instagram'
+                  Cartodb.config[:oauth]['instagram']['app_key'].present? && has_feature_flag?('instagram_import')
+                else
+                  true
+                end
+
+      if enabled
+        oauth = oauths.select(serv)
+
+        obj['name'] = serv
+        obj['title'] = title
+        obj['revoke_url'] = revoke_url
+        obj['connected'] = !oauth.nil? ? true : false
+
+        array.push(obj)
+      end
+    end
+
+    array
+  end
+
+  def account_url(request_protocol)
+    if CartoDB.account_host
+      request_protocol + CartoDB.account_host + CartoDB.account_path + '/' + username
+    end
+  end
+
+  # Special url that goes to Central if active
+  def plan_url(request_protocol)
+    account_url(request_protocol) + '/plan'
+  end
+
+  def update_payment_url(request_protocol)
+    account_url(request_protocol) + '/update_payment'
+  end
+
+  def active?
+    state == STATE_ACTIVE
+  end
+
+  def locked?
+    state == STATE_LOCKED
+  end
+
+  def fullstory_enabled?
+    FULLSTORY_SUPPORTED_PLANS.include?(account_type) && created_at > FULLSTORY_ENABLED_MIN_DATE
+  end
+
+  def password_expired?
+    return false unless password_expiration_in_d && password_set?
+    password_date + password_expiration_in_d.days.to_i < Time.now
+  end
+
+  def password_expiration_in_d
+    organization_user? ? organization.password_expiration_in_d : Cartodb.get_config(:passwords, 'expiration_in_d')
+  end
+
+  def password_date
+    last_password_change_date || created_at
+  end
+
+  def send_password_reset!
+    generate_token(:password_reset_token)
+    self.password_reset_sent_at = Time.zone.now
+    save!
+
+    Resque.enqueue(::Resque::UserJobs::Mail::PasswordReset, id)
+  end
+
+  def multifactor_authentication_configured?
+    user_multifactor_auths.any?
+  end
+
+  def active_multifactor_authentication
+    user_multifactor_auths.order(created_at: :desc).first
+  end
+
+  def multifactor_authentication_status
+    if user_multifactor_auths.setup.any?
+      MULTIFACTOR_AUTHENTICATION_NEEDS_SETUP
+    elsif user_multifactor_auths.enabled.any?
+      MULTIFACTOR_AUTHENTICATION_ENABLED
+    else
+      MULTIFACTOR_AUTHENTICATION_DISABLED
+    end
+  end
+
+  def remaining_trial_days
+    return 0 unless trial_ends_at
+    ((trial_ends_at - Time.now) / 1.day).round
+  end
+
+  def trial_user?
+    TRIAL_PLANS.include?(account_type.to_s.downcase)
+  end
+
+  def get_database_roles
+    api_key_roles = api_keys.reject { |k| k.db_role =~ /^publicuser/ }.map(&:db_role)
+    oauth_app_owner_roles = api_keys.reject { |k| k.effective_ownership_role_name == nil }.map(&:effective_ownership_role_name)
+    (api_key_roles + oauth_app_owner_roles).uniq
+  end
+
+  private
+
+  def password_rate_limit_configured?
+    @max_burst ||= Cartodb.get_config(:passwords, 'rate_limit', 'max_burst')
+    @count ||= Cartodb.get_config(:passwords, 'rate_limit', 'count')
+    @period ||= Cartodb.get_config(:passwords, 'rate_limit', 'period')
+
+    [@max_burst, @count, @period].all?(&:present?)
+  end
+
+  def set_database_host
+    self.database_host ||= ::SequelRails.configuration.environment_for(Rails.env)['host']
+  end
+
+  def generate_api_key
+    self.api_key ||= make_token
+  end
+
+  def generate_token(column)
+    begin
+      self[column] = SecureRandom.urlsafe_base64
+    end while Carto::User.exists?(column => self[column])
+  end
+
+  def ds_metrics_parameters_from_options(options)
+    date_from = (options[:from] ? options[:from].to_date : last_billing_cycle)
+    date_to = (options[:to] ? options[:to].to_date : Date.today)
+    orgwise = options.fetch(:orgwise, true)
+    [date_from, date_to, orgwise]
+  end
+
+  def make_token
+    Carto::Common::EncryptionService.make_token
+  end
+end

app/models/carto/user_creation.rb

@@ -0,0 +1,350 @@
+require 'cartodb-common'
+require_dependency 'carto/user_authenticator'
+
+class Carto::UserCreation < ActiveRecord::Base
+
+  # Synced with CartoGearsApi::Events::UserCreationEvent
+  CREATED_VIA_SAML = 'saml'.freeze
+  CREATED_VIA_LDAP = 'ldap'.freeze
+  CREATED_VIA_ORG_SIGNUP = 'org_signup'.freeze
+  CREATED_VIA_API = 'api'.freeze
+  CREATED_VIA_HTTP_AUTENTICATION = 'http_authentication'.freeze
+
+  VALID_CREATED_VIA = [
+    CREATED_VIA_LDAP, CREATED_VIA_SAML, CREATED_VIA_ORG_SIGNUP,
+    CREATED_VIA_API, CREATED_VIA_HTTP_AUTENTICATION
+  ].freeze
+
+  IN_PROGRESS_STATES = [:initial, :enqueuing, :creating_user, :validating_user, :saving_user, :promoting_user, :load_common_data, :creating_user_in_central]
+  FINAL_STATES = [:success, :failure]
+
+  scope :http_authentication, -> { where(created_via: CREATED_VIA_HTTP_AUTENTICATION) }
+  scope :in_progress, -> { where(state: IN_PROGRESS_STATES) }
+
+  belongs_to :log, class_name: Carto::Log
+  belongs_to :user, class_name: Carto::User
+
+  after_create :use_invitation
+
+  def self.new_user_signup(user, created_via = CREATED_VIA_ORG_SIGNUP)
+    # Normal validation breaks state_machine method generation
+    raise 'User needs username' unless user.username
+    raise 'User needs email' unless user.email
+    raise "Not valid #{created_via}: #{VALID_CREATED_VIA.join(', ')}" unless VALID_CREATED_VIA.include?(created_via)
+
+    user_creation = Carto::UserCreation.new
+    user_creation.username = user.username
+    user_creation.email = user.email
+    user_creation.crypted_password = user.crypted_password
+    user_creation.organization_id = user.organization.nil? ? nil : user.organization.id
+    user_creation.quota_in_bytes = user.quota_in_bytes
+    user_creation.soft_geocoding_limit = user.soft_geocoding_limit
+    user_creation.soft_here_isolines_limit = user.soft_here_isolines_limit
+    user_creation.soft_obs_snapshot_limit = user.soft_obs_snapshot_limit
+    user_creation.soft_obs_general_limit = user.soft_obs_general_limit
+    user_creation.soft_twitter_datasource_limit = user.soft_twitter_datasource_limit.nil? ? false : user.soft_twitter_datasource_limit
+    user_creation.soft_mapzen_routing_limit = user.soft_mapzen_routing_limit
+    user_creation.google_sign_in = user.google_sign_in
+    user_creation.github_user_id = user.github_user_id
+    user_creation.log = Carto::Log.new_user_creation
+    user_creation.created_via = created_via
+    user_creation.viewer = user.viewer || false
+    user_creation.org_admin = user.org_admin || false
+    user_creation.last_password_change_date = user.last_password_change_date
+
+    user_creation
+  end
+
+  state_machine :state, :initial => :enqueuing do
+
+    before_transition any => any, :do => :log_transition_begin
+    after_transition any => any, :do => :log_transition_end
+
+    after_failure do
+      self.fail_user_creation
+    end
+
+    after_transition any => :creating_user, :do => :initialize_user
+    after_transition any => :validating_user, :do => :validate_user
+    after_transition any => :saving_user, :do => :save_user
+    after_transition any => :promoting_user, :do => :promote_user
+    after_transition any => :load_common_data, :do => :load_common_data
+    after_transition any => :creating_user_in_central, :do => :create_in_central
+
+    before_transition any => :success, :do => :close_creation
+    before_transition any => :failure, :do => :clean_user
+
+    event :next_creation_step do
+      transition :enqueuing => :creating_user,
+          :creating_user => :validating_user,
+          :validating_user => :saving_user,
+          :saving_user => :promoting_user
+
+      # This looks more complex than it actually is. The flow is always:
+      # promoting_user -> creating_user_in_central -> load_common_data -> success
+      #   creating_user_in_central is skipped if central is not configured
+      #   load_common_data is skipped for viewers
+      transition promoting_user: :creating_user_in_central, if: :sync_data_with_cartodb_central?
+      transition promoting_user: :load_common_data, unless: :viewer?
+      transition promoting_user: :success
+
+      transition creating_user_in_central: :load_common_data, unless: :viewer?
+      transition creating_user_in_central: :success
+
+      transition load_common_data: :success
+    end
+
+    event :fail_user_creation do
+      transition any => :failure
+    end
+
+    state all - [:success, :failure] do
+      def finished?
+        false
+      end
+    end
+
+    state :success, :failure do
+      def finished?
+        true
+      end
+    end
+
+  end
+
+  def set_owner_promotion(promote_to_organization_owner)
+    @promote_to_organization_owner = promote_to_organization_owner
+  end
+
+  def set_common_data_url(common_data_url)
+    @common_data_url = common_data_url
+  end
+
+  # TODO: Shorcut, search for a better solution to detect requirement
+  def requires_validation_email?
+    google_sign_in != true &&
+      !github_user_id &&
+      !has_valid_invitation? &&
+      !Carto::Ldap::Manager.new.configuration_present? &&
+      !created_via_api? &&
+      !created_via_http_authentication? &&
+      !created_via_saml?
+  end
+
+  def autologin?
+    state == 'success' && created_at > Time.now - 1.minute && enabled? && cartodb_user.dashboard_viewed_at.nil?
+  end
+
+  def subdomain
+    cartodb_user.subdomain
+  end
+
+  def with_invitation_token(invitation_token)
+    self.invitation_token = invitation_token
+    self
+  end
+
+  def created_via_api?
+    created_via == CREATED_VIA_API
+  end
+
+  def created_via_http_authentication?
+    created_via == CREATED_VIA_HTTP_AUTENTICATION
+  end
+
+  def created_via_saml?
+    created_via == CREATED_VIA_SAML
+  end
+
+  def has_valid_invitation?
+    return false unless invitation_token
+    !valid_invitation.nil?
+  end
+
+  def pertinent_invitation
+    @pertinent_invitation ||= select_valid_invitation_token(Carto::Invitation.query_with_unused_email(email).all)
+  end
+
+  private
+
+  def enabled?
+    cartodb_user.enable_account_token.nil? && cartodb_user.enabled
+  end
+
+  def valid_invitation
+    select_valid_invitation_token(Carto::Invitation.query_with_valid_email(email).all)
+  end
+
+  # Returns the first matching token invitation, and raises error if none is found
+  # but a token is set, since it might be fake
+  def select_valid_invitation_token(invitations)
+    return nil if invitations.empty?
+
+    invitation = invitations.select do |i|
+      i.token(email) == invitation_token &&
+      organization_id == i.organization_id
+    end.first
+
+    raise "Fake token sent for email #{email}, #{invitation_token}" if invitation_token && invitation.nil?
+
+    invitation
+  end
+
+  def cartodb_user
+    @cartodb_user ||= ::User.where(id: user_id).first
+  end
+
+  def log_transition_begin
+    log_transition('Beginning')
+  end
+
+  def log_transition_end
+    log_transition('End')
+  end
+
+  def log_transition(prefix)
+    self.log.append("#{prefix}: State: #{self.state}")
+  end
+
+  def initialize_user
+    @cartodb_user = ::User.new
+    @cartodb_user.username = username
+    @cartodb_user.email = email
+    @cartodb_user.crypted_password = crypted_password
+    @cartodb_user.google_sign_in = google_sign_in
+    @cartodb_user.github_user_id = github_user_id
+    @cartodb_user.invitation_token = invitation_token
+    @cartodb_user.enable_account_token = Carto::Common::EncryptionService.make_token if requires_validation_email?
+
+    unless organization_id.nil? || @promote_to_organization_owner
+      organization = ::Organization.where(id: organization_id).first
+      raise "Trying to copy organization settings from one without owner" if organization.owner.nil?
+      @cartodb_user.organization = organization
+      @cartodb_user.organization.owner.copy_account_features(@cartodb_user)
+    end
+
+    @cartodb_user.quota_in_bytes = quota_in_bytes unless quota_in_bytes.nil?
+    @cartodb_user.soft_geocoding_limit = soft_geocoding_limit unless soft_geocoding_limit.nil?
+    @cartodb_user.soft_here_isolines_limit = soft_here_isolines_limit unless soft_here_isolines_limit.nil?
+    @cartodb_user.soft_obs_snapshot_limit = soft_obs_snapshot_limit unless soft_obs_snapshot_limit.nil?
+    @cartodb_user.soft_obs_general_limit = soft_obs_general_limit unless soft_obs_general_limit.nil?
+    @cartodb_user.soft_twitter_datasource_limit = soft_twitter_datasource_limit unless soft_twitter_datasource_limit.nil?
+    @cartodb_user.soft_mapzen_routing_limit = soft_mapzen_routing_limit unless soft_mapzen_routing_limit.nil?
+    @cartodb_user.viewer = viewer if viewer
+    @cartodb_user.org_admin = org_admin if org_admin
+    @cartodb_user.last_password_change_date = last_password_change_date unless last_password_change_date.nil?
+
+    if pertinent_invitation
+      @cartodb_user.viewer = pertinent_invitation.viewer
+    end
+
+    @cartodb_user
+  rescue => e
+    handle_failure(e, mark_as_failure = true)
+    nil
+  end
+
+  # Central validation
+  def validate_user
+    @cartodb_user.validate_credentials_not_taken_in_central
+    raise "Credentials already used" unless @cartodb_user.errors.empty?
+  rescue => e
+    handle_failure(e, mark_as_failure = true)
+  end
+
+  def save_user
+    # INFO: until here we haven't user_id, so in-memory @cartodb_user is needed.
+    # After this, self.cartodb_user is used, which can be either @cartodb_user or loaded from database.
+    # This enables resuming.
+    @cartodb_user.save(raise_on_failure: true)
+    self.user_id = @cartodb_user.id
+    self.save
+  rescue => e
+    handle_failure(e, mark_as_failure = true)
+  end
+
+  def use_invitation
+    return unless invitation_token
+    invitation = pertinent_invitation
+    return unless invitation
+
+    invitation.use(email, invitation_token)
+  end
+
+  def promote_user
+    return unless @promote_to_organization_owner
+
+    organization = ::Organization.where(id: self.organization_id).first
+    raise "Trying to set organization owner when there's already one" unless organization.owner.nil?
+
+    user_organization = CartoDB::UserOrganization.new(organization_id, @cartodb_user.id)
+    user_organization.promote_user_to_admin
+    @cartodb_user.reload
+  rescue => e
+    handle_failure(e, mark_as_failure = true)
+  end
+
+  def load_common_data
+    @cartodb_user.load_common_data(@common_data_url) unless @common_data_url.nil?
+  rescue => e
+    handle_failure(e, mark_as_failure = false)
+  end
+
+  def create_in_central
+    cartodb_user.create_in_central
+  rescue => e
+    handle_failure(e, mark_as_failure = true)
+  end
+
+  def close_creation
+    clean_password
+    cartodb_user.notify_new_organization_user unless has_valid_invitation?
+    cartodb_user.organization.notify_if_disk_quota_limit_reached if cartodb_user.organization && !cartodb_user.viewer?
+    cartodb_user.organization.notify_if_seat_limit_reached if cartodb_user.organization && !cartodb_user.viewer?
+    CartoGearsApi::Events::EventManager.instance.notify(
+      CartoGearsApi::Events::UserCreationEvent.new(created_via, cartodb_user)
+    )
+  rescue => e
+    handle_failure(e, mark_as_failure = false)
+  end
+
+  def handle_failure(e, mark_as_failure)
+    CartoDB.notify_exception(e, { user_creation: self, mark_as_failure: mark_as_failure })
+    self.log.append("Error on state #{self.state}, mark_as_failure: #{mark_as_failure}. Error: #{e.message}")
+    self.log.append(e.backtrace.join("\n"))
+
+    process_failure if mark_as_failure
+  end
+
+  def process_failure
+    self.state = 'failure'
+    self.save
+    self.fail_user_creation
+  end
+
+  def clean_user
+    return unless cartodb_user && !cartodb_user.id.nil?
+
+    begin
+      cartodb_user.destroy
+    rescue => e
+      CartoDB.notify_exception(e, action: 'safe user destruction', user: cartodb_user)
+      begin
+        cartodb_user.delete
+      rescue => ee
+        CartoDB.notify_exception(ee, action: 'safe user deletion', user: cartodb_user)
+      end
+
+    end
+  end
+
+  def clean_password
+    self.crypted_password = ''
+    self.save
+  end
+
+  # INFO: state_machine needs guard methods to be instance methods
+  def sync_data_with_cartodb_central?
+    Cartodb::Central.sync_data_with_cartodb_central?
+  end
+end

app/models/carto/user_db_service.rb

@@ -0,0 +1,175 @@
+module Carto
+  class UserDBService
+    # Also default schema for new users
+    SCHEMA_PUBLIC = 'public'.freeze
+    SCHEMA_CARTODB = 'cartodb'.freeze
+    SCHEMA_IMPORTER = 'cdb_importer'.freeze
+    SCHEMA_CDB_DATASERVICES_API = 'cdb_dataservices_client'.freeze
+
+    def initialize(user)
+      @user = user
+    end
+
+    def build_search_path(user_schema = nil, quote_user_schema = true)
+      user_schema ||= @user.database_schema
+      UserDBService.build_search_path(user_schema, quote_user_schema)
+    end
+
+    # Centralized method to provide the (ordered) search_path
+    def self.build_search_path(user_schema, quote_user_schema = true)
+      # TODO Add SCHEMA_CDB_GEOCODER when we open the geocoder API to all the people
+      quote_char = quote_user_schema ? "\"" : ""
+      "#{quote_char}#{user_schema}#{quote_char}, #{SCHEMA_CARTODB}, #{SCHEMA_CDB_DATASERVICES_API}, #{SCHEMA_PUBLIC}"
+    end
+
+    def public_user_roles
+      @user.organization_user? ? [CartoDB::PUBLIC_DB_USER, @user.service.database_public_username] : [CartoDB::PUBLIC_DB_USER]
+    end
+
+    # Execute a query in the user database
+    # @param [String] query, using $1, $2 ... for placeholders
+    # @param ... values for the placeholders
+    # @return [Array<Hash<String, Value>>] Something ({ActiveRecord::Result} in this case) that behaves like an Array
+    #                                      of Hashes that map column name (as string) to value
+    # @raise [PG::Error] if the query fails
+    def execute_in_user_database(query, *binds)
+      @user.in_database.exec_query(query, 'ExecuteUserDb', binds.map { |v| [nil, v] })
+    rescue ActiveRecord::StatementInvalid => exception
+      raise exception.cause
+    end
+
+    def tables_effective(schema = 'public')
+      query = "select table_name::text from information_schema.tables where table_schema = '#{schema}'"
+      execute_in_user_database(query).map { |i| i['table_name'] }
+    end
+
+    def all_user_roles
+      roles = [@user.database_username]
+      if @user.organization_user?
+        roles << organization_member_group_role_member_name
+        roles += @user.groups.map(&:database_role)
+      end
+
+      roles
+    end
+
+    def all_tables_granted(role = nil)
+      roles_str = role ? "'#{role}'" : all_user_roles.map { |r| "'#{r}'" }.join(',')
+      query = %{
+        SELECT
+          s.nspname as schema,
+          c.relname as t,
+          string_agg(lower(acl.privilege_type), ',') as permission
+        FROM
+          pg_class c
+          JOIN pg_namespace s ON c.relnamespace = s.oid
+          JOIN LATERAL aclexplode(COALESCE(c.relacl, acldefault('r'::"char", c.relowner))) acl ON TRUE
+          JOIN pg_roles r ON acl.grantee = r.oid
+        WHERE
+          r.rolname IN (#{roles_str}) AND
+          s.nspname NOT IN ('cartodb', 'cdb', 'cdb_importer')
+        GROUP BY schema, t;
+      }
+
+      @user.in_database(as: :superuser) do |database|
+        database.execute(query)
+      end
+    end
+
+    def all_tables_granted_hashed(role = nil)
+      results = all_tables_granted(role)
+      privileges_hashed = {}
+
+      if !results.nil?
+        results.each do |row|
+          privileges_hashed[row['schema']] = {} if privileges_hashed[row['schema']].nil?
+          privileges_hashed[row['schema']][row['t']] = row['permission'].split(',')
+        end
+      end
+
+      privileges_hashed
+    end
+
+    def all_schemas_granted(role)
+      roles_str = role ? role : all_user_roles.join(',')
+      permissions = 'create,usage'
+      query = %{
+        WITH
+          roles AS (
+            SELECT unnest('{#{roles_str}}'::text[]) AS rname
+          ),
+          permissions AS (
+            SELECT 'SCHEMA' AS ptype, unnest('{#{permissions}}'::text[]) AS pname
+          ),
+          schemas AS (
+            SELECT schema_name AS sname
+            FROM information_schema.schemata
+            WHERE schema_name !~ '^pg_'
+              AND schema_name NOT IN ('cartodb', 'cdb', 'cdb_importer')
+          ),
+          schemas_roles_permissions AS (
+          SELECT
+              permissions.ptype,
+              schemas.sname AS obj_name,
+              roles.rname,
+              permissions.pname,
+              has_schema_privilege(roles.rname, schemas.sname, permissions.pname) AS has_permission
+          FROM
+            schemas
+            CROSS JOIN roles
+            CROSS JOIN permissions
+          WHERE
+            permissions.ptype = 'SCHEMA'
+          ),
+          schemas_and_grants AS (
+            SELECT obj_name AS object_name,
+            COALESCE(string_agg(DISTINCT CASE WHEN has_permission THEN pname END, ','), '') AS granted_permissions
+            FROM schemas_roles_permissions
+            GROUP BY 1
+            ORDER BY 1
+          )
+          SELECT
+            object_name, granted_permissions
+          FROM
+            schemas_and_grants
+          WHERE
+            granted_permissions is not null and granted_permissions <> '';
+      }
+
+      @user.in_database(as: :superuser) do |database|
+        database.execute(query)
+      end
+    end
+
+    def all_schemas_granted_hashed(role = nil)
+      results = all_schemas_granted(role)
+      return {} if results.nil?
+
+      results.map { |row| [row['object_name'], row['granted_permissions'].split(',')] }.to_h
+    end
+
+    def exists_role?(rolname)
+      query = %{
+        SELECT 1
+        FROM   pg_catalog.pg_roles
+        WHERE  rolname = '#{rolname}'
+      }
+
+      result = @user.in_database(as: :superuser).execute(query)
+      result.count > 0
+    end
+
+    def organization_member_group_role_member_name
+      query = "SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() as org_member_role;"
+      execute_in_user_database(query).first['org_member_role']
+    end
+
+    def create_oauth_reassign_ownership_event_trigger
+      @user.in_database(as: :superuser).execute('SELECT CDB_EnableOAuthReassignTablesTrigger()')
+    end
+
+    def drop_oauth_reassign_ownership_event_trigger
+      @user.in_database(as: :superuser).execute('SELECT CDB_DisableOAuthReassignTablesTrigger()')
+    end
+  end
+end

app/models/carto/user_migration_export.rb

@@ -0,0 +1,132 @@
+require 'active_record'
+require 'fileutils'
+require_relative '../../../services/user-mover/export_user'
+require_dependency 'file_upload'
+require_dependency 'resque/user_migration_jobs'
+require_dependency 'carto/ghost_tables_manager'
+
+module Carto
+  class UserMigrationExport < ::ActiveRecord::Base
+    belongs_to :organization, class_name: Carto::Organization
+    belongs_to :user, class_name: Carto::User
+    belongs_to :log, class_name: Carto::Log
+
+    STATE_PENDING = 'pending'.freeze
+    STATE_EXPORTING = 'exporting'.freeze
+    STATE_UPLOADING = 'uploading'.freeze
+    STATE_COMPLETE = 'complete'.freeze
+    STATE_FAILURE = 'failure'.freeze
+    VALID_STATES = [STATE_PENDING, STATE_EXPORTING, STATE_UPLOADING, STATE_COMPLETE, STATE_FAILURE].freeze
+
+    after_initialize :set_defaults
+
+    validates :state, inclusion: { in: VALID_STATES }
+    validate  :user_or_organization_present
+    validate  :validate_export_data
+
+    def run_export
+      check_valid_user(user) if user && export_metadata
+      check_valid_organization(organization) if organization && export_metadata
+
+      log.append("=== Exporting #{organization ? 'user' : 'org'} data ===")
+      update_attributes(state: STATE_EXPORTING)
+
+      package = if backup
+                  UserMigrationPackage.for_backup("backup_#{user.username}_#{Time.now.iso8601}", log)
+                else
+                  UserMigrationPackage.for_export(id, log)
+                end
+
+      export_job = CartoDB::DataMover::ExportJob.new(export_job_arguments(package.data_dir)) if export_data?
+
+      run_metadata_export(package.meta_dir) if export_metadata?
+
+      log.append("=== Uploading ===")
+      update_attributes(
+        state: STATE_UPLOADING,
+        json_file: export_data? ? "#{id}/#{export_job.json_file}" : 'no_data_exported'
+      )
+      uploaded_path = package.upload
+
+      state = uploaded_path.present? ? STATE_COMPLETE : STATE_FAILURE
+      log.append("=== Finishing. State: #{state}. File: #{uploaded_path} ===")
+      update_attributes(state: state, exported_file: uploaded_path)
+
+      true
+    rescue => e
+      log.append_exception('Exporting', exception: e)
+      CartoDB::Logger.error(exception: e, message: 'Error exporting user data', job: inspect)
+      update_attributes(state: STATE_FAILURE)
+
+      false
+    ensure
+      package.try(:cleanup)
+    end
+
+    def enqueue
+      Resque.enqueue(Resque::UserMigrationJobs::Export, export_id: id)
+    end
+
+    private
+
+    def check_valid_user(user)
+      Carto::GhostTablesManager.new(user.id).link_ghost_tables_synchronously
+    end
+
+    def check_valid_organization(organization)
+      organization.users.each do |user|
+        check_valid_user(user)
+      end
+    end
+
+    def run_metadata_export(meta_dir)
+      if organization
+        Carto::OrganizationMetadataExportService.new.export_to_directory(organization, meta_dir)
+      elsif user
+        Carto::UserMetadataExportService.new.export_to_directory(user, meta_dir)
+      else
+        raise 'Unrecognized export type for exporting metadata'
+      end
+    end
+
+    def export_job_arguments(data_dir)
+      args = if user.present?
+               {
+                 id: user.username,
+                 schema_mode: false
+               }
+             else
+               {
+                 organization_name: organization.name,
+                 schema_mode: true,
+                 split_user_schemas: false
+               }
+             end
+
+      args.merge(
+        path: data_dir,
+        job_uuid: id,
+        export_job_logger: log.logger,
+        logger: log.logger,
+        metadata: false,
+        data: export_data?
+      )
+    end
+
+    def user_or_organization_present
+      unless (user.present? && organization.blank?) || (organization.present? && user.blank?)
+        errors.add(:user, 'exactly one user or organization required')
+      end
+    end
+
+    def validate_export_data
+      errors.add(:export_metadata, 'needs to be true if export_data is set to false') if !export_data? && !export_metadata?
+    end
+
+    def set_defaults
+      self.log = Carto::Log.create(type: 'user_migration_export') unless log
+      self.state = STATE_PENDING unless state
+      save
+    end
+  end
+end

app/models/carto/user_migration_import.rb

@@ -0,0 +1,228 @@
+require 'active_record'
+require 'fileutils'
+require_relative '../../../services/user-mover/import_user'
+require_dependency 'resque/user_migration_jobs'
+require_dependency 'carto/user_metadata_export_service'
+require_dependency 'carto/organization_metadata_export_service'
+
+module Carto
+  class UserMigrationImport < ::ActiveRecord::Base
+    belongs_to :organization, class_name: Carto::Organization
+    belongs_to :user, class_name: Carto::User
+    belongs_to :log, class_name: Carto::Log
+
+    STATE_PENDING = 'pending'.freeze
+    STATE_DOWNLOADING = 'downloading'.freeze
+    STATE_IMPORTING = 'importing'.freeze
+    STATE_COMPLETE = 'complete'.freeze
+    STATE_FAILURE = 'failure'.freeze
+    VALID_STATES = [STATE_PENDING, STATE_DOWNLOADING, STATE_IMPORTING, STATE_COMPLETE, STATE_FAILURE].freeze
+
+    after_initialize :set_defaults
+
+    validates :state, inclusion: { in: VALID_STATES }
+    validates :database_host, presence: true
+    validates :exported_file, presence: true
+    validates :json_file, presence: true
+    validate :valid_org_import
+    validate :valid_dry_settings
+    validate :validate_import_data
+
+    def run_import
+      raise errors.full_messages.join(', ') unless valid?
+
+      log.append('=== Downloading ===')
+      update_attributes(state: STATE_DOWNLOADING)
+      package = UserMigrationPackage.for_import(id, log)
+      package.download(exported_file)
+
+      log.append('=== Importing ===')
+      update_attributes(state: STATE_IMPORTING)
+
+      service = (org_import? ? Carto::OrganizationMetadataExportService : Carto::UserMetadataExportService).new
+      import(service, package)
+
+      log.append('=== Complete ===')
+      update_attributes(state: STATE_COMPLETE)
+    rescue StandardError => e
+      puts "ERROR: #{e}"
+      log.append_exception('Importing', exception: e)
+      CartoDB::Logger.error(exception: e, message: 'Error importing user data', job: inspect)
+      update_attributes(state: STATE_FAILURE)
+      false
+    ensure
+      package.try(:cleanup)
+    end
+
+    def enqueue
+      Resque.enqueue(Resque::UserMigrationJobs::Import, import_id: id)
+    end
+
+    private
+
+    def valid_org_import
+      if org_import?
+        errors.add(:user_id, "user_id can't be present") if user_id.present?
+      else
+        errors.add(:organization_id, "organization_id can't be present") if organization_id.present?
+      end
+    end
+
+    def valid_dry_settings
+      errors.add(:dry, 'dry cannot be true while import_metadata is true') if import_metadata && dry
+    end
+
+    def validate_import_data
+      errors.add(:import_metadata, 'needs to be true if export_data is set to false') if !import_data? && !import_metadata?
+    end
+
+    def import(service, package)
+      if import_data?
+        import_job = CartoDB::DataMover::ImportJob.new(import_job_arguments(package.data_dir))
+        raise "DB already exists at DB host" if import_job.db_exists?
+      end
+
+      imported = do_import_metadata(package, service) if import_metadata?
+
+      begin
+        do_import_data(import_job) if import_data?
+      rescue => e
+        log.append('=== Error importing data. Rollback! ===')
+        rollback_import_data(package)
+        service.rollback_import_from_directory(package.meta_dir) if import_metadata?
+        raise e
+      end
+
+      import_visualizations(imported, package, service) if import_metadata?
+
+      reconfigure_dataservices if import_metadata?
+      reconfigure_aggregation_tables if import_metadata?
+    end
+
+    def do_import_metadata(package, service)
+      log.append('=== Importing metadata ===')
+      begin
+        imported = service.import_from_directory(package.meta_dir)
+      rescue UserAlreadyExists, OrganizationAlreadyExists => e
+        log.append('Organization already exists. Skipping!')
+        raise e
+      rescue => e
+        log.append('=== Error importing metadata. Rollback! ===')
+        service.rollback_import_from_directory(package.meta_dir)
+        raise e
+      end
+      org_import? ? self.organization = imported : self.user = imported
+      update_database_host
+      save!
+      imported
+    end
+
+    def do_import_data(import_job)
+      log.append('=== Importing data ===')
+      begin
+        import_job.run!
+      ensure
+        import_job.terminate_connections
+      end
+    end
+
+    def reconfigure_dataservices
+      if org_import?
+        ::Organization[organization.id].owner.db_service.install_and_configure_geocoder_api_extension
+      else
+        ::User[user.id].db_service.install_and_configure_geocoder_api_extension
+      end
+    end
+
+    def reconfigure_aggregation_tables
+      u = org_import? ? ::Organization[organization.id].owner : ::User[user.id]
+      begin
+        u.db_service.connect_to_aggregation_tables
+      rescue => e
+        CartoDB::Logger.error(message: "Error trying to refresh aggregation tables for user",
+                              exception: e,
+                              user: u,
+                              organization: (org_import? ? organization : nil),
+                              user_migration_import_id: id)
+      end
+    end
+
+    def import_visualizations(imported, package, service)
+      log.append('=== Importing visualizations and search tweets ===')
+      begin
+        ActiveRecord::Base.transaction do
+          service.import_metadata_from_directory(imported, package.meta_dir)
+        end
+      rescue => e
+        log.append('=== Error importing visualizations and search tweets. Rollback! ===')
+        rollback_import_data(package)
+        service.rollback_import_from_directory(package.meta_dir)
+        raise e
+      end
+    end
+
+    def rollback_import_data(package)
+      org_import? ? self.organization = nil : self.user = nil
+      save!
+      return unless import_data?
+
+      import_job = CartoDB::DataMover::ImportJob.new(
+        import_job_arguments(package.data_dir).merge(rollback: true,
+                                                     mode: :rollback,
+                                                     drop_database: true,
+                                                     drop_roles: true)
+      )
+
+      import_job.run!
+      import_job.terminate_connections
+    rescue => e
+      log.append('There was an error while rolling back import data:' + e.to_s)
+    end
+
+    def update_database_host
+      users.each do |user|
+        user.database_host = database_host
+        user.save!
+        # This is because Sequel models are being cached along request. This forces reload.
+        # It's being used in visualizations_export_persistence_service.rb#save_import
+        ::User[user.id].reload
+      end
+    end
+
+    def users
+      org_import? ? organization.users : [user]
+    end
+
+    def import_only_data?
+      # If the destination user/org already exists, import only data (same cloud, different DBs)
+      org_import? ? organization.present? : user.present?
+    end
+
+    def import_job_arguments(data_dir)
+      export_file = json_file.split('/').last
+
+      {
+        job_uuid: id,
+        file: "#{data_dir}/#{export_file}",
+        data: true,
+        metadata: false,
+        host: database_host,
+        rollback: false,
+        # This is used to import a non-org user into an organization. It is untested and unsupported.
+        # Disabling it unconditionally until we need it makes sense.
+        # into_org_name: org_import? || organization.nil? ? nil : organization.name,
+        into_org_name: nil,
+        mode: :import,
+        logger: log.logger,
+        import_job_logger: log.logger,
+        update_metadata: !dry
+      }
+    end
+
+    def set_defaults
+      self.log = Carto::Log.create(type: 'user_migration_import') unless log
+      self.state = STATE_PENDING unless state
+      save
+    end
+  end
+end

app/models/carto/user_migration_package.rb

@@ -0,0 +1,130 @@
+module Carto
+  class UserMigrationPackage
+    private_class_method :new
+
+    def self.for_export(id, log)
+      base_dir = Cartodb.get_config(:user_migrator, 'user_exports_folder')
+      instance = new(base_dir, id, log, Cartodb.get_config(:user_migrator))
+
+      FileUtils.mkdir_p(instance.data_dir)
+      FileUtils.mkdir_p(instance.meta_dir)
+
+      instance
+    end
+
+    def self.for_import(id, log)
+      base_dir = Cartodb.get_config(:user_migrator, 'user_imports_folder')
+      instance = new(base_dir, id, log, Cartodb.get_config(:user_migrator))
+
+      FileUtils.mkdir_p(instance.work_dir)
+
+      instance
+    end
+
+    def self.for_backup(id, log)
+      base_dir = Cartodb.get_config(:user_backup, 'user_exports_folder')
+      config = Cartodb.get_config(:user_backup)
+      instance = new(base_dir, id, log, config.merge(upload_path: 'backups'))
+
+      FileUtils.mkdir_p(instance.data_dir)
+      FileUtils.mkdir_p(instance.meta_dir)
+
+      instance
+    end
+
+    attr_reader :data_dir, :meta_dir, :work_dir
+
+    def initialize(base_dir, id, log, config)
+      @id = id
+      @log = log
+      @config = config
+
+      @base_dir = base_dir
+      @work_dir = "#{base_dir}/#{id}"
+      @data_dir = "#{work_dir}/data"
+      @meta_dir = "#{work_dir}/meta"
+    end
+
+    def upload
+      upload_package(compress)
+    end
+
+    def download(exported_file)
+      uncompress(download_package(exported_file))
+    end
+
+    def cleanup
+      FileUtils.remove_dir(work_dir, true)
+    end
+
+    private
+
+    attr_reader :id, :log, :base_dir
+
+    def compress
+      filename = "user_export_#{id}.zip"
+
+      log.append('=== Compressing export ===')
+      `cd #{work_dir}/ && zip -0 -r \"../#{filename}\" . && cd -`
+      FileUtils.remove_dir(work_dir)
+
+      "#{base_dir}/#{filename}"
+    end
+
+    def upload_package(filepath)
+      log.append('=== Uploading user data package ===')
+      file = CartoDB::FileUploadFile.new(filepath)
+      s3_config = @config['s3'] || {}
+      results = file_upload_helper.upload_file_to_storage(
+        file_param: file,
+        s3_config: s3_config,
+        allow_spaces: true,
+        force_s3_upload: true,
+        random_token: @config[:upload_path]
+      )
+
+      export_path = if results[:file_path].present?
+                      log.append("Ad-hoc export download: #{results[:file_path]}")
+                      results[:file_path]
+                    else
+                      log.append("By file_upload_helper: #{results[:file_uri]}")
+                      results[:file_uri]
+                    end
+
+      log.append('=== Deleting tmp file ===')
+      FileUtils.rm(filepath)
+
+      export_path
+    end
+
+    def download_package(exported_file)
+      destination = "#{work_dir}/export.zip"
+
+      log.append("=== Downloading #{exported_file} to #{destination} ===")
+
+      if exported_file.starts_with?('http')
+        http_client.get_file(exported_file, destination)
+      else
+        FileUtils.cp(exported_file, destination)
+      end
+
+      destination
+    end
+
+    def uncompress(filename)
+      log.append("=== Unzipping #{filename} ===")
+      `cd #{work_dir}; unzip -u #{filename}; cd -`
+
+      log.append('=== Deleting zip package ===')
+      FileUtils.rm(filename)
+    end
+
+    def http_client
+      Carto::Http::Client.get('user_imports')
+    end
+
+    def file_upload_helper
+      CartoDB::FileUpload.new(@config[:uploads_path])
+    end
+  end
+end

app/models/carto/user_multifactor_auth.rb

@@ -0,0 +1,91 @@
+require 'rotp'
+require 'rqrcode'
+
+module Carto
+  class UserMultifactorAuth < ActiveRecord::Base
+
+    TYPE_TOTP = 'totp'.freeze
+    VALID_TYPES = [TYPE_TOTP].freeze
+    DRIFT = 60.seconds
+    INTERVAL = 30.seconds
+    ISSUER = 'CARTO'.freeze
+    QR_CODE_SIZE = 400
+
+    belongs_to :user, inverse_of: :user_multifactor_auths, foreign_key: :user_id
+
+    validates :type, inclusion: { in: VALID_TYPES }
+    validates_uniqueness_of :type, scope: :user_id
+
+    before_create :create_shared_secret
+    after_save    :sync_central, unless: :skip_central_sync
+    after_destroy :sync_central, unless: :skip_central_sync
+
+    self.inheritance_column = :_type
+
+    scope :enabled, -> { where(enabled: true) }
+    scope :setup, -> { where(enabled: false) }
+
+    attr_accessor :skip_central_sync
+
+    def self.new_from_hash(uma_hash, skip_central_sync = true)
+      new(
+        created_at: uma_hash[:created_at],
+        updated_at: uma_hash[:updated_at],
+        last_login: uma_hash[:last_login],
+        type: uma_hash[:type],
+        shared_secret: uma_hash[:shared_secret],
+        user_id: uma_hash[:user_id],
+        enabled: uma_hash[:enabled],
+        skip_central_sync: skip_central_sync
+      )
+    end
+
+    def verify!(code)
+      timestamp = verify(code)
+      raise Carto::UnauthorizedError.new('The code is not valid') unless timestamp
+      update!(enabled: true, last_login: timestamp)
+    end
+
+    def needs_setup?
+      !enabled
+    end
+
+    def provisioning_uri
+      totp.provisioning_uri(user.username)
+    end
+
+    def qr_code
+      qrcode = RQRCode::QRCode.new(totp.provisioning_uri(user.username))
+      qrcode.as_png(size: QR_CODE_SIZE).to_data_url
+    end
+
+    def to_h
+      attributes.symbolize_keys
+    end
+
+    private
+
+    def sync_central
+      # due to AR/Sequel transactions the user might not exist in the database yet
+      # this happens when cascade saving a new user with user_multifactor_auths (i.e. in user migrations)
+      ::User[user.id].update_in_central
+    end
+
+    def last_login_in_seconds
+      last_login.strftime('%s').to_i if last_login
+    end
+
+    def totp
+      @totp ||= ROTP::TOTP.new(shared_secret, issuer: ISSUER, interval: INTERVAL)
+    end
+
+    def verify(code)
+      timestamp = totp.verify_with_drift_and_prior(code.to_s, DRIFT, last_login_in_seconds)
+      Time.at(timestamp) if timestamp
+    end
+
+    def create_shared_secret
+      self.shared_secret = ROTP::Base32.random_base32 unless shared_secret.present?
+    end
+  end
+end

app/models/carto/user_notification.rb

@@ -0,0 +1,21 @@
+require_relative './carto_json_serializer'
+
+module Carto
+  class UserNotification < ActiveRecord::Base
+    belongs_to :user
+    serialize :notifications, ::Carto::CartoJsonSymbolizerSerializer
+
+    validates :user, presence: true
+    validate  :only_valid_categories
+
+    VALID_CATEGORIES = [:builder, :dashboard].freeze
+
+    private
+
+    def only_valid_categories
+      notifications.keys.none? do |category|
+        errors.add(:notifications, "Invalid category: #{category}") unless VALID_CATEGORIES.include?(category)
+      end
+    end
+  end
+end

app/models/carto/user_service.rb

@@ -0,0 +1,187 @@
+require 'active_record'
+require 'cartodb-common'
+require_dependency 'carto/db/connection'
+
+module Carto
+  class UserService
+
+    def initialize(user_model)
+      @user = user_model
+    end
+
+    # TODO: Review usage to move to UserPresenter if not used anywhere else
+
+    # Only returns owned tables (not shared ones)
+    def table_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.new
+                                      .with_user_id(@user.id)
+                                      .with_type(Carto::Visualization::TYPE_CANONICAL)
+                                      .build
+                                      .count
+    end
+
+    def owned_visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.new
+                                      .with_user_id(@user.id)
+                                      .with_type(Carto::Visualization::TYPE_DERIVED)
+                                      .build
+                                      .count
+    end
+
+    def visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.new
+                                      .with_owned_by_or_shared_with_user_id(@user.id)
+                                      .build
+                                      .count
+    end
+
+    def public_visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.user_public_visualizations(@user)
+                                      .build
+                                      .count
+    end
+
+    def public_privacy_visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.user_public_privacy_visualizations(@user)
+                                      .build
+                                      .count
+    end
+
+    def link_privacy_visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.user_link_privacy_visualizations(@user)
+                                      .build
+                                      .count
+    end
+
+    def password_privacy_visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.user_password_privacy_visualizations(@user)
+                                      .build
+                                      .count
+    end
+
+    def all_visualization_count
+      return 0 unless @user.id
+
+      Carto::VisualizationQueryBuilder.user_all_visualizations(@user)
+        .build
+        .count
+    end
+
+    def twitter_imports_count(options={})
+      date_to = (options[:to] ? options[:to].to_date : Date.today)
+      date_from = (options[:from] ? options[:from].to_date : @user.last_billing_cycle)
+      Carto::SearchTweet.twitter_imports_count(@user.search_tweets, date_from, date_to)
+    end
+
+    # Returns an array representing the last 30 days, populated with api_calls
+    # from three different sources
+    def get_api_calls(options = {})
+      return CartoDB::Stats::APICalls.new.get_api_calls_without_dates(@user.username, {old_api_calls: false})
+    end
+
+    # This method is innaccurate and understates point based tables (the /2 is to account for the_geom_webmercator)
+    # TODO: Without a full table scan, ignoring the_geom_webmercator, we cannot accuratly asses table size
+    # Needs to go on a background job.
+    def db_size_in_bytes
+      return 0 if @user.new_record?
+
+      attempts = 0
+      begin
+        # Hack to support users without the new MU functiones loaded
+        # TODO: Check this works as expected
+        user_data_size_function = cartodb_extension_version_pre_mu? ?
+          "CDB_UserDataSize()" :
+          "CDB_UserDataSize('#{@user.database_schema}')"
+        in_database(as: :superuser) do |user_database|
+          user_database.transaction do
+            user_database.execute(%{SET LOCAL lock_timeout = '1s'})
+            user_database.execute(%{SELECT cartodb.#{user_data_size_function}}).first['cdb_userdatasize'].to_i
+          end
+        end
+      rescue => e
+        attempts += 1
+        begin
+          in_database(as: :superuser).execute("ANALYZE")
+        rescue => ee
+          CartoDB.report_exception(ee, "Failed to get user db size, retrying...", user: @user)
+          raise ee
+        end
+        retry unless attempts > 1
+        CartoDB.notify_exception(e, user: @user)
+        # INFO: we need to return something to avoid 'disabled' return value
+        nil
+      end
+    end
+
+    def database_username
+      if Rails.env.production?
+        "cartodb_user_#{@user.id}"
+      elsif Rails.env.staging?
+        "cartodb_staging_user_#{@user.id}"
+      else
+        "#{Rails.env}_cartodb_user_#{@user.id}"
+      end
+    end
+
+    def cartodb_extension_version_pre_mu?
+      current_version = cartodb_extension_semver(cartodb_extension_version)
+      if current_version.size == 3
+        major, minor, _ = current_version
+        major == 0 and minor < 3
+      else
+        raise 'Current cartodb extension version does not match standard x.y.z format'
+      end
+    end
+
+    def database_public_username
+      @user.database_schema != CartoDB::DEFAULT_DB_SCHEMA ? "cartodb_publicuser_#{@user.id}" : CartoDB::PUBLIC_DB_USER
+    end
+
+    def organization_member_group_role_member_name
+      @user.in_database.execute(
+        "SELECT cartodb.CDB_Organization_Member_Group_Role_Member_Name() as org_member_role;"
+      ).first['org_member_role']
+    end
+
+    # Returns a tree elements array with [major, minor, patch] as in http://semver.org/
+    def cartodb_extension_semver(extension_version)
+      extension_version.split('.').take(3).map(&:to_i)
+    end
+
+    def cartodb_extension_version
+      @cartodb_extension_version ||= in_database(:as => :superuser).execute('select cartodb.cdb_version() as v')
+                                                                   .first['v']
+    end
+
+    def database_password
+      Carto::Common::EncryptionService.hex_digest(@user.crypted_password) + database_username
+    end
+
+    def in_database(options = {})
+      options[:username] = @user.database_username
+      options[:password] = @user.database_password
+      options[:user_schema] = @user.database_schema
+      Carto::Db::Connection.connect(@user.database_host, @user.database_name, options) do |_, connection|
+        if block_given?
+          yield(connection)
+        else
+          connection
+        end
+      end
+    end
+  end
+end

app/models/carto/user_table.rb

@@ -0,0 +1,296 @@
+require 'active_record'
+require_dependency 'carto/db/sanitize'
+
+# Integer type for PostgreSQL oid columns, with proper range
+class Carto::OidType < ActiveRecord::Type::Integer
+  def max_value
+    (1 << 32) - 1
+  end
+  def min_value
+    0
+  end
+end
+
+module Carto
+  class UserTable < ActiveRecord::Base
+    PRIVACY_PRIVATE = 0
+    PRIVACY_PUBLIC = 1
+    PRIVACY_LINK = 2
+
+    PRIVACY_VALUES_TO_TEXTS = {
+      PRIVACY_PRIVATE => 'private',
+      PRIVACY_PUBLIC => 'public',
+      PRIVACY_LINK => 'link'
+    }.freeze
+
+    # Column table_id is of type oid and the type casting provided by ActiveRecord is not
+    # good for it since it only supports signed values
+    attribute 'table_id', Carto::OidType.new
+
+    # AR sets privacy = 0 (private) by default, taken from the DB. We want it to be `nil`
+    # so the `before_validation` hook sets an appropriate privacy based on the table owner
+    attribute 'privacy', Type::Integer.new, default: nil
+
+    # The ::Table service depends on the constructor not being able to set all parameters, only these are allowed
+    # This is done so things like name changes are forced to go through ::Table.name= to ensure renaming behaviour
+    attr_accessible :privacy, :tags, :description
+
+    belongs_to :user
+
+    belongs_to :map, inverse_of: :user_table
+
+    belongs_to :data_import
+
+    has_many :automatic_geocodings, inverse_of: :table, class_name: Carto::AutomaticGeocoding,
+                                    foreign_key: :table_id, dependent: :destroy
+
+    # Disabled to avoid conflicting with the `tags` field. This relation is updated by ::Table.manage_tags.
+    # TODO: We can remove both the `user_tables.tags` field and the `tags` table in favour of the canonical viz tags.
+    # has_many :tags, foreign_key: :table_id
+
+    has_many :layers_user_table
+    has_many :layers, through: :layers_user_table
+
+    before_validation :set_default_table_privacy
+
+    validates :user, presence: true
+    validate :validate_user_not_viewer
+    validates :name, uniqueness: { scope: :user_id }
+    validates :name, exclusion: Carto::DB::Sanitize::RESERVED_TABLE_NAMES
+    validates :privacy, inclusion: [PRIVACY_PRIVATE, PRIVACY_PUBLIC, PRIVACY_LINK].freeze
+    validate :validate_privacy_changes
+
+    before_create :service_before_create
+    after_create :create_canonical_visualization, unless: :map
+    after_create :service_after_create
+    after_save :service_after_save
+
+    # The `destroyed?` check is needed to avoid the hook running twice when deleting a table from the ::Table service
+    # as it is triggered directly, and a second time from canonical visualization destruction hooks.
+    # TODO: This can be simplified after deleting the old UserTable model
+    before_destroy :ensure_not_viewer
+    before_destroy :cache_dependent_visualizations, unless: :destroyed?
+    after_destroy :destroy_dependent_visualizations
+    after_destroy :service_after_destroy
+
+    def geometry_types
+      @geometry_types ||= service.geometry_types
+    end
+
+    # Estimated size
+    def size
+      row_count_and_size[:size]
+    end
+
+    def table_size
+      service.table_size
+    end
+
+    # Estimated row_count. Preferred: `estimated_row_count`
+    def row_count
+      row_count_and_size[:row_count]
+    end
+
+    # Estimated row count and size. Preferred `estimated_row_count` for row count.
+    def row_count_and_size
+      @row_count_and_size ||= service.row_count_and_size
+    end
+
+    def service
+      @service ||= ::Table.new(user_table: self)
+    end
+
+    def set_service(table)
+      @service = table
+    end
+
+    def visualization
+      map.visualization if map
+    end
+
+    def synchronization
+      visualization.synchronization if visualization
+    end
+
+    def fully_dependent_visualizations
+      affected_visualizations.select { |v| v.fully_dependent_on?(self) }
+    end
+
+    def accessible_dependent_derived_maps
+      affected_visualizations.select { |v| (v.has_read_permission?(user) && v.derived?) ? v : nil }
+    end
+
+    def partially_dependent_visualizations
+      affected_visualizations.select { |v| v.partially_dependent_on?(self) }
+    end
+
+    def dependent_visualizations
+      affected_visualizations.select { |v| v.dependent_on?(self) }
+    end
+
+    def affected_visualizations
+      layers.map(&:visualization).uniq.compact
+    end
+
+    def name_for_user(other_user)
+      is_owner?(other_user) ? name : fully_qualified_name
+    end
+
+    def private?
+      privacy == PRIVACY_PRIVATE
+    end
+
+    def public?
+      privacy == PRIVACY_PUBLIC
+    end
+
+    def public_with_link_only?
+      privacy == PRIVACY_LINK
+    end
+
+    def privacy_text
+      visualization_privacy.upcase
+    end
+
+    def visualization_privacy
+      PRIVACY_VALUES_TO_TEXTS[privacy]
+    end
+
+    def readable_by?(user)
+      !private? || is_owner?(user) || visualization_readable_by?(user)
+    end
+
+    def raster?
+      service.is_raster?
+    end
+
+    def geometry_type
+      service.the_geom_type || 'geometry'
+    end
+
+    def estimated_row_count
+      service.estimated_row_count
+    end
+
+    def actual_row_count
+      service.actual_row_count
+    end
+
+    def sync_table_id
+      self.table_id = service.get_table_id
+    end
+
+    def permission
+      visualization.permission if visualization
+    end
+
+    def external_source_visualization
+      data_import.try(:external_data_imports).try(:first).try(:external_source).try(:visualization)
+    end
+
+    def table_visualization
+      map.visualization if map
+    end
+
+    def update_cdb_tablemetadata
+      service.update_cdb_tablemetadata
+    end
+
+    def save_changes
+      # TODO: Compatibility with Sequel model, can be removed afterwards. Used in ::Table.set_the_geom_column!
+      save if changed?
+    end
+
+    def tags=(value)
+      return unless value
+      super(value.split(',').map(&:strip).reject(&:blank?).uniq.join(','))
+    end
+
+    # TODO: Compatibility with Sequel model, can be removed afterwards.
+    def set_tag_array(tag_array)
+      self.tags = tag_array.join(',')
+    end
+
+    # TODO: This is related to an incompatibility between visualizations models, `get_related_tables`, See #11705
+    def privacy_text_for_vizjson
+      privacy == PRIVACY_LINK ? 'PUBLIC' : privacy_text
+    end
+
+    def automatic_geocoding
+      automatic_geocodings.first
+    end
+
+    def is_owner?(user)
+      return false unless user
+      user_id == user.id
+    end
+
+    private
+
+    def default_privacy_value
+      user.try(:default_table_privacy)
+    end
+
+    def set_default_table_privacy
+      self.privacy ||= default_privacy_value
+    end
+
+    def fully_qualified_name
+      "\"#{user.database_schema}\".#{name}"
+    end
+
+    def visualization_readable_by?(user)
+      user && permission && permission.user_has_read_permission?(user)
+    end
+
+    def validate_user_not_viewer
+      errors.add(:user, "Viewer users can't create tables") if user.try(:viewer)
+    end
+
+    def validate_privacy_changes
+      if !user.try(:private_tables_enabled) && !public? && (new_record? || privacy_changed?)
+        errors.add(:privacy, 'unauthorized to create private tables')
+      end
+    end
+
+    def create_canonical_visualization
+      visualization = Carto::VisualizationFactory.create_canonical_visualization(self)
+      update_attribute(:map, visualization.map)
+      visualization.map.set_default_boundaries!
+    end
+
+    def cache_dependent_visualizations
+      @fully_dependent_visualizations_cache = fully_dependent_visualizations
+      @partially_dependent_visualizations_cache = partially_dependent_visualizations
+    end
+
+    def destroy_dependent_visualizations
+      table_visualization.try(:delete_from_table)
+      @fully_dependent_visualizations_cache.each(&:destroy)
+      @partially_dependent_visualizations_cache.each do |visualization|
+        visualization.unlink_from(self)
+      end
+    end
+
+    def ensure_not_viewer
+      # Loading ::User is a workaround for User deletion: viewer attribute change is not visible at AR transaction
+      raise "Viewer users can't destroy tables" if user && user.viewer && ::User[user_id].viewer
+    end
+
+    def service_before_create
+      service.before_create
+    end
+
+    def service_after_create
+      service.after_create
+    end
+
+    def service_after_save
+      service.after_save
+    end
+
+    def service_after_destroy
+      service.after_destroy
+    end
+  end
+end

app/models/carto/users_group.rb

@@ -0,0 +1,10 @@
+require 'active_record'
+
+module Carto
+  class UsersGroup < ActiveRecord::Base
+    # autosave must be explicitly disabled due to https://github.com/rails/rails/issues/9336
+    # but we probably should not autosave from a ternary table anyway
+    belongs_to :user, class_name: Carto::User, autosave: false
+    belongs_to :group, class_name: Carto::Group, autosave: false
+  end
+end

app/models/carto/visualization.rb

@@ -0,0 +1,834 @@
+require 'active_record'
+require 'cartodb-common'
+require_relative '../visualization/stats'
+require_dependency 'carto/named_maps/api'
+require_dependency 'carto/helpers/auth_token_generator'
+require_dependency 'carto/uuidhelper'
+require_dependency 'carto/visualization_invalidation_service'
+require_dependency 'carto/visualization_backup_service'
+
+module Carto::VisualizationDependencies
+  def fully_dependent_on?(user_table)
+    derived? && layers_dependent_on(user_table).count == data_layers.count
+  end
+
+  def partially_dependent_on?(user_table)
+    derived? && layers_dependent_on(user_table).count.between?(1, data_layers.count - 1)
+  end
+
+  def dependent_on?(user_table)
+    derived? && layers_dependent_on(user_table).any?
+  end
+
+  private
+
+  def layers_dependent_on(user_table)
+    data_layers.select { |l| l.depends_on?(user_table) }
+  end
+end
+
+class Carto::Visualization < ActiveRecord::Base
+  include CacheHelper
+  include Carto::UUIDHelper
+  include Carto::AuthTokenGenerator
+  include Carto::VisualizationDependencies
+  include Carto::VisualizationBackupService
+
+  TYPE_CANONICAL = 'table'.freeze
+  TYPE_DERIVED = 'derived'.freeze
+  TYPE_SLIDE = 'slide'.freeze
+  TYPE_REMOTE = 'remote'.freeze
+  TYPE_KUVIZ = 'kuviz'.freeze
+
+  VALID_TYPES = [TYPE_CANONICAL, TYPE_DERIVED, TYPE_SLIDE, TYPE_REMOTE, TYPE_KUVIZ].freeze
+
+  KIND_GEOM   = 'geom'.freeze
+  KIND_RASTER = 'raster'.freeze
+
+  PRIVACY_PUBLIC = 'public'.freeze
+  PRIVACY_PRIVATE = 'private'.freeze
+  PRIVACY_LINK = 'link'.freeze
+  PRIVACY_PROTECTED = 'password'.freeze
+  PRIVACIES = [PRIVACY_LINK, PRIVACY_PROTECTED, PRIVACY_PUBLIC, PRIVACY_PRIVATE].freeze
+
+  VERSION_BUILDER = 3
+
+  V2_VISUALIZATIONS_REDIS_KEY = 'vizjson2_visualizations'.freeze
+
+  scope :remotes, -> { where(type: TYPE_REMOTE) }
+
+  # INFO: disable ActiveRecord inheritance column
+  self.inheritance_column = :_type
+
+  belongs_to :user, -> { select(Carto::User::DEFAULT_SELECT) }, inverse_of: :visualizations
+  belongs_to :full_user, -> { readonly(true) }, class_name: Carto::User, inverse_of: :visualizations,
+                                                primary_key: :id, foreign_key: :user_id
+
+  belongs_to :permission, inverse_of: :visualization, dependent: :destroy
+
+  has_many :likes, foreign_key: :subject
+  has_many :shared_entities, foreign_key: :entity_id, inverse_of: :visualization, dependent: :destroy
+
+  has_one :external_source, class_name: Carto::ExternalSource, dependent: :destroy, inverse_of: :visualization
+  has_many :unordered_children, class_name: Carto::Visualization, foreign_key: :parent_id
+
+  has_many :overlays, -> { order(:order) }, dependent: :destroy, inverse_of: :visualization
+
+  belongs_to :active_layer, class_name: Carto::Layer
+
+  belongs_to :map, class_name: Carto::Map, inverse_of: :visualization, dependent: :destroy
+
+  has_one :asset, class_name: Carto::Asset, inverse_of: :visualization, dependent: :destroy
+
+  has_many :related_templates, class_name: Carto::Template, foreign_key: :source_visualization_id
+
+  has_one :synchronization, class_name: Carto::Synchronization, dependent: :destroy
+  has_many :external_sources, class_name: Carto::ExternalSource
+
+  has_many :analyses, class_name: Carto::Analysis
+  has_many :mapcaps, -> { order('created_at DESC') }, class_name: Carto::Mapcap, dependent: :destroy
+
+  has_one :state, class_name: Carto::State, autosave: true
+
+  has_many :snapshots, class_name: Carto::Snapshot, dependent: :destroy
+
+  validates :name, :privacy, :type, :user_id, :version, presence: true
+  validates :privacy, inclusion: { in: PRIVACIES }
+  validates :type, inclusion: { in: VALID_TYPES }
+  validate :validate_password_presence
+  validate :validate_privacy_changes
+  validate :validate_user_not_viewer, on: :create
+
+  before_validation :set_default_version, :set_register_table_only
+  before_create :set_random_id, :set_default_permission
+
+  before_save :remove_password_if_unprotected
+  after_save :propagate_attribution_change
+  after_save :propagate_privacy_and_name_to, if: :table
+
+  before_destroy :backup_visualization
+  after_commit :perform_invalidations
+
+  attr_accessor :register_table_only
+
+  def set_register_table_only
+    self.register_table_only = false
+    # This is a callback, returning `true` avoids halting because of assignment `false` return value
+    true
+  end
+
+  def set_default_version
+    self.version ||= user.try(:new_visualizations_version)
+  end
+
+  DELETED_COLUMNS = ['state_id', 'url_options'].freeze
+
+  def self.columns
+    super.reject { |c| DELETED_COLUMNS.include?(c.name) }
+  end
+
+  def size
+    # Only canonical visualizations (Datasets) have a related table and then count against disk quota,
+    # but we want to not break and even allow ordering by size multiple types
+    if user_table
+      user_table.size
+    elsif remote? && external_source
+      external_source.size
+    else
+      0
+    end
+  end
+
+  def tags
+    tags = super
+    tags == nil ? [] : tags
+  end
+
+  def tags=(tags)
+    tags.reject!(&:blank?) if tags
+    super(tags)
+  end
+
+  def user_table
+    map.user_table if map
+  end
+
+  def table
+    @table ||= user_table.try(:service)
+  end
+
+  def layers_with_data_readable_by(user)
+    return [] unless map
+    map.layers.select { |l| l.data_readable_by?(user) }
+  end
+
+  def related_tables
+    @related_tables ||= get_related_tables
+  end
+
+  def related_tables_readable_by(user)
+    layers_with_data_readable_by(user).map { |l| l.user_tables_readable_by(user) }.flatten.uniq
+  end
+
+  def related_canonical_visualizations
+    @related_canonical_visualizations ||= get_related_canonical_visualizations
+  end
+
+  def stats
+    @stats ||= CartoDB::Visualization::Stats.new(self).to_poro
+  end
+
+  def transition_options
+    @transition_options ||= (slide_transition_options.nil? ? {} : JSON.parse(slide_transition_options).symbolize_keys)
+  end
+
+  def transition_options=(value)
+    self.slide_transition_options = ::JSON.dump(value.nil? ? DEFAULT_OPTIONS_VALUE : value)
+  end
+
+  def children
+    ordered = []
+    children_vis = self.unordered_children
+    if children_vis.count > 0
+      ordered << children_vis.select { |vis| vis.prev_id.nil? }.first
+      while !ordered.last.next_id.nil?
+        target = ordered.last.next_id
+        unless target.nil?
+          ordered << children_vis.select { |vis| vis.id == target }.first
+        end
+      end
+    end
+    ordered
+  end
+
+  # TODO: refactor next methods, all have similar naming but some receive user and some others user_id
+  def liked_by?(user)
+    likes_by_user(user).any?
+  end
+
+  def likes_by_user(user)
+    likes.where(actor: user.id)
+  end
+
+  def add_like_from(user)
+    unless has_read_permission?(user)
+      raise UnauthorizedLikeError
+    end
+
+    likes.create!(actor: user.id)
+
+    self
+  rescue ActiveRecord::RecordNotUnique
+    raise AlreadyLikedError
+  end
+
+  def remove_like_from(user)
+    unless has_read_permission?(user)
+      raise UnauthorizedLikeError
+    end
+
+    item = likes.where(actor: user.id)
+    item.first.destroy unless item.first.nil?
+
+    self
+  end
+
+  def send_like_email(current_viewer, vis_preview_image)
+    if self.type == Carto::Visualization::TYPE_CANONICAL
+      ::Resque.enqueue(::Resque::UserJobs::Mail::TableLiked, self.id, current_viewer.id, vis_preview_image)
+    elsif self.type == Carto::Visualization::TYPE_DERIVED
+      ::Resque.enqueue(::Resque::UserJobs::Mail::MapLiked, self.id, current_viewer.id, vis_preview_image)
+    end
+  end
+
+  def is_viewable_by_user?(user)
+    is_publically_accesible? || has_read_permission?(user)
+  end
+
+  def is_accesible_by_user?(user)
+    is_viewable_by_user?(user) || password_protected?
+  end
+
+  def is_accessible_with_password?(user, password)
+    is_viewable_by_user?(user) || password_valid?(password)
+  end
+
+  def is_publically_accesible?
+    (public? || public_with_link?) && published?
+  end
+
+  def writable_by?(user)
+    (user_id == user.id && !user.viewer?) || has_write_permission?(user)
+  end
+
+  def varnish_key
+    "#{user.database_name}:#{sorted_related_table_names},#{id}"
+  end
+
+  def surrogate_key
+    get_surrogate_key(CartoDB::SURROGATE_NAMESPACE_VISUALIZATION, id)
+  end
+
+  def qualified_name(viewer_user = nil)
+    if viewer_user.nil? || owner?(viewer_user)
+      name
+    else
+      "#{user.sql_safe_database_schema}.#{name}"
+    end
+  end
+
+  # Despite storing always a named map, no need to retrieve it for "public" visualizations
+  def retrieve_named_map?
+    password_protected? || has_private_tables?
+  end
+
+  def has_password?
+    !password_salt.nil? && !encrypted_password.nil?
+  end
+
+  def type_slide?
+    type == TYPE_SLIDE
+  end
+
+  def kind_raster?
+    kind == KIND_RASTER
+  end
+
+  def canonical?
+    type == TYPE_CANONICAL
+  end
+
+  # TODO: remove. Kept for backwards compatibility with ::Permission model
+  def table?
+    type == TYPE_CANONICAL
+  end
+
+  def derived?
+    type == TYPE_DERIVED
+  end
+
+  def remote?
+    type == TYPE_REMOTE
+  end
+
+  def kuviz?
+    type == TYPE_KUVIZ
+  end
+
+  def layers
+    map ? map.layers : []
+  end
+
+  def data_layers
+    map ? map.data_layers : []
+  end
+
+  def carto_layers
+    map ? map.carto_layers : []
+  end
+
+  def user_layers
+    map ? map.user_layers : []
+  end
+
+  def torque_layers
+    map ? map.torque_layers : []
+  end
+
+  def other_layers
+    map ? map.other_layers : []
+  end
+
+  def base_layers
+    map ? map.base_layers : []
+  end
+
+  def named_map_layers
+    map ? map.named_map_layers : []
+  end
+
+  def password_valid?(password)
+    password_protected? &&
+      Carto::Common::EncryptionService.verify(password: password, secure_password: encrypted_password,
+                                              salt: password_salt, secret: Cartodb.config[:password_secret])
+  end
+
+  def organization?
+    privacy == PRIVACY_PRIVATE && !permission.acl.empty?
+  end
+
+  def password_protected?
+    privacy == PRIVACY_PROTECTED
+  end
+
+  def private?
+    is_privacy_private? && !organization?
+  end
+
+  def is_privacy_private?
+    privacy == PRIVACY_PRIVATE
+  end
+
+  def public?
+    privacy == PRIVACY_PUBLIC
+  end
+
+  def public_with_link?
+    self.privacy == PRIVACY_LINK
+  end
+
+  def editable?
+    !(kind_raster? || type_slide?)
+  end
+
+  def get_auth_tokens
+    [get_auth_token]
+  end
+
+  def mapviews
+    @mapviews ||= CartoDB::Visualization::Stats.mapviews(stats)
+  end
+
+  def total_mapviews
+    @total_mapviews ||= CartoDB::Visualization::Stats.new(self, nil).total_mapviews
+  end
+
+  def geometry_types
+    @geometry_types ||= user_table.geometry_types if user_table
+  end
+
+  def table_service
+    user_table.try(:service)
+  end
+
+  def has_read_permission?(user)
+    user && (owner?(user) || (permission && permission.user_has_read_permission?(user)))
+  end
+  alias :can_view_private_info? :has_read_permission?
+
+  def estimated_row_count
+    user_table.try(:row_count)
+  end
+
+  def actual_row_count
+    user_table.try(:actual_row_count)
+  end
+
+  def license_info
+    if !license.nil?
+      Carto::License.find(license.to_sym)
+    end
+  end
+
+  def can_be_cached?
+    !is_privacy_private?
+  end
+
+  def widgets
+    # Preload widgets for all layers
+    ActiveRecord::Associations::Preloader.new.preload(layers, :widgets)
+    layers.map(&:widgets).flatten
+  end
+
+  def analysis_widgets
+    widgets.select { |w| w.source_id.present? }
+  end
+
+  def attributions_from_derived_visualizations
+    related_canonical_visualizations.map(&:attributions).reject(&:blank?)
+  end
+
+  def delete_from_table
+    destroy if persisted?
+  end
+
+  def allowed_auth_tokens
+    entities = [user] + permission.entities_with_read_permission
+    entities.map(&:get_auth_token)
+  end
+
+  # - v2 (Editor): not private
+  # - v3 (Builder): not derived or not private, mapcapped
+  # This Ruby code should match the SQL code at Carto::VisualizationQueryBuilder#build section for @only_published.
+  def published?
+    !is_privacy_private? && (!builder? || !derived? || mapcapped?)
+  end
+
+  def builder?
+    version == VERSION_BUILDER
+  end
+
+  MAX_MAPCAPS_PER_VISUALIZATION = 1
+
+  def create_mapcap!
+    unless mapcaps.count < MAX_MAPCAPS_PER_VISUALIZATION
+      mapcaps.last.destroy
+    end
+
+    auto_generate_indices_for_all_layers
+    mapcaps.create!
+  end
+
+  def mapcapped?
+    latest_mapcap.present?
+  end
+
+  def latest_mapcap
+    mapcaps.first
+  end
+
+  def uses_builder_features?
+    builder? || analyses.any? || widgets.any? || mapcapped?
+  end
+
+  def add_source_analyses
+    return unless analyses.empty?
+
+    data_layers.each_with_index do |layer, index|
+      analysis = Carto::Analysis.source_analysis_for_layer(layer, index)
+
+      if analysis.save
+        layer.options[:source] = analysis.natural_id
+        layer.options[:letter] = analysis.natural_id.first
+        layer.save
+      else
+        CartoDB::Logger.warning(message: 'Couldn\'t add source analysis for layer', user: user, layer: layer)
+      end
+    end
+  end
+
+  def ids_json
+    layers_for_hash = layers.map do |layer|
+      { layer_id: layer.id, widgets: layer.widgets.map(&:id) }
+    end
+
+    {
+      visualization_id: id,
+      map_id: map.id,
+      layers: layers_for_hash
+    }
+  end
+
+  def populate_ids(ids_json)
+    self.id = ids_json[:visualization_id]
+    map.id = ids_json[:map_id]
+
+    map.layers.each_with_index do |layer, index|
+      stored_layer_ids = ids_json[:layers][index]
+      stored_layer_id = stored_layer_ids[:layer_id]
+
+      layer.id = stored_layer_id
+      layer.maps = [map]
+
+      layer.widgets.each_with_index do |widget, widget_index|
+        widget.id = stored_layer_ids[:widgets][widget_index]
+        widget.layer_id = stored_layer_id
+      end
+    end
+  end
+
+  def for_presentation
+    mapcapped? ? latest_mapcap.regenerate_visualization : self
+  end
+
+  # TODO: we should make visualization privacy/security methods aware of mapcaps and make those
+  # deal with all the different the cases internally.
+  # See https://github.com/CartoDB/cartodb/pull/9678
+  def non_mapcapped
+    persisted? ? self : Carto::Visualization.find(id)
+  end
+
+  def mark_as_vizjson2
+    $tables_metadata.SADD(V2_VISUALIZATIONS_REDIS_KEY, id)
+  end
+
+  def uses_vizjson2?
+    $tables_metadata.SISMEMBER(V2_VISUALIZATIONS_REDIS_KEY, id) > 0
+  end
+
+  def open_in_editor?
+    !builder? && uses_vizjson2?
+  end
+
+  def can_be_automatically_migrated_to_v3?
+    overlays.builder_incompatible.none?
+  end
+
+  def state
+    super ? super : build_state
+  end
+
+  def can_be_private?
+    derived? ? user.try(:private_maps_enabled) : user.try(:private_tables_enabled)
+  end
+
+  # TODO: Backward compatibility with Sequel
+  def store_using_table(_table_privacy_changed = false)
+    store
+  end
+
+  def store
+    save!
+    self
+  end
+
+  def is_owner?(user)
+    user && user.id == user_id
+  end
+
+  def unlink_from(user_table)
+    layers_dependent_on(user_table).each do |layer|
+      Carto::Analysis.find_by_natural_id(id, layer.source_id).try(:destroy) if layer.source_id
+
+      layer.destroy
+    end
+  end
+
+  def invalidate_after_commit
+    # This marks this visualization as affected by this transaction, so AR will call its `after_commit` hook, which
+    # performs the actual invalidations. This takes this operation outside of the DB transaction to avoid long locks
+    if self.class.connection.open_transactions.zero?
+      raise 'invalidate_after_commit should be called within a transaction'
+    end
+    add_to_transaction
+    true
+  end
+  # TODO: Privacy manager compatibility, can be removed after removing ::UserTable
+  alias :invalidate_cache :invalidate_after_commit
+
+  def has_permission?(user, permission_type)
+    return false if user.viewer && permission_type == Carto::Permission::ACCESS_READWRITE
+    return is_owner?(user) if permission_id.nil?
+    is_owner?(user) || permission.permitted?(user, permission_type)
+  end
+
+  def ensure_valid_privacy
+    self.privacy = default_privacy if privacy.nil?
+    self.privacy = PRIVACY_PUBLIC unless can_be_private?
+  end
+
+  def default_privacy
+    can_be_private? ? PRIVACY_LINK : PRIVACY_PUBLIC
+  end
+
+  def privacy=(privacy)
+    super(privacy.try(:downcase))
+  end
+
+  def password=(value)
+    if value.present?
+      self.password_salt = ""
+      self.encrypted_password = Carto::Common::EncryptionService.encrypt(password: value,
+                                                                         secret: Cartodb.config[:password_secret])
+    end
+  end
+
+  def synced?
+    synchronization.present?
+  end
+
+  def dependent_visualizations
+    user_table.try(:dependent_visualizations) || []
+  end
+
+  def backup_visualization(category = Carto::VisualizationBackup::CATEGORY_VISUALIZATION)
+    return true if remote?
+
+    if map && !destroyed?
+      create_visualization_backup(visualization: self, category: category)
+    end
+  end
+
+  private
+
+  def remove_password
+    self.password_salt = nil
+    self.encrypted_password = nil
+  end
+
+  def perform_invalidations
+    invalidation_service.invalidate
+  rescue => e
+    # This is called at an after_commit. If there's any error, we won't notice
+    # but the after_commit chain stops.
+    # This was discovered during #12844, because "Updates changes even if named maps communication fails" test
+    # begun failing because Overlay#invalidate_cache invokes this method directly.
+    # We chose to log and continue to keep coherence on calls to this outside the callback.
+    CartoDB::Logger.error(message: "Error on visualization invalidation", exception: e, visualization_id: id)
+  end
+
+  def propagate_privacy_and_name_to
+    raise "Empty table sent to propagate_privacy_and_name_to()" unless table
+    propagate_privacy if privacy_changed? && canonical?
+    propagate_name if name_was != name # name_changed? returns false positives in changes like a->A->a (sanitization)
+  end
+
+  def propagate_privacy
+    table.reload
+    if privacy && table.privacy_text.casecmp(privacy) != 0 # privacy is different, case insensitive
+      CartoDB::TablePrivacyManager.new(table).set_from_visualization(self).update_cdb_tablemetadata
+    end
+  end
+
+  def propagate_name
+    # TODO: Move this to ::Table?
+    return if table.changing_name?
+    table.register_table_only = register_table_only
+    table.name = name
+    if table.name != name
+      # Sanitization. For example, spaces -> _
+      update_column(:name, table.name)
+    end
+    table.update(name: name)
+
+    if name_changed?
+      support_tables.rename(name_was, name, true, name_was)
+    end
+    self
+  rescue => exception
+    if name_changed? && !(exception.to_s =~ /relation.*does not exist/)
+      revert_name_change(name_was)
+    end
+    raise CartoDB::InvalidMember.new(exception.to_s)
+  end
+
+  def revert_name_change(previous_name)
+    self.name = previous_name
+    store
+  rescue => exception
+    raise CartoDB::InvalidMember.new(exception.to_s)
+  end
+
+  def propagate_attribution_change
+    table.propagate_attribution_change(attributions) if table && attributions_changed?
+  end
+
+  def support_tables
+    @support_tables ||= CartoDB::Visualization::SupportTables.new(
+      user.in_database, parent_id: id, parent_kind: kind, public_user_roles: user.db_service.public_user_roles
+    )
+  end
+
+  def auto_generate_indices_for_all_layers
+    user_tables = data_layers.map(&:user_tables).flatten.uniq
+    user_tables.each do |ut|
+      ::Resque.enqueue(::Resque::UserDBJobs::UserDBMaintenance::AutoIndexTable, ut.id)
+    end
+  end
+
+  def set_random_id
+    # This should be done with a DB default
+    self.id ||= random_uuid
+  end
+
+  def set_default_permission
+    self.permission ||= Carto::Permission.create(owner: user, owner_username: user.username)
+  end
+
+  def has_private_tables?
+    !related_tables.index { |table| table.private? }.nil?
+  end
+
+  def sorted_related_table_names
+    mapped_table_names = related_tables.map { |table| "#{user.database_schema}.#{table.name}" }
+
+    mapped_table_names.sort { |i, j| i <=> j }.join(',')
+  end
+
+  def get_related_tables
+    return [] unless map
+
+    map.data_layers.flat_map(&:user_tables).uniq
+  end
+
+  def get_related_canonical_visualizations
+    get_related_visualizations_by_types([TYPE_CANONICAL])
+  end
+
+  def get_related_visualizations_by_types(types)
+    Carto::Visualization.where(map_id: related_tables.map(&:map_id), type: types).all
+  end
+
+  def has_write_permission?(user)
+    user && !user.viewer? && (owner?(user) || (permission && permission.user_has_write_permission?(user)))
+  end
+
+  def owner?(user)
+    user_id == user.id
+  end
+
+  def validate_password_presence
+    errors.add(:password, 'required for protected visualization') if password_protected? && !has_password?
+  end
+
+  def remove_password_if_unprotected
+    remove_password unless password_protected?
+  end
+
+  def validate_privacy_changes
+    return unless privacy_changed? && derived?
+
+    if is_privacy_private? && !user.try(:private_maps_enabled?)
+      errors.add(:privacy, 'cannot be set to private')
+    elsif (privacy_was == Carto::Visualization::PRIVACY_PRIVATE ||
+          (!privacy_was && privacy != Carto::Visualization::PRIVACY_PRIVATE)) &&
+          CartoDB::QuotaChecker.new(user).will_be_over_public_map_quota?
+      errors.add(:privacy, 'over account public map quota')
+    end
+  end
+
+  def validate_user_not_viewer
+    if user.viewer
+      errors.add(:user, 'cannot be viewer')
+    end
+  end
+
+  def invalidation_service
+    @invalidation_service ||= Carto::VisualizationInvalidationService.new(self)
+  end
+
+  class Watcher
+    # watcher:_orgid_:_vis_id_:_user_id_
+    KEY_FORMAT = "watcher:%s".freeze
+
+    # @params user Carto::User
+    # @params visualization Carto::Visualization
+    # @throws Carto::Visualization::WatcherError
+    def initialize(user, visualization, notification_ttl = nil)
+      raise WatcherError.new('User must belong to an organization') if user.organization.nil?
+      @user = user
+      @visualization = visualization
+
+      default_ttl = Cartodb.config[:watcher].present? ? Cartodb.config[:watcher].try("fetch", 'ttl', 60) : 60
+      @notification_ttl = notification_ttl.nil? ? default_ttl : notification_ttl
+    end
+
+    # Notifies that is editing the visualization
+    # NOTE: Expiration is handled internally by redis
+    def notify
+      key = KEY_FORMAT % @visualization.id
+      $tables_metadata.multi do
+        $tables_metadata.hset(key, @user.username, current_timestamp + @notification_ttl)
+        $tables_metadata.expire(key, @notification_ttl)
+      end
+    end
+
+    # Returns a list of usernames currently editing the visualization
+    def list
+      key = KEY_FORMAT % @visualization.id
+      users_expiry = $tables_metadata.hgetall(key)
+      now = current_timestamp
+      users_expiry.select { |_, expiry| expiry.to_i > now }.keys
+    end
+
+    private
+
+    def current_timestamp
+      Time.now.getutc.to_i
+    end
+  end
+
+  class WatcherError < CartoDB::BaseCartoDBError; end
+  class AlreadyLikedError < StandardError; end
+  class UnauthorizedLikeError < StandardError; end
+end

app/models/carto/visualization_backup.rb

@@ -0,0 +1,21 @@
+require 'active_record'
+
+module Carto
+  class VisualizationBackup < ActiveRecord::Base
+
+    # @param Uuid id
+    # @param Uuid user_id
+    # @param Uuid visualization_id
+    # @param DateTime created_at (CURRENT_TIMESTAMP)
+    # @param String category
+    # @param JSON export
+
+    attr_accessible :id, :user_id, :visualization_id, :category, :export
+    validates :user_id, :visualization_id, :category, :export, presence: true
+    serialize :export, CartoJsonSymbolizerSerializer
+    validates :export, carto_json_symbolizer: true
+
+    CATEGORY_VISUALIZATION = 'visualization'.freeze
+    CATEGORY_LAYER = 'layer'.freeze
+  end
+end

app/models/carto/visualization_export.rb

@@ -0,0 +1,114 @@
+require 'uri'
+require 'fileutils'
+require 'active_record'
+require_relative '../../services/carto/visualizations_export_service_2'
+require_relative '../../../services/sql-api/sql_api'
+require_relative './visualization'
+require_dependency 'carto/visualization_exporter'
+
+module Carto
+  class VisualizationExport < ::ActiveRecord::Base
+    include VisualizationExporter
+    include Carto::VisualizationsExportService2Validator
+
+    belongs_to :visualization, class_name: Carto::Visualization
+    belongs_to :user, class_name: Carto::User
+    belongs_to :log, class_name: Carto::Log
+
+    validate :visualization_exportable_by_user?, if: :new_record?
+    validate :user_tables_ids_valid?, if: :new_record?
+    validate :valid_visualization_type?, if: :new_record?
+
+    STATE_PENDING = 'pending'.freeze
+    STATE_EXPORTING = 'exporting'.freeze
+    STATE_UPLOADING = 'uploading'.freeze
+    STATE_COMPLETE = 'complete'.freeze
+    STATE_FAILURE = 'failure'.freeze
+
+    def exporter_folder
+      ensure_folder(exporter_config['exporter_temporal_folder'] || DEFAULT_EXPORTER_TMP_FOLDER)
+    end
+
+    def run_export!(file_upload_helper: default_file_upload_helper, download_path: nil)
+      logger = Carto::Log.new(type: 'visualization_export')
+
+      logger.append('Exporting')
+      update_attributes(state: STATE_EXPORTING, log: logger)
+      filepath = export(visualization, user, user_tables_ids: user_tables_ids)
+
+      logger.append('Uploading')
+      update_attributes(state: STATE_UPLOADING, file: filepath)
+
+      file = CartoDB::FileUploadFile.new(filepath)
+
+      s3_config = Cartodb.config[:exporter]['s3'].deep_dup || {}
+
+      plain_name = header_encode(file.original_filename.force_encoding('iso-8859-1'))
+      utf_name = header_encode(file.original_filename)
+      s3_config['content-disposition'] = %{attachment;filename="#{plain_name}";filename*=utf-8''#{utf_name}}
+
+      results = file_upload_helper.upload_file_to_storage(
+        file_param: file,
+        s3_config: s3_config,
+        allow_spaces: true,
+        force_s3_upload: true
+      )
+
+      if results[:file_uri].present?
+        logger.append("By file_upload_helper: #{results[:file_uri]}, #{filepath}, (ignored: #{results[:file_path]})")
+        export_url = results[:file_uri]
+        export_file = filepath
+      else
+        logger.append("Ad-hoc export download: #{results[:file_path]} (ignored: #{filepath})")
+        export_url = download_path
+        export_file = results[:file_path]
+      end
+
+      logger.append('Deleting tmp file')
+      FileUtils.rm(filepath)
+
+      state = export_url.present? && export_file.present? ? STATE_COMPLETE : STATE_FAILURE
+      logger.append("Finishing. State: #{state}. File: #{export_file}. URL: #{url}")
+      update_attributes(state: state, file: export_file, url: export_url)
+      true
+    rescue => e
+      logger.append_exception('Exporting', exception: e)
+      CartoDB::Logger.error(
+        exception: e,
+        message: "Visualization export error",
+        user: user,
+        visualization_id: visualization.id,
+        visualization_export_id: id)
+      update_attributes(state: STATE_FAILURE)
+
+      false
+    end
+
+    private
+
+    def header_encode(name)
+      URI.encode(name, /[^#{URI::PATTERN::UNRESERVED}]/)
+    end
+
+    def default_file_upload_helper
+      CartoDB::FileUpload.new(Cartodb.get_config(:exporter, "uploads_path"))
+    end
+
+    def visualization_exportable_by_user?
+      errors.add(:user, 'Must be able to view the visualization') unless visualization.is_viewable_by_user?(user)
+    end
+
+    def user_tables_ids_valid?
+      return unless user_tables_ids.present?
+      related_tables_ids = visualization.related_tables_readable_by(user).map(&:id)
+      not_valid = user_tables_ids.split(',').select { |user_table_id| !related_tables_ids.include?(user_table_id) }
+      not_valid.each do |user_table_id|
+        errors.add(:user_tables_ids, "User table #{user_table_id} is not related to visualization #{visualization.id}")
+      end
+    end
+
+    def valid_visualization_type?
+      errors.add(:visualization, 'Only derived visualizations can be exported') unless visualization.derived?
+    end
+  end
+end

app/models/carto/widget.rb

@@ -0,0 +1,101 @@
+require_relative './carto_json_serializer'
+
+class Carto::Widget < ActiveRecord::Base
+  # INFO: disable ActiveRecord inheritance column
+  self.inheritance_column = :_type
+
+  serialize :options, ::Carto::CartoJsonSymbolizerSerializer
+  validates :options, carto_json_symbolizer: true
+
+  serialize :style, ::Carto::CartoJsonSymbolizerSerializer
+  validates :style, carto_json_symbolizer: true
+
+  belongs_to :layer, class_name: Carto::Layer
+
+  before_validation :set_style_if_nil
+  validates :layer, :order, :type, :options, :source_id, presence: true
+  validates :source_id, presence: true
+
+  validate :validate_user_not_viewer
+  validate :valid_source_id
+
+  before_destroy :validate_user_not_viewer
+
+  after_save    :notify_maps_change, :auto_generate_indices
+  after_destroy :notify_maps_change, :auto_generate_indices
+
+  def self.from_visualization_id(visualization_id)
+    visualization = Carto::Visualization.where(id: visualization_id).first
+    visualization.nil? ? [] : visualization.widgets.flatten
+  end
+
+  def self.visualization_analysis_widgets(visualization_id)
+    visualization = Carto::Visualization.where(id: visualization_id).first
+    visualization.nil? ? [] : visualization.analysis_widgets
+  end
+
+  def self.layer_widgets(layer_id)
+    Carto::Widget.where(layer_id: layer_id).where(source_id: nil).all
+  end
+
+  def belongs_to_map?(map_id)
+    !layer.nil? && !layer.maps.nil? && layer.maps.map(&:id).include?(map_id)
+  end
+
+  def writable_by_user?(user)
+    return false unless layer
+    return false unless layer.maps
+
+    layer.maps { |l| l.writable_by_user?(user) }.select { |writable| !writable }.empty?
+  end
+
+  def visualization
+    layer.visualization
+  end
+
+  def analysis_node
+    return nil unless source_id
+    analysis_nodes = visualization.analyses.map(&:analysis_node)
+    analysis_nodes.lazy.map { |node| node.find_by_id(source_id) }.find { |node| node }
+  end
+
+  def column
+    options[:column]
+  end
+
+  private
+
+  def set_style_if_nil
+    self.style ||= {}
+  end
+
+  def notify_maps_change
+    layer.maps.each do |m|
+      map = Map.where(id: m.id).first
+      map.notify_map_change if map
+    end
+  end
+
+  def auto_generate_indices
+    layer.user_tables.each do |ut|
+      ::Resque.enqueue(::Resque::UserDBJobs::UserDBMaintenance::AutoIndexTable, ut.id)
+    end
+  end
+
+  def validate_user_not_viewer
+    if user && user.viewer
+      errors.add(:layer, "Viewer users can't edit widgets")
+      false
+    end
+  end
+
+  def valid_source_id
+    unless source_id.is_a?(String) && source_id =~ /^\w*$/
+      errors.add(:source_id, "Source id must be a string")
+    end
+  end
+
+  def user
+    @user ||= layer.nil? ? nil : layer.user
+  end
+end

app/models/client_application.rb

@@ -0,0 +1,73 @@
+require 'oauth'
+
+class DomainPatcherRequestProxy < OAuth::RequestProxy::RackRequest
+  def uri
+    super.sub('carto.com', 'cartodb.com')
+  end
+end
+
+class ClientApplication < Sequel::Model
+  extend CartoDB::ConfigUtils
+
+  one_to_many :tokens, :class_name => :OauthToken
+  one_to_many :access_tokens
+  one_to_many :oauth_tokens
+
+  plugin :association_dependencies
+  add_association_dependencies :oauth_tokens => :destroy
+
+  attr_accessor :token_callback_url
+
+  def self.find_token(token_key)
+    return nil if token_key.nil?
+    token = ::RequestToken.first(:token => token_key) || ::AccessToken.first(:token => token_key)
+    token && token.authorized? ? token : nil
+  end
+
+  def self.find_by_key(key)
+    first(:key => key)
+  end
+
+  def user
+    ::User[user_id]
+  end
+
+  def user=(value)
+    set(:user_id => value.id)
+  end
+
+  def self.verify_request(request, options = {}, &block)
+    value = OAuth::Signature.build(request, options, &block).verify
+    if !value && !cartodb_com_hosted?
+      # Validation failed, try to see if it has been signed for cartodb.com
+      cartodb_request = DomainPatcherRequestProxy.new(request, options)
+      value = OAuth::Signature.build(cartodb_request, options, &block).verify
+    end
+    value
+  rescue OAuth::Signature::UnknownSignatureMethod
+    false
+  end
+
+  def oauth_server
+    @oauth_server ||= OAuth::Server.new("http://your.site")
+  end
+
+  def credentials
+    @oauth_client ||= OAuth::Consumer.new(key, secret)
+  end
+
+  # If your application requires passing in extra parameters handle it here
+  def create_request_token(params={})
+    RequestToken.create :client_application => self, :callback_url=>self.token_callback_url
+  end
+
+  def before_create
+    self.key        = OAuth::Helper.generate_key(40)[0,40]
+    self.secret     = OAuth::Helper.generate_key(40)[0,40]
+    self.created_at = Time.now
+  end
+
+  def before_save
+    self.updated_at = Time.now
+  end
+end

app/models/common/map_common.rb

@@ -0,0 +1,62 @@
+require_dependency 'carto/bounding_box_service'
+
+module Carto::MapBoundaries
+  MAXIMUM_ZOOM = 18
+
+  def set_default_boundaries!
+    bounds = get_map_bounds
+    bounds ? set_viewport_from_bounds(bounds) : set_default_viewport
+    save
+  rescue => exception
+    CartoDB::Logger.error(exception: exception, message: 'Error setting default bounds')
+  end
+
+  def recalculate_bounds!
+    set_boundaries(get_map_bounds || Carto::BoundingBoxUtils::DEFAULT_BOUNDS)
+    save
+  end
+
+  private
+
+  def set_viewport_from_bounds(bounds)
+    set_boundaries(bounds)
+    recenter_using_bounds(bounds)
+    recalculate_zoom(bounds)
+  end
+
+  def set_default_viewport
+    set_boundaries(Carto::BoundingBoxUtils::DEFAULT_BOUNDS)
+    self.center = Carto::Map::DEFAULT_OPTIONS[:center]
+    self.zoom = Carto::Map::DEFAULT_OPTIONS[:zoom]
+  end
+
+  def set_boundaries(bounds)
+    # switch to (lat,lon) for the frontend
+    self.view_bounds_ne = "[#{bounds[:maxy]}, #{bounds[:maxx]}]"
+    self.view_bounds_sw = "[#{bounds[:miny]}, #{bounds[:minx]}]"
+  end
+
+  def recenter_using_bounds(bounds)
+    x = (bounds[:maxx] + bounds[:minx]) / 2
+    y = (bounds[:maxy] + bounds[:miny]) / 2
+    self.center = "[#{y},#{x}]"
+  end
+
+  def recalculate_zoom(bounds)
+    latitude_size = bounds[:maxy] - bounds[:miny]
+    longitude_size = bounds[:maxx] - bounds[:minx]
+
+    # Don't touch zoom if the table is empty or has no bounds
+    return if longitude_size.zero? && latitude_size.zero?
+
+    zoom = -1 * ((Math.log([longitude_size, latitude_size].max) / Math.log(2)) - (Math.log(360) / Math.log(2)))
+    zoom = [[zoom.round, 1].max, MAXIMUM_ZOOM].min
+
+    self.zoom = zoom
+  end
+
+  def get_map_bounds
+    # (lon,lat) as comes out from postgis
+    Carto::BoundingBoxService.new(user, table_name).table_bounds
+  end
+end

app/models/common/organization_common.rb

@@ -0,0 +1,25 @@
+module Carto::OrganizationSoftLimits
+  def soft_geocoding_limit?
+    owner.try(:soft_geocoding_limit)
+  end
+
+  def soft_twitter_datasource_limit?
+    owner.try(:soft_twitter_datasource_limit)
+  end
+
+  def soft_here_isolines_limit?
+    owner.try(:soft_here_isolines_limit)
+  end
+
+  def soft_obs_snapshot_limit?
+    owner.try(:soft_obs_snapshot_limit)
+  end
+
+  def soft_obs_general_limit?
+    owner.try(:soft_obs_general_limit)
+  end
+
+  def soft_mapzen_routing_limit?
+    owner.try(:soft_mapzen_routing_limit)
+  end
+end

app/models/common_data.rb

@@ -0,0 +1,112 @@
+require_relative '../../lib/carto/http/client'
+require_relative '../../services/sql-api/sql_api'
+require_relative '../helpers/common_data_redis_cache'
+require_relative '../../lib/carto_api/json_client'
+
+class CommonData
+
+  def initialize(visualizations_api_url)
+    @datasets = nil
+    @http_client = Carto::Http::Client.get('common_data', log_requests: true)
+    @visualizations_api_url = visualizations_api_url
+  end
+
+  def datasets
+    return @datasets unless @datasets.nil?
+
+    @datasets = []
+
+    if is_enabled?
+      is_https_request = (@visualizations_api_url =~ /^https:\/\//)
+      cached_data = redis_cache.get(is_https_request)
+      if cached_data.nil?
+        client = CartoAPI::JsonClient.new(http_client_tag: 'common_data')
+        begin
+          response = client.get_visualizations_v1_from_url(@visualizations_api_url)
+          if response.code == 200
+            @datasets = get_datasets(response.response_body)
+            redis_cache.set(is_https_request, response.headers, response.response_body)
+          else
+            CartoDB::Logger.warning(message: "Error retrieving common data datasets", response: response.to_s)
+          end
+        rescue StandardError => e
+          CartoDB::Logger.error(exception: e)
+        end
+      else
+        @datasets = get_datasets(cached_data[:body])
+      end
+
+      CartoDB::Logger.error(message: 'common-data empty', url: @visualizations_api_url) if @datasets.empty?
+    end
+
+    @datasets
+  end
+
+  def is_enabled?
+    !config('username').nil?
+  end
+
+  private
+
+  def get_datasets(json)
+    begin
+      rows = JSON.parse(json).fetch('visualizations', [])
+    rescue StandardError => e
+      CartoDB::Logger.error(exception: e)
+      rows = []
+    end
+
+    datasets = []
+    rows.each do |row|
+      # Common-data's meta tables starts with meta_ and we want to avoid them
+      next if row["name"] =~ /meta_/
+      datasets << get_common_data_from_visualization(row)
+    end
+    datasets
+  end
+
+  def get_common_data_from_visualization(row)
+      {
+        "name" => row["name"],
+        "display_name" => row["display_name"].nil? ? row["name"] : row["display_name"],
+        "tabname" => row["name"],
+        "attributions" => row["attributions"],
+        "description" => row["description"],
+        "source" => row["source"],
+        "license" => row["license"],
+        "tags" => row["tags"],
+        "geometry_types" => %Q[{#{row["table"]["geometry_types"].join(',')}}],
+        "rows" => row["table"]["row_count"],
+        "size" => row["table"]["size"],
+        "url" => export_url(row["name"]),
+        "created_at" => row["created_at"],
+        "updated_at" => row["updated_at"]
+      }
+  end
+
+  def export_url(table_name)
+    query = %Q[select * from "#{table_name}"]
+    sql_api_url(query, table_name, config('format', 'gpkg'))
+  end
+
+  def sql_api_url(query, filename, format)
+    common_data_base_url = config('base_url')
+    CartoDB::SQLApi.new({
+      base_url: common_data_base_url,
+      protocol: 'https',
+      username: config('username'),
+    }).url(query, format, filename)
+  end
+
+  def config(key, default=nil)
+    if Cartodb.config[:common_data].present?
+      Cartodb.config[:common_data][key].present? ? Cartodb.config[:common_data][key] : default
+    else
+      default
+    end
+  end
+
+  def redis_cache
+    @redis_cache ||= CommonDataRedisCache.new
+  end
+end

app/models/common_data/singleton.rb

@@ -0,0 +1,25 @@
+class CommonDataSingleton
+  include Singleton
+
+  def initialize
+    @common_data = nil
+    @last_usage = Time.now
+  end
+
+  def datasets(visualizations_api_url)
+    now = Time.now
+    if @common_data.nil? || (now - @last_usage > (cache_ttl * 60))
+      @common_data = CommonData.new(visualizations_api_url)
+      @last_usage = now
+    end
+    @common_data.datasets
+  end
+
+  def cache_ttl
+    ttl = 0
+    if Cartodb.config[:common_data].present?
+      ttl = Cartodb.config[:common_data]['cache_ttl'] || ttl
+    end
+    ttl
+  end
+end

app/models/concerns/cartodb_central_synchronizable.rb

@@ -0,0 +1,208 @@
+module Concerns
+  module CartodbCentralSynchronizable
+    # This validation can't be added to the model because if a user creation begins at Central we can't know if user is the same or existing
+    def validate_credentials_not_taken_in_central
+      return true unless self.is_a?(::User)
+      return true unless Cartodb::Central.sync_data_with_cartodb_central?
+
+      central_client = Cartodb::Central.new
+
+      errors.add(:username, "Username taken") if central_client.get_user(self.username)['username'] == self.username
+      errors.add(:email, "Email taken") if central_client.get_user(self.email)['email'] == self.email
+      errors.empty?
+    end
+
+    def create_in_central
+      return true unless sync_data_with_cartodb_central?
+      if self.is_a?(::User)
+        if organization.present?
+          cartodb_central_client.create_organization_user(organization.name, allowed_attributes_to_central(:create))
+        else
+          CartoDB.notify_debug("User creations at box without organization are not notified to Central", user: self)
+        end
+      elsif self.is_a?(Organization)
+        raise "Can't create organizations in editor"
+      end
+      return true
+    end
+
+    def update_in_central
+      return true unless sync_data_with_cartodb_central?
+      if self.is_a?(::User)
+        if organization.present?
+          cartodb_central_client.update_organization_user(organization.name, username, allowed_attributes_to_central(:update))
+        else
+          cartodb_central_client.update_user(username, allowed_attributes_to_central(:update))
+        end
+      elsif self.is_a?(Organization)
+        cartodb_central_client.update_organization(name, allowed_attributes_to_central(:update))
+      end
+      return true
+    end
+
+    def delete_in_central
+      return true unless sync_data_with_cartodb_central?
+      if self.is_a?(::User)
+        if organization.nil?
+          cartodb_central_client.delete_user(self.username)
+        else
+          if self.organization.owner && self.organization.owner != self
+            cartodb_central_client.delete_organization_user(organization.name, username)
+          else
+            raise "Can't destroy the organization owner"
+          end
+        end
+      elsif is_a?(Organization)
+        # See Organization#destroy_cascade
+        raise "Delete organizations is not allowed" if Carto::Configuration.saas?
+        cartodb_central_client.delete_organization(name)
+      end
+      return true
+    end
+
+    def allowed_attributes_from_central(action)
+      if is_a?(Organization)
+        case action
+        when :create
+          %i(name seats viewer_seats quota_in_bytes display_name description website
+             discus_shortname twitter_username geocoding_quota map_view_quota
+             geocoding_block_price map_view_block_price
+             twitter_datasource_enabled twitter_datasource_block_size
+             twitter_datasource_block_price twitter_datasource_quota
+             google_maps_key google_maps_private_key auth_username_password_enabled
+             auth_google_enabled here_isolines_quota here_isolines_block_price
+             obs_snapshot_quota obs_snapshot_block_price obs_general_quota
+             obs_general_block_price salesforce_datasource_enabled geocoder_provider
+             isolines_provider routing_provider engine_enabled builder_enabled
+             mapzen_routing_quota mapzen_routing_block_price no_map_logo auth_github_enabled
+             password_expiration_in_d)
+        when :update
+          %i(seats viewer_seats quota_in_bytes display_name description website
+             discus_shortname twitter_username geocoding_quota map_view_quota
+             geocoding_block_price map_view_block_price
+             twitter_datasource_enabled twitter_datasource_block_size
+             twitter_datasource_block_price twitter_datasource_quota
+             google_maps_key google_maps_private_key auth_username_password_enabled
+             auth_google_enabled here_isolines_quota here_isolines_block_price
+             obs_snapshot_quota obs_snapshot_block_price obs_general_quota
+             obs_general_block_price salesforce_datasource_enabled geocoder_provider
+             isolines_provider routing_provider engine_enabled builder_enabled
+             mapzen_routing_quota mapzen_routing_block_price no_map_logo auth_github_enabled
+             password_expiration_in_d)
+        end
+      elsif is_a?(::User)
+        %i(account_type admin org_admin crypted_password database_host
+           database_timeout description disqus_shortname available_for_hire email
+           geocoding_block_price geocoding_quota map_view_block_price
+           map_view_quota max_layers
+           max_import_file_size max_import_table_row_count max_concurrent_import_count
+           name last_name notification organization_id
+           period_end_date private_tables_enabled quota_in_bytes
+           sync_tables_enabled table_quota public_map_quota regular_api_key_quota twitter_username upgraded_at
+           user_timeout username website soft_geocoding_limit
+           batch_queries_statement_timeout
+           twitter_datasource_enabled twitter_datasource_block_size
+           twitter_datasource_block_price twitter_datasource_quota
+           soft_twitter_datasource_limit
+           google_sign_in last_password_change_date github_user_id
+           google_maps_key google_maps_private_key
+           private_maps_enabled here_isolines_quota here_isolines_block_price soft_here_isolines_limit
+           obs_snapshot_quota obs_snapshot_block_price soft_obs_snapshot_limit
+           obs_general_quota obs_general_block_price soft_obs_general_limit
+           mobile_xamarin mobile_custom_watermark mobile_offline_maps
+           mobile_gis_extension mobile_max_open_users mobile_max_private_users
+           salesforce_datasource_enabled viewer geocoder_provider
+           isolines_provider routing_provider engine_enabled builder_enabled
+           mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo
+           user_render_timeout database_render_timeout state industry company phone job_role
+           password_reset_token password_reset_sent_at maintenance_mode company_employees use_case)
+      end
+    end
+
+    def allowed_attributes_to_central(action)
+      if is_a?(Organization)
+        case action
+        when :create
+          raise "Can't create organizations from editor"
+        when :update
+          allowed_attributes = %i(seats viewer_seats display_name description website discus_shortname twitter_username
+                                  auth_username_password_enabled auth_google_enabled password_expiration_in_d)
+          values.slice(*allowed_attributes)
+        end
+      elsif is_a?(::User)
+        allowed_attributes = %i(
+          account_type admin org_admin crypted_password database_host database_timeout description disqus_shortname
+          available_for_hire email geocoding_block_price geocoding_quota map_view_block_price map_view_quota max_layers
+          max_import_file_size max_import_table_row_count max_concurrent_import_count name last_name notification
+          organization_id period_end_date private_tables_enabled quota_in_bytes sync_tables_enabled table_quota
+          public_map_quota regular_api_key_quota twitter_username upgraded_at user_timeout username website
+          soft_geocoding_limit twitter_datasource_enabled soft_twitter_datasource_limit google_sign_in
+          last_password_change_date github_user_id google_maps_key google_maps_private_key here_isolines_quota
+          here_isolines_block_price soft_here_isolines_limit obs_snapshot_quota obs_snapshot_block_price
+          soft_obs_snapshot_limit obs_general_quota obs_general_block_price soft_obs_general_limit viewer
+          geocoder_provider isolines_provider routing_provider builder_enabled engine_enabled mapzen_routing_quota
+          mapzen_routing_block_price soft_mapzen_routing_limit industry company phone job_role password_reset_token
+          password_reset_sent_at company_employees use_case
+        )
+        attrs = values.slice(*allowed_attributes)
+        attrs[:multifactor_authentication_status] = multifactor_authentication_status
+        case action
+        when :create
+          attrs[:remote_user_id] = id
+          attrs.delete(:organization_id)
+          return attrs
+        when :update
+          attrs[:batch_queries_statement_timeout] = batch_queries_statement_timeout
+          attrs
+        end
+      end
+    end
+
+    def set_fields_from_central(params, action)
+      return self unless params.present? && action.present?
+      self.set(params.slice(*allowed_attributes_from_central(action)))
+
+      if self.is_a?(::User) && params.has_key?(:password)
+        self.password = self.password_confirmation = params[:password]
+      end
+      self
+    end
+
+    def set_relationships_from_central(params)
+      if params.present? && params.has_key?(:feature_flags)
+        update_feature_flags(params[:feature_flags])
+      end
+    end
+
+    def sync_data_with_cartodb_central?
+      Cartodb::Central.sync_data_with_cartodb_central?
+    end
+
+    def cartodb_central_client
+      @cartodb_central_client ||= Cartodb::Central.new
+    end
+
+    def update_feature_flags(feature_flag_ids)
+      feature_flag_ids = feature_flag_ids.compact.reject(&:empty?)
+      current_feature_flag_ids = self.feature_flags_user.map { | ffu | ffu.feature_flag_id }
+      to_add = feature_flag_ids - current_feature_flag_ids
+      to_remove = current_feature_flag_ids - feature_flag_ids
+
+      removed_feature_flags_user = self.feature_flags_user.select { | ffu | to_remove.include?(ffu.feature_flag_id) }
+      removed_feature_flags_user.map do | rffu |
+        rffu.destroy
+      end
+
+      to_add.map { | ff_id |
+        ffu = FeatureFlagsUser.new
+        ffu.user_id = self.id
+        ffu.feature_flag_id = ff_id
+        ffu.save
+      }.each { |ffu|
+        self.feature_flags_user << ffu
+      }
+
+    end
+
+  end
+end

app/models/external_data_import.rb

@@ -0,0 +1,26 @@
+require_relative './data_import'
+require_relative './visualization/external_source'
+
+class ExternalDataImport < Sequel::Model
+  many_to_one :data_import
+  many_to_one :external_source, class: CartoDB::Visualization::ExternalSource
+
+  def initialize(data_import_id, external_source_id, synchronization_id = nil)
+    super({
+        external_source_id: external_source_id,
+        data_import_id: data_import_id,
+        synchronization_id: synchronization_id
+      })
+  end
+
+  def self.by_user_id(user_id)
+    user_data_imports = DataImport.where(user_id: user_id)
+    ExternalDataImport.where(data_import_id: user_data_imports.select(:id)).all
+  end
+
+  def synchronization
+    CartoDB::Synchronization::Member.new(id: self.synchronization_id).fetch
+  rescue KeyError
+    nil
+  end
+end

app/models/feature_flag.rb

@@ -0,0 +1,16 @@
+# Feature flags are synchronized through API, so there's no need for validation
+class FeatureFlag < Sequel::Model
+  include CartoDB::MiniSequel
+
+  one_to_many :feature_flags_user
+  
+  plugin :association_dependencies
+  add_association_dependencies :feature_flags_user => :destroy
+
+  # @param name       String
+  # @param restricted boolean
+
+  def self.allowed?(name)
+    !!where(name: name, restricted: false).first
+  end
+end

app/models/feature_flags_user.rb

@@ -0,0 +1,8 @@
+class FeatureFlagsUser < Sequel::Model
+  include CartoDB::MiniSequel
+
+  many_to_one :feature_flag
+  many_to_one :user
+
+end 
+

app/models/geocoding.rb

@@ -0,0 +1,396 @@
+require 'socket'
+require_relative '../../services/table-geocoder/lib/table_geocoder_factory'
+require_relative '../../services/table-geocoder/lib/exceptions'
+require_relative '../../services/table-geocoder/lib/mail_geocoder'
+require_relative '../../services/geocoder/lib/geocoder_config'
+require_relative '../../lib/cartodb/metrics'
+require_relative 'log'
+require_relative '../../lib/cartodb/stats/geocoding'
+
+require_dependency 'carto/configuration'
+
+class Geocoding < Sequel::Model
+  include Carto::Configuration
+
+  ALLOWED_KINDS   = %w(admin0 admin1 namedplace postalcode high-resolution ipaddress)
+
+
+  PUBLIC_ATTRIBUTES = [:id, :table_id, :table_name, :state, :kind, :country_code, :region_code, :formatter,
+                       :geocoder_type, :geometry_type, :error, :processed_rows, :cache_hits, :processable_rows,
+                       :real_rows, :price, :used_credits, :remaining_quota, :country_column, :region_column,
+                       :data_import_id, :error_code]
+
+  # Characters in the following Unicode categories: Letter, Mark, Number and Connector_Punctuation,
+  # plus spaces and single quotes
+  SANITIZED_FORMATTER_REGEXP = /\A[[[:word:]]\s\,\']*\z/
+
+  MIN_GEOCODING_TIME_TO_NOTIFY = 3 * 60 #seconds
+
+  many_to_one :user
+  many_to_one :user_table, :key => :table_id
+  many_to_one :automatic_geocoding
+  many_to_one :data_import
+
+  attr_reader :table_geocoder
+  attr_reader :started_at, :finished_at
+  attr_reader :log
+
+  def self.get_not_aggregated_user_geocoding_calls(db, user_id, date_from, date_to)
+    geocoding_calls_sql = "SELECT date(created_at), sum(processed_rows) as processed_rows, " \
+                          "sum(cache_hits) as cache_hits FROM geocodings WHERE kind = 'high-resolution' " \
+                          "AND user_id = ? AND geocodings.created_at >= ? and geocodings.created_at <= ?" \
+                          "GROUP BY date(created_at) ORDER BY date(created_at) DESC"
+    db.fetch(geocoding_calls_sql, user_id, date_from, date_to + 1.days).all
+  end
+
+  def public_values
+    Hash[PUBLIC_ATTRIBUTES.map{ |k| [k, (self.send(k) rescue self[k].to_s)] }]
+  end
+
+  def validate
+    super
+    validates_presence :formatter
+    # INFO: since we want to register geocodings during importing we can't register the table
+    # validates_presence :table_id
+    validates_includes ALLOWED_KINDS, :kind
+  end
+
+  def before_save
+    super
+    self.updated_at = Time.now
+    # cancel if state == 'cancelled'
+  end
+
+  def geocoding_logger
+    @@geocoding_logger ||= CartoDB.unformatted_logger(log_file_path('geocodings.log'))
+  end
+
+  def error
+    additional_info = Carto::GeocoderErrors.additional_info(error_code)
+    if additional_info
+      { title: additional_info.title, description: additional_info.what_about }
+    else
+      { title: 'Geocoding error', description: '' }
+    end
+  end
+
+  # The table geocoder is meant to be instantiated just once.
+  # Memoize the table geocoder or nil if it couldn't be instantiated
+  def table_geocoder
+    if !defined?(@table_geocoder)
+      begin
+        @table_geocoder = Carto::TableGeocoderFactory.get(user,
+                                                          self,
+                                                          CartoDB::GeocoderConfig.instance.get,
+                                                          table_service,
+                                                          original_formatter: formatter,
+                                                          formatter: sanitize_formatter,
+                                                          remote_id: remote_id,
+                                                          countries: country_code,
+                                                          regions: region_code,
+                                                          geometry_type: geometry_type,
+                                                          kind: kind,
+                                                          max_rows: max_geocodable_rows,
+                                                          country_column: country_column,
+                                                          region_column: region_column,
+                                                          log: log)
+      rescue => e
+        @table_geocoder = nil
+        raise e
+      end
+    else
+      @table_geocoder
+    end
+  end
+
+  # INFO: table_geocoder method is very coupled to table model, and we want to use this model during imports, without table yet.
+  # this method allows to inject the dependency to the geocoder
+  # TODO: refactor geocoder instantiation out of this class
+  def force_geocoder geocoder
+    @table_geocoder = geocoder
+  end
+
+  def cancel
+    log.append_and_store "Cancelling job because of user request"
+    table_geocoder.cancel
+  rescue => e
+    log.append_and_store "Error trying to cancel a job because of user request: #{e.inspect}"
+    count ||= 1
+    retry unless (count = count.next) > 5
+    CartoDB::notify_exception(e, user: user)
+  end
+
+  # INFO: this method shall always be called from a queue processor
+  def run!
+    @started_at = Time.now
+    log.append_and_store "Running geocoding job on server #{Socket.gethostname} with PID: #{Process.pid}"
+    if self.force_all_rows == true
+      table_geocoder.reset_cartodb_georef_status
+    else
+      table_geocoder.mark_rows_to_geocode
+    end
+
+    store_geocoder_info(Process.pid)
+    processable_rows = self.class.processable_rows(table_service)
+    if processable_rows == 0
+      self.update(state: 'finished', real_rows: 0, used_credits: 0, processed_rows: 0, cache_hits: 0)
+      self.report
+      return
+    end
+
+    rows_geocoded_before = table_service.owner.in_database.select.from(table_service.sequel_qualified_table_name).where(cartodb_georef_status: true).count rescue 0
+
+    self.run_geocoding!(processable_rows, rows_geocoded_before)
+  rescue => e
+    # state == nil probably means it has failed even before run_geocoding begun
+    handle_geocoding_failure(e, rows_geocoded_before || 0) if state == nil
+    log.append_and_store "Unexpected exception: #{e}"
+    log.append_and_store e.backtrace
+    CartoDB.notify_exception(e)
+    raise e
+  ensure
+    log.store # Make sure the log is stored in DB
+    user.db_service.reset_pooled_connections
+  end
+
+  def run_geocoding!(processable_rows, rows_geocoded_before = 0)
+    log.append_and_store "run_geocoding!()"
+    self.update state: 'started', processable_rows: processable_rows
+    @started_at ||= Time.now
+
+    # INFO: this is where the real stuff is done
+    table_geocoder.run
+    # Sync state to avoid overwrites for state and log
+    refresh
+    return false if cancelled?
+
+    self.update(table_geocoder.update_geocoding_status)
+
+    # TODO better exception handling here
+    raise 'Geocoding failed'  if state == 'failed'
+    raise 'Geocoding timed out'  if state == 'timeout'
+
+    handle_geocoding_success(rows_geocoded_before)
+  rescue => e
+    handle_geocoding_failure(e, rows_geocoded_before)
+  end
+
+  def report(error = nil)
+    payload = metrics_payload(error)
+    CartoDB::Metrics.new.report(:geocoding, payload)
+    payload.delete_if {|k,v| %w{distinct_id email table_id}.include?(k.to_s)}
+    geocoding_logger.info(payload.to_json)
+  end
+
+  def self.processable_rows(table_service, force_all_rows=false)
+    dataset = table_service.owner.in_database.select.from(table_service.sequel_qualified_table_name)
+    if !force_all_rows && dataset.columns.include?(:cartodb_georef_status)
+      dataset = dataset.exclude(cartodb_georef_status: true)
+    end
+    dataset.count
+  end # self.processable_rows
+
+  def calculate_used_credits
+    return 0 unless kind == 'high-resolution' && geocoder_type == 'heremaps'
+    total_rows       = processed_rows.to_i + cache_hits.to_i
+    geocoding_quota  = user.organization.present? ? user.organization.geocoding_quota.to_i : user.geocoding_quota
+    used_geocoding_calls = user.organization_user? ? user.organization.get_geocoding_calls : user.get_geocoding_calls
+    # ::User#get_geocoding_calls includes this geocoding run, so we discount it
+    remaining_quota  = geocoding_quota + total_rows - used_geocoding_calls
+    remaining_quota  = (remaining_quota > 0 ? remaining_quota : 0)
+    used_credits     = total_rows - remaining_quota
+    (used_credits > 0 ? used_credits : 0)
+  end # calculate_used_credits
+
+  def price
+    return 0 unless used_credits.to_i > 0
+    (user.geocoding_block_price * used_credits) / ::User::GEOCODING_BLOCK_SIZE.to_f
+  end # price
+
+  def cost
+    return 0 unless kind == 'high-resolution'
+    processed_rows.to_i * CartoDB::GeocoderConfig.instance.get['cost_per_hit_in_cents'] rescue 0
+  end
+
+  def remaining_quota
+    user.remaining_geocoding_quota
+  end # remaining_quota
+
+  def sanitize_formatter
+    translated_formatter = translate_formatter
+    if translated_formatter =~ SANITIZED_FORMATTER_REGEXP
+      translated_formatter
+    else
+      CartoDB::Logger.warning(message: %{Incorrect formatter string received: "#{formatter}"}, user: user)
+      ''
+    end
+  end
+
+  # {field}, SPAIN => field, ', SPAIN'
+  def translate_formatter
+    translated = formatter.to_s.squish
+    translated.prepend("'") unless /^\{/.match(formatter)
+    translated << "'" unless /\}$/.match(formatter)
+    translated.gsub(/^\{/, '')
+              .gsub(/\}$/, '')
+              .gsub(/\}/, ", '")
+              .gsub(/\{/, "', ")
+  end # translate_formatter
+
+  def max_geocodable_rows
+    # This is an arbitrary number, previously set to 1M
+    return 50000 if user.soft_geocoding_limit?
+    user.remaining_geocoding_quota
+  end
+
+  def successful_rows
+    real_rows.to_i
+  end
+
+  def failed_rows
+    processable_rows.to_i - real_rows.to_i
+  end
+
+  def metrics_payload(exception = nil)
+    payload = {
+      created_at:       created_at,
+      distinct_id:      user.username,
+      username:         user.username,
+      email:            user.email,
+      id:               id,
+      table_id:         table_id,
+      kind:             kind,
+      country_code:     country_code,
+      formatter:        formatter,
+      geocoder_type:    geocoder_type,
+      geometry_type:    geometry_type,
+      processed_rows:   processed_rows,
+      cache_hits:       cache_hits,
+      processable_rows: processable_rows,
+      real_rows:        real_rows,
+      successful_rows:  successful_rows,
+      failed_rows:      failed_rows,
+      price:            price,
+      cost:             cost,
+      used_credits:     used_credits,
+      remaining_quota:  remaining_quota,
+      data_import_id:   data_import_id,
+      batched: self.batched
+    }
+    if state == 'finished' && exception.nil?
+      payload.merge!(
+        success: true
+      )
+    elsif exception.present?
+      payload.merge!(
+        success: false,
+        error: exception.message,
+        backtrace: exception.backtrace
+      )
+    else
+      payload.merge!(
+        success: false,
+        error: error
+      )
+    end
+
+    payload.merge!(queue_time: started_at - created_at) if started_at && created_at
+    payload.merge!(processing_time: finished_at - started_at) if started_at && finished_at
+
+    payload
+  end
+
+  def log
+    @log ||= instantiate_log
+  end
+
+  def cancelled?
+    state == 'cancelled'
+  end
+
+  def timeout?
+    state == 'timeout'
+  end
+
+  private
+
+  def table_service
+    @table_service ||= Table.new(user_table: user_table) if user_table.present?
+  end
+
+  def instantiate_log
+    if self.log_id
+      @log = CartoDB::Log.where(id: log_id).first
+    else
+      @log = CartoDB::Log.new({
+          type: CartoDB::Log::TYPE_GEOCODING,
+          user_id: user.id
+        })
+      @log.store
+      self.log_id = @log.id
+      self.save
+    end
+    @log
+  end
+
+  def handle_geocoding_success(rows_geocoded_before)
+    self.update(cache_hits: table_geocoder.cache.hits) if table_geocoder.respond_to?(:cache)
+    stats_aggregator = CartoDB::Stats::Geocoding.instance
+    stats_aggregator.gauge("requests", "+#{self.processed_rows}") rescue nil
+    stats_aggregator.gauge("cache_hits", "+#{self.cache_hits}") rescue nil
+
+    @finished_at = Time.now
+    self.batched = table_geocoder.used_batch_request?
+    self.geocoder_type = table_geocoder.name
+    geocoded_rows = total_geocoded_rows(rows_geocoded_before)
+    self.update(state: 'finished', real_rows: geocoded_rows, used_credits: calculate_used_credits)
+    send_report_mail(state, self.table_name, nil, self.processable_rows, geocoded_rows)
+    log.append_and_store "Geocoding finished"
+    # In the import table_service could be nil
+    if !table_service.nil?
+      # To store the bbox in visualizations
+      table_service.update_bounding_box
+    end
+    self.report
+  end
+
+  def handle_geocoding_failure(raised_exception, rows_geocoded_before)
+    @finished_at = Time.now
+    self.batched = table_geocoder.nil? ? false : table_geocoder.used_batch_request?
+    self.geocoder_type = table_geocoder.nil? ? '' : table_geocoder.name
+    if raised_exception.is_a? Carto::GeocoderErrors::GeocoderBaseError
+      self.error_code = raised_exception.class.additional_info.error_code
+    end
+    self.update(state: 'failed', processed_rows: 0, cache_hits: 0)
+    CartoDB::notify_exception(raised_exception, user: user)
+    geocoded_rows = total_geocoded_rows(rows_geocoded_before)
+    send_report_mail(state, self.table_name, error_code, self.processable_rows, geocoded_rows)
+    log.append_and_store "Unexpected exception: #{raised_exception}"
+    log.append_and_store raised_exception.backtrace
+    self.report(raised_exception)
+  end
+
+  def total_geocoded_rows(rows_geocoded_before)
+    rows_geocoded_after = table_service.owner.in_database.select.from(table_service.sequel_qualified_table_name).where('cartodb_georef_status is true and the_geom is not null').count rescue 0
+    rows_geocoded_after - rows_geocoded_before
+  end
+
+  # Used in the run! method
+  def send_report_mail(state, table_name, error_code=nil, processable_rows, number_geocoded_rows)
+    geocoding_time = @finished_at - @started_at
+    CartoDB::Geocoder::MailNotifier.new(user.id, state, table_name, error_code, processable_rows, number_geocoded_rows, geocoding_time).notify_if_needed
+  end
+
+  def store_geocoder_info(pid)
+    # we use the object instead of the method because
+    # we don't want to build the object until it has
+    # all the needed data that is created in the run method
+    # of the geocoder object. If not is going to fail because
+    # the object is memoized.
+    self.pid = pid
+    if defined?(@table_geocoder)
+      self.geocoder_type = table_geocoder.name
+    end
+    self.save
+  end
+end

app/models/group.rb

@@ -0,0 +1,8 @@
+require_dependency 'carto/helpers/auth_token_generator'
+
+class Group < Sequel::Model
+
+  many_to_one :organization
+
+  include Carto::AuthTokenGenerator
+end

app/models/layer.rb

@@ -0,0 +1,256 @@
+require_relative 'layer/presenter'
+require_relative 'table/user_table'
+require_relative '../../lib/cartodb/stats/editor_apis'
+require_dependency 'carto/table_utils'
+require_dependency 'carto/query_rewriter'
+require_relative 'carto/layer'
+
+class Layer < Sequel::Model
+  include Carto::TableUtils
+  include Carto::LayerTableDependencies
+  include Carto::QueryRewriter
+
+  plugin :serialization, :json, :options, :infowindow, :tooltip
+
+  ALLOWED_KINDS = %W{ carto tiled background gmapsbase torque wms }
+  BASE_LAYER_KINDS  = %w(tiled background gmapsbase wms)
+  DATA_LAYER_KINDS = ALLOWED_KINDS - BASE_LAYER_KINDS
+
+  PUBLIC_ATTRIBUTES = %W{ options kind infowindow tooltip id order }
+
+  TEMPLATES_MAP = {
+    'table/views/infowindow_light' =>               'infowindow_light',
+    'table/views/infowindow_dark' =>                'infowindow_dark',
+    'table/views/infowindow_light_header_blue' =>   'infowindow_light_header_blue',
+    'table/views/infowindow_light_header_yellow' => 'infowindow_light_header_yellow',
+    'table/views/infowindow_light_header_orange' => 'infowindow_light_header_orange',
+    'table/views/infowindow_light_header_green' =>  'infowindow_light_header_green',
+    'table/views/infowindow_header_with_image' =>   'infowindow_header_with_image'
+  }
+
+  # Sets default order to the maximum order of the sibling layers + 1
+  def set_default_order(parent)
+    max_order = parent.layers_dataset.select(:order).map(&:order).compact.max
+    order = (max_order == nil ? 0 : max_order + 1)
+    self.update(:order => order) if self.order.blank?
+  end
+
+  one_to_many  :layer_node_styles
+  many_to_many :maps,  after_add: proc { |layer, parent| layer.after_added_to_map(parent) }
+  many_to_many :users, after_add: proc { |layer, parent| layer.set_default_order(parent) }
+  many_to_many :user_tables,
+                join_table: :layers_user_tables,
+                left_key: :layer_id, right_key: :user_table_id,
+                reciprocal: :layers, class: ::UserTable
+
+  plugin  :association_dependencies, :maps => :nullify, :users => :nullify, :user_tables => :nullify
+
+  def public_values
+    Hash[ PUBLIC_ATTRIBUTES.map { |attribute| [attribute, send(attribute)] } ]
+  end
+
+  def validate
+    super
+    errors.add(:kind, "not accepted") unless ALLOWED_KINDS.include?(kind)
+    errors.add(:maps, "Viewer users can't edit layers") if maps.find { |m| m.user && m.user.viewer }
+
+    if ((Cartodb.config[:enforce_non_empty_layer_css] rescue true))
+      style = options.include?('tile_style') ? options['tile_style'] : nil
+      if style.nil? || style.strip.empty?
+        errors.add(:options, 'Tile style is empty')
+        stats_aggregator = CartoDB::Stats::EditorAPIs.instance
+        stats_aggregator.increment("errors.layer.empty-css")
+        stats_aggregator.increment("errors.total")
+      end
+    end
+  end
+
+  def before_save
+    super
+    self.updated_at = Time.now
+  end
+
+  def to_json(*args)
+    public_values.merge(
+      infowindow: self.values[:infowindow].nil? ? {} : JSON.parse(self.values[:infowindow]),
+      tooltip: JSON.parse(self.values[:tooltip]),
+      options: self.values[:options].nil? ? {} : JSON.parse(self.values[:options])
+    ).to_json(*args)
+  end
+
+  def after_save
+    super
+    maps.each(&:notify_map_change)
+
+    if data_layer?
+      register_table_dependencies
+      update_layer_node_style
+    end
+  end
+
+  def before_destroy
+    raise CartoDB::InvalidMember.new(user: "Viewer users can't destroy layers") if user && user.viewer
+    maps.each(&:notify_map_change)
+    super
+  end
+
+  def key
+    "rails:layer_styles:#{self.id}"
+  end
+
+  def infowindow_template_path
+    if self.infowindow.present? && self.infowindow['template_name'].present?
+      template_name = TEMPLATES_MAP.fetch(self.infowindow['template_name'], self.infowindow['template_name'])
+      Rails.root.join("lib/assets/javascripts/cartodb/table/views/infowindow/templates/#{template_name}.jst.mustache")
+    else
+      nil
+    end
+  end
+
+  def tooltip_template_path
+    if self.tooltip.present? && self.tooltip['template_name'].present?
+      template_name = TEMPLATES_MAP.fetch(self.tooltip['template_name'], self.tooltip['template_name'])
+      Rails.root.join("lib/assets/javascripts/cartodb/table/views/tooltip/templates/#{template_name}.jst.mustache")
+    else
+      nil
+    end
+  end
+
+  def copy(override_attributes={})
+    attributes = public_values.select { |k, v| k != 'id' }.merge(override_attributes)
+    ::Layer.new(attributes)
+  end
+  alias dup copy
+
+  def data_layer?
+    !base_layer?
+  end
+
+  def torque_layer?
+    kind == 'torque'
+  end
+
+  def base_layer?
+    BASE_LAYER_KINDS.include?(kind)
+  end
+
+  def basemap?
+    ["gmapsbase", "tiled"].include?(kind)
+  end
+
+  def supports_labels_layer?
+    basemap? && options["labels"] && options["labels"]["urlTemplate"]
+  end
+
+  def register_table_dependencies(db=SequelRails.connection)
+    db.transaction do
+      delete_table_dependencies
+      insert_table_dependencies
+    end
+  end
+
+  def rename_table(current_table_name, new_table_name)
+    return self unless data_layer? or torque_layer?
+    target_keys = %w{ table_name tile_style query }
+
+    options = JSON.parse(self[:options])
+    targets = options.select { |key, value| target_keys.include?(key) }
+    renamed = targets.map do |key, value|
+      [key, rename_in(value, current_table_name, new_table_name)]
+    end
+
+    self.options = options.merge(Hash[renamed])
+    self
+  end
+
+  def uses_private_tables?
+    user_tables.select(&:private?).any?
+  end
+
+  def legend
+    options['legend']
+  end
+
+  def get_presenter(options, configuration)
+    CartoDB::LayerModule::Presenter.new(self, options, configuration)
+  end
+
+  def set_option(key, value)
+    return unless data_layer?
+
+    self.options[key] = value
+  end
+
+  def qualified_table_name(viewer_user)
+    "#{viewer_user.sql_safe_database_schema}.#{safe_table_name_quoting(options['table_name'])}"
+  end
+
+  def user
+    map.user if map
+  end
+
+  def qualify_for_organization(owner_username)
+    options['query'] = qualify_query(query, options['table_name'], owner_username) if query
+  end
+
+  def after_added_to_map(map)
+    set_default_order(map)
+    register_table_dependencies
+  end
+
+  def source_id
+    options && options.symbolize_keys[:source]
+  end
+
+  def depends_on?(table)
+    user_tables.map(&:id).include?(table.id)
+  end
+
+  private
+
+  def rename_in(target, anchor, substitution)
+    return if target.nil? || target.empty?
+    regex = /(\A|\W+)(#{anchor})(\W+|\z)/
+    target.gsub(regex) { |match| match.gsub(anchor, substitution) }
+  end
+
+  def delete_table_dependencies
+    # remove_* and remove_all_* do not delete the object from the database
+    # only disassociate the associated object from the receiver
+    user_tables.map { |table| remove_user_table(table) }
+  end
+
+  def insert_table_dependencies
+    affected_tables.map { |table| add_user_table(table) }
+  end
+
+  def tables_from_names(table_names, user)
+    ::Table.get_all_by_names(table_names, user)
+  end
+
+  def affected_table_names(query)
+    query_tables = user.in_database["SELECT unnest(CDB_QueryTablesText(?))", query]
+    query_tables.map(:unnest)
+  end
+
+  def map
+    maps.first
+  end
+
+  def query
+    options.symbolize_keys[:query]
+  end
+
+  def update_layer_node_style
+    style = current_layer_node_style
+    if style
+      style.update_from_layer(self)
+      style.save
+    end
+  end
+
+  def current_layer_node_style
+    return nil unless source_id
+    LayerNodeStyle.find(layer_id: id, source_id: source_id) || LayerNodeStyle.new(layer: self, source_id: source_id)
+  end
+end

app/models/layer/presenter.rb

@@ -0,0 +1,302 @@
+require 'json'
+require 'ejs'
+require_dependency 'carto/table_utils'
+
+module CartoDB
+  # Natural naming would be "Layer", but it collides with Layer class
+  module LayerModule
+    class Presenter
+      include Carto::TableUtils
+
+      EMPTY_CSS = '#dummy{}'
+
+      TORQUE_ATTRS = %w(
+        table_name
+        user_name
+        property
+        blendmode
+        resolution
+        countby
+        torque-duration
+        torque-steps
+        torque-blend-mode
+        query
+        source
+        tile_style
+        named_map
+        visible
+      )
+
+      INFOWINDOW_KEYS = %w(
+        fields template_name template alternative_names width maxHeight
+      )
+
+      def initialize(layer, options={}, configuration={}, decoration_data={})
+        @layer            = layer
+        @options          = options
+        @configuration    = configuration
+        @decoration_data  = decoration_data
+      end
+
+      def to_vizjson_v2
+        if base?(layer)
+          with_kind_as_type(layer.public_values).symbolize_keys
+        elsif torque?(layer)
+          as_torque
+        else
+          {
+            id:         layer.id,
+            type:       'CartoDB',
+            infowindow: infowindow_data_v2,
+            tooltip:    tooltip_data_v2,
+            legend:     layer.legend,
+            order:      layer.order,
+            visible:    layer.public_values['options']['visible'],
+            options:    options_data_v2
+          }
+        end
+      end
+
+      # Old layers_controller directly does layer.to_json, but to be uniform with new controller,
+      # always call through the presenter at least in tests
+      def to_json(*args)
+        layer.to_json(*args)
+      end
+
+      def to_vizjson_v1
+        return layer.public_values.symbolize_keys if base?(layer)
+        {
+          id:         layer.id,
+          kind:       'CartoDB',
+          infowindow: infowindow_data_v1,
+          order:      layer.order,
+          options:    options_data_v1
+        }
+      end
+
+      def to_poro
+        # .merge left for backwards compatibility
+        poro = layer.public_values
+
+        return poro unless poro['options']
+
+        # INFO changed to support new  presenter's way of sending owner
+        if @options[:user] && !poro['options']['user_name'].present?
+          user_name = @options[:user].username
+          schema_name = @options[:user].sql_safe_database_schema
+        elsif poro['options']['user_name'].present?
+          user_name = poro['options']['user_name']
+          schema_name = poro['options']['user_name']
+        end
+
+        if options[:viewer_user] && user_name && poro['options']['table_name']
+          # if the table_name already have a schema don't add another one
+          # this case happens when you share a layer already shared with you
+          if user_name != options[:viewer_user].username && !poro['options']['table_name'].include?('.')
+            poro['options']['table_name'] = safe_schema_and_table_quoting(schema_name, poro['options']['table_name'])
+          end
+        end
+        poro
+      end
+
+      private
+
+      attr_reader :layer, :options, :configuration
+
+      # Decorates the layer presentation with data if needed. nils on the decoration act as removing the field
+      def decorate_with_data(source_hash, decoration_data)
+        decoration_data.each { |key, value|
+          source_hash[key] = value
+          source_hash.delete_if { |k, v|
+            v.nil?
+          }
+        }
+        source_hash
+      end
+
+      def base?(layer)
+        ['tiled', 'background', 'gmapsbase', 'wms'].include? layer.kind
+      end
+
+      def torque?(layer)
+        layer.kind == 'torque'
+      end
+
+      def with_kind_as_type(attributes)
+        decorate_with_data(attributes.merge(type: attributes.delete('kind')), @decoration_data)
+      end
+
+      def as_torque
+        api_templates_type = options.fetch(:https_request, false) ? 'private' : 'public'
+        layer_options = decorate_with_data(
+            # Make torque always have a SQL query too (as vizjson v2)
+            layer.options.merge({ 'query' => wrap(sql_from(layer.options), layer.options) }),
+            @decoration_data
+          )
+
+        {
+          id:         layer.id,
+          type:       'torque',
+          order:      layer.order,
+          legend:     layer.legend,
+          options:    {
+            stat_tag:           options.fetch(:visualization_id),
+            maps_api_template: fix_torque_maps_api_template(ApplicationHelper.maps_api_template(api_templates_type)),
+            sql_api_template: ApplicationHelper.sql_api_template(api_templates_type),
+            # tiler_* is kept for backwards compatibility
+            tiler_protocol:     (configuration[:tiler]["public"]["protocol"] rescue nil),
+            tiler_domain:       (configuration[:tiler]["public"]["domain"] rescue nil),
+            tiler_port:         (configuration[:tiler]["public"]["port"] rescue nil),
+            # sql_api_* is kept for backwards compatibility
+            sql_api_protocol:   (configuration[:sql_api]["public"]["protocol"] rescue nil),
+            sql_api_domain:     (configuration[:sql_api]["public"]["domain"] rescue nil),
+            sql_api_endpoint:   (configuration[:sql_api]["public"]["endpoint"] rescue nil),
+            sql_api_port:       (configuration[:sql_api]["public"]["port"] rescue nil),
+            layer_name:         name_for(layer),
+          }.merge(
+            layer_options.select { |k| TORQUE_ATTRS.include? k })
+        }
+      end
+
+      def infowindow_data_v1
+        with_template(layer.infowindow, layer.infowindow_template_path)
+      rescue => e
+        CartoDB::Logger.error(exception: e)
+        throw e
+      end
+
+      def infowindow_data_v2
+        whitelisted_infowindow(with_template(layer.infowindow, layer.infowindow_template_path))
+      rescue => e
+        CartoDB::Logger.error(exception: e)
+        throw e
+      end
+
+      def tooltip_data_v2
+        whitelisted_infowindow(with_template(layer.tooltip, layer.tooltip_template_path))
+      rescue => e
+        CartoDB::Logger.error(exception: e)
+        throw e
+      end
+
+      def with_template(infowindow, path)
+        # Careful with this logic:
+        # - nil means absolutely no infowindow (e.g. a torque)
+        # - path = nil or template filled: either pre-filled or custom infowindow, nothing to do here
+        # - template and path not nil but template not filled: stay and fill
+        return nil if infowindow.nil?
+        template = infowindow['template']
+        return infowindow if (!template.nil? && !template.empty?) || path.nil?
+
+        infowindow.merge!(template: File.read(path))
+        infowindow
+      end
+
+      def options_data_v2
+        if options[:full]
+          decorate_with_data(layer.options, @decoration_data)
+        else
+          layer_options = layer.options
+          data = {
+            layer_name:         name_for(layer),
+            cartocss:           css_from(layer.options),
+            cartocss_version:   layer.options.fetch('style_version'),
+            interactivity:      layer.options.fetch('interactivity')
+          }
+          source = layer.options['source']
+          if options[:for_named_map] && source
+            data[:source] = { id: source }
+          else
+            data[:sql] = wrap(sql_from(layer.options), layer.options)
+          end
+
+          if layer_options['style_properties'] && (layer_options['style_properties']['autogenerated'] == false || layer_options['style_properties']['autogenerated'].nil?)
+            sql_wrap = layer_options['sql_wrap'] || layer_options['query_wrapper']
+            data['sql_wrap'] = sql_wrap if sql_wrap.present?
+          end
+
+          data = decorate_with_data(data, @decoration_data)
+
+          viewer = options[:viewer_user]
+          if viewer
+            unless data['user_name'] == viewer.username
+              data['table_name'] = "\"#{data['user_name']}\".#{data['table_name']}"
+            end
+          end
+          data
+        end
+      end
+
+      def name_for(layer)
+        layer_alias = layer.options.fetch('table_name_alias', nil)
+        table_name  = layer.options.fetch('table_name')
+
+        return table_name unless layer_alias && !layer_alias.empty?
+        layer_alias
+      end
+
+      def options_data_v1
+        return layer.options if options[:full]
+        layer.options.select { |key, value| public_options.include?(key.to_s) }
+      end
+
+      def sql_from(options)
+        query = options.fetch('query', '')
+        return default_query_for(options) if query.nil? || query.empty?
+        query
+      end
+
+      def css_from(options)
+        style = options.include?('tile_style') ? options['tile_style'] : nil
+        (style.nil? || style.strip.empty?) ? EMPTY_CSS : options.fetch('tile_style')
+      end
+
+      def wrap(query, options)
+        wrapper = options.fetch('query_wrapper', nil)
+        return query if wrapper.nil? || wrapper.empty?
+        EJS.evaluate(wrapper, sql: query)
+      end
+
+      def default_query_for(layer_options)
+        if options[:viewer_user]
+          unless layer_options['user_name'] == options[:viewer_user].username
+            name = if layer_options['user_name'] && layer_options['user_name'].include?('-')
+                     "\"#{layer_options['user_name']}\""
+                   else
+                     layer_options['user_name']
+                   end
+
+            return "select * from #{name}.#{safe_table_name_quoting(layer_options['table_name'])}"
+          end
+        end
+        "select * from #{safe_table_name_quoting(layer_options.fetch('table_name'))}"
+      end
+
+      def public_options
+        return configuration if configuration.empty?
+        configuration.fetch(:layer_opts).fetch("public_opts")
+      end
+
+      def whitelisted_infowindow(infowindow)
+        infowindow.nil? ? nil : infowindow.select { |key, value|
+                                                    INFOWINDOW_KEYS.include?(key) || INFOWINDOW_KEYS.include?(key.to_s)
+                                                  }
+      end
+
+      def fix_torque_maps_api_template(maps_api_template)
+        if visualization_owner_is_table_owner?
+          maps_api_template
+        else
+          # This fixes #9017 avoding Torque request to table owner named map instead of visualization owner.
+          # See https://github.com/CartoDB/torque/blob/8a14fe546ac411829b5f52bb575526ad5ecb79f8/lib/torque/provider/windshaft.js#L361
+          maps_api_template.gsub('{user}', layer.user.username)
+        end
+      end
+
+      def visualization_owner_is_table_owner?
+        layer.options.nil? || layer.options['user_name'].nil? || layer.user.nil? ||
+          layer.options['user_name'] == layer.user.username
+      end
+    end
+  end
+end

app/models/layer_group/presenter.rb

@@ -0,0 +1,56 @@
+require_relative '../layer/presenter'
+
+module CartoDB
+  module LayerGroup
+    class Presenter
+      LAYER_GROUP_VERSION = '1.0.1'
+      DEFAULT_TILER_FILTER = 'mapnik'
+
+      def initialize(layers, options, configuration)
+        @layers         = layers
+        @options        = options
+        @configuration  = configuration
+      end #initialize
+
+      def to_poro
+        return nil if cartodb_layers.empty?
+
+        api_templates_type = options.fetch(:https_request, false) ? 'private' : 'public'
+
+        {
+          type:               'layergroup',
+          options:            {
+            user_name:          options.fetch(:user_name),
+            maps_api_template:  ApplicationHelper.maps_api_template(api_templates_type),
+            sql_api_template:   ApplicationHelper.sql_api_template(api_templates_type),
+            # tiler_* and sql_api_* are kept for backwards compatibility
+            tiler_protocol:     (configuration[:tiler]["public"]["protocol"] rescue nil),
+            tiler_domain:       (configuration[:tiler]["public"]["domain"] rescue nil),
+            tiler_port:         (configuration[:tiler]["public"]["port"] rescue nil),
+            sql_api_protocol:   (configuration[:sql_api]["public"]["protocol"] rescue nil),
+            sql_api_domain:     (configuration[:sql_api]["public"]["domain"] rescue nil),
+            sql_api_endpoint:   (configuration[:sql_api]["public"]["endpoint"] rescue nil),
+            sql_api_port:       (configuration[:sql_api]["public"]["port"] rescue nil),
+            filter:             @configuration[:tiler].fetch('filter', DEFAULT_TILER_FILTER),
+            layer_definition:   {
+              stat_tag:           options.fetch(:visualization_id),
+              version:            LAYER_GROUP_VERSION,
+              layers:             cartodb_layers
+            },
+            attribution: options.fetch(:attributions).join(', ')
+          }
+        }
+      end
+
+      private
+
+      attr_reader :layers, :configuration, :options
+
+      def cartodb_layers
+        @cartodb_layers ||= layers.map do |layer|
+          CartoDB::LayerModule::Presenter.new(layer, options, configuration).to_vizjson_v2
+        end
+      end
+    end
+  end
+end