Initial commit

lib/tasks/acceptance/ghost_tables.rake

@@ -0,0 +1,176 @@
+require 'uuidtools'
+
+namespace :cartodb do
+  namespace :acceptance do
+
+    DEFAULT_SLEEP_TIME = 10
+
+    desc 'Acceptance tests regarding Ghost Tables trigger'
+    task :ghost_tables, [:username, :sleep_time] => :environment do |_t, args|
+      raise 'usage: rake cartodb:acceptance:ghost_tables[username]' if args[:username].blank?
+      @user = User.where(username: args[:username]).first
+      raise "user with username '#{args[:username]}' not found" unless @user
+      args.with_defaults(sleep_time: DEFAULT_SLEEP_TIME)
+      @sleep_time = args[:sleep_time].to_i
+      @results = []
+
+      9.times.each do |index|
+        reset_user
+        @results << send("test#{index + 1}")
+      end
+
+      reset_user
+
+      puts "\n=======================================\n\n"
+      @results.each { |result| p result }
+      puts "\n=======================================\n\n"
+    end
+
+    # helpers
+
+    def reset_user
+      @user.db_service.drop_ghost_tables_event_trigger
+      disable_feature_flag
+      Carto::Visualization.where(user_id: @user.id, name: 'casper').all.each(&:destroy)
+      @user.tables.where(name: 'casper').all.each(&:destroy)
+      @user.in_database.run('DROP TABLE IF EXISTS casper;')
+      @user.db_service.create_ghost_tables_event_trigger
+    end
+
+    def run_and_wait(query)
+      @user.in_database.run(query)
+      p "Waiting #{@sleep_time} seconds..."
+      sleep @sleep_time
+    end
+
+    def create_table_and_cartodbfy
+      run_and_wait(%{
+        CREATE TABLE casper ();
+        SELECT * FROM CDB_CartodbfyTable('casper');
+      })
+    end
+
+    def create_map
+      source = Carto::Visualization.where(user_id: @user.id, name: 'casper').first
+      map = CartoDB::Visualization::TableBlender.new(Carto::User.find(@user.id), [source.user_table]).blend
+      Carto::Visualization.create!(
+        name: 'casper',
+        type: Carto::Visualization::TYPE_DERIVED,
+        user_id: @user.id,
+        map_id: map.id,
+        privacy: Carto::Visualization::PRIVACY_PUBLIC
+      )
+    end
+
+    def enable_feature_flag
+      ff = FeatureFlag.where(name: 'ghost_tables_trigger_disabled').first
+      FeatureFlagsUser.new(feature_flag_id: ff.id, user_id: @user.id).save
+      @user.reload
+    end
+
+    def disable_feature_flag
+      ff = FeatureFlag.where(name: 'ghost_tables_trigger_disabled').first
+      FeatureFlagsUser.where(feature_flag_id: ff.id, user_id: @user.id).first.try(:destroy)
+      @user.reload
+    end
+
+    def build_result(description, condition)
+      result = condition ? "OK!" : "Fail!"
+      "#{description}... #{result}"
+    end
+
+    def table_linked?
+      @user.tables.where(name: 'casper').count == 1
+    end
+
+    # tests
+
+    def test1
+      description = "Create table"
+      run_and_wait("CREATE TABLE casper ();")
+      p build_result(description, !table_linked?)
+    end
+
+    def test2
+      description = "Create table and cartodbfy"
+      create_table_and_cartodbfy
+      p build_result(description, table_linked?)
+    end
+
+    def test3
+      description = "Select into and cartodbfy"
+      run_and_wait(%{
+        SELECT INTO casper FROM (SELECT 1) AS tmp;
+        SELECT * FROM CDB_CartodbfyTable('casper');
+      })
+      p build_result(description, table_linked?)
+    end
+
+    def test4
+      description = "Drop table"
+      create_table_and_cartodbfy
+      assert1 = table_linked?
+
+      run_and_wait("DROP TABLE casper;")
+      assert2 = !table_linked?
+
+      p build_result(description, assert1 && assert2)
+    end
+
+    def test5
+      description = "Create table and cartodbfy without trigger"
+      @user.db_service.drop_ghost_tables_event_trigger
+      create_table_and_cartodbfy
+      p build_result(description, !table_linked?)
+    end
+
+    def test6
+      description = "Drop and create inside a transaction"
+      create_table_and_cartodbfy
+      create_map
+      assert1 = Carto::Visualization.where(user_id: @user.id, type: Carto::Visualization::TYPE_DERIVED).all.count == 1
+
+      run_and_wait(%{
+        DROP TABLE casper;
+        CREATE TABLE casper ();
+        SELECT * FROM CDB_CartodbfyTable('casper');
+      })
+      assert2 = Carto::Visualization.where(user_id: @user.id, type: Carto::Visualization::TYPE_DERIVED).all.count == 1
+
+      p build_result(description, assert1 && assert2)
+    end
+
+    def test7
+      description = "Drop and create without transaction"
+      create_table_and_cartodbfy
+      create_map
+      assert1 = Carto::Visualization.where(user_id: @user.id, type: Carto::Visualization::TYPE_DERIVED).all.count == 1
+
+      run_and_wait("DROP TABLE casper;")
+      create_table_and_cartodbfy
+      assert2 = Carto::Visualization.where(user_id: @user.id, type: Carto::Visualization::TYPE_DERIVED).all.empty?
+
+      p build_result(description, assert1 && assert2)
+    end
+
+    def test8
+      description = "With ghost_tables_trigger_disabled FF"
+      @user.db_service.drop_ghost_tables_event_trigger
+      enable_feature_flag
+      @user.db_service.create_ghost_tables_event_trigger
+
+      create_table_and_cartodbfy
+
+      p build_result(description, !table_linked?)
+    end
+
+    def test9
+      description = "Without TIS configuration"
+      @user.in_database(as: :superuser).run("SELECT cartodb.CDB_Conf_RemoveConf('invalidation_service')")
+
+      create_table_and_cartodbfy
+
+      p build_result(description, !table_linked?)
+    end
+  end
+end

lib/tasks/api_key.rake

@@ -0,0 +1,122 @@
+namespace :carto do
+  namespace :api_key do
+    module ApiKeyRake
+      def for_each_api_key(query, description)
+        all_count = query.count
+        current_count = 0
+        reflection_seconds = ENV['ENABLE_FF_REFLECTION_SECONDS'] || 10
+
+        puts "############################"
+        puts "#"
+        puts "# #{description}"
+        puts "#"
+        puts "# #{all_count} api keys will be processed"
+        puts "#"
+        puts "# You have #{reflection_seconds} seconds to cancel "
+        puts "# the task before it starts"
+        puts "#"
+        puts "############################"
+
+        sleep reflection_seconds.to_i
+
+        query.find_each do |api_key|
+          begin
+            current_count += 1
+            puts "#{current_count} of #{all_count}"
+
+            yield api_key
+          rescue StandardError => e
+            puts "ERROR - API Key #{api_key.id}: #{e}"
+          end
+        end
+      end
+
+      def create_api_keys_for_user(carto_user)
+        if carto_user.api_keys.master.empty?
+          puts "INFO: Creating master API Key for user #{carto_user.username}"
+          carto_user.api_keys.create_master_key!
+        end
+
+        if carto_user.api_keys.default_public.empty?
+          puts "INFO: Creating default public API Key for user #{carto_user.username}"
+          carto_user.api_keys.create_default_public_key!
+        end
+      end
+
+      def rename_api_keys_for_user_id(user_id)
+        Carto::ApiKey.where(user_id: user_id).each do |api_key|
+          api_key.reload
+          n = 1
+          Carto::ApiKey.where(name: api_key.name).where('id != ?', api_key.id).each do |renamed_api_key|
+            renamed_api_key.update_column(:name, "#{renamed_api_key.name} #{n}")
+            n += 1
+          end
+        end
+      end
+    end
+
+    # This should only be used for development, before CartoDB/cartodb/pull/13396 migration
+    desc 'Rename Api Keys to avoid name collisions within the same user'
+    task rename: :environment do
+      include ApiKeyRake
+      Carto::ApiKey.uniq.pluck(:user_id).each { |user_id| rename_api_keys_for_user_id(user_id) }
+    end
+
+    desc 'Creates default API Keys for user'
+    task :create_default_for_emails, [:emails_file] => [:environment] do |_, args|
+      line = 0
+      File.foreach(args[:emails_file]) do |email|
+        email.strip!
+        line += 1
+
+        unless email =~ /.*@.*\..*/
+          puts "Row #{line} is not an email: #{email}"
+          next
+        end
+
+        user = User.first(email: email)
+
+        if user.nil?
+          puts "WARN: User #{email} not found"
+        else
+          begin
+            user.create_api_keys
+            puts "INFO: Api Keys created for #{user.email}"
+          rescue => e
+            puts "WARN: Unable to create api keys for user with email #{email}: #{e}"
+          end
+        end
+      end
+    end
+
+    desc 'Creates default API Keys for users who don\'t have them yet'
+    task create_default: :environment do
+      include ApiKeyRake
+      Carto::User.joins('LEFT JOIN api_keys ON api_keys.user_id = users.id').where(api_keys: { id: nil }).find_each { |u| create_api_keys_for_user(u) }
+    end
+
+    desc 'Creates default API Keys for users who don\'t have them yet under given organization'
+    task :create_default_for_organization, [:org_name] => [:environment] do |_, args|
+      include ApiKeyRake
+      puts "INFO: Creating default API Keys for all users in organization #{args[:org_name]}"
+      org = Carto::Organization.find_by_name(args[:org_name])
+      raise "ERROR: Organization not found" unless org
+      org.users.each { |u| create_api_keys_for_user(u) }
+    end
+
+    desc 'Creates grants for every API Key'
+    task create_api_key_grants: :environment do
+      include ApiKeyRake
+      MESSAGE = "Creating permissions for every API Key".freeze
+
+      for_each_api_key(Carto::ApiKey, MESSAGE) do |api_key|
+        if api_key.master?
+          api_key.grants = Carto::ApiKey::MASTER_API_KEY_GRANTS
+          api_key.save!
+        else
+          api_key.save_cdb_conf_info
+        end
+      end
+    end
+  end
+end

lib/tasks/assets.rake

@@ -0,0 +1,13 @@
+namespace :assets do
+  desc 'Copy digest assets files to non-digest files'
+  task :copy_non_digest_files do
+    manifest = File.read("#{Rails.root}/public/assets/manifest.yml")
+    manifest.each_line do |asset|
+      if asset.include?(':')
+        non_digest = asset.gsub(' ', '').split(':')[0].strip
+        digest = asset.gsub(' ', '').split(':')[1].strip
+        FileUtils.cp "#{Rails.root}/public/assets/#{digest}", "#{Rails.root}/public/assets/#{non_digest}"
+      end
+    end
+  end
+end

lib/tasks/blog.rake

@@ -0,0 +1,42 @@
+namespace :cartodb do
+  namespace :blog do
+    desc 'Get last posts'
+    task :get_last_posts => :environment do
+      # <div class="block">
+      #   <h3>Latest from the blog</h3>
+      #   <h4><a href="#">World Database on Protected Areas</a></h4>
+      #   <p>ProtectedPlanet.net hosts information on over 130,000 protected areas in a distributed CartoDB infrastructure.  The UN Environment Programme is turning to the wiki-world in an attempt to... <a href="#">Read more</a></p>
+      # </div>
+      # <div class="block">
+      #   <h3></h3>
+      #   <h4><a href="#">GROMS Migration data on CartoDB</a></h4>
+      #   <p>The Global Registry of Migratory Species summarizes knowledge about Migratory Species for conservation. The old database was developed in Access and had the geospatial information attached as... <a href="#">Read more</a></p>
+      # </div>
+      # <div class="block last">
+      #   <h3></h3>
+      #   <h4><a href="#">A new status update on blog.carto.com</a></h4>
+      #   <p>Wanted to give everyone an update that we’ve disabled our status blog located at temporarily while we work out all of the kinks in our two shiny new product of two brothers of the man in red... <a href="#">Read more</a></p>
+      # </div>
+
+      feed_url = "https://blog.carto.com/rss"
+      doc = Nokogiri.parse(open(feed_url))
+      content = ""
+      items = doc.search('item')
+      i = 0
+      items[0..2].each do |item|
+        text = item.search('description').first.inner_text.strip_tags.gsub(/^(.{150}[^\s]*)(.*)/m) {$2.empty? ? $1 : $1 + "... <a href=\"#{item.search('guid').first.inner_text}\">Read more</a>"}
+        content += <<-HTML
+<div class="block#{i == 2 ? ' last' : ''}">
+  <h3>#{i == 0 ? "Latest from the blog" : ''}</h3>
+  <h4><a href="#{item.search('guid').first.inner_text}" >#{item.search('title').first.inner_text.truncate(60)}</a></h4>
+  <p>#{text}</p>
+</div>
+HTML
+        i+=1
+      end
+      fd = File.open(CartoDB::LAST_BLOG_POSTS_FILE_PATH, 'w+')
+      fd.write(content)
+      fd.close
+    end
+  end
+end

lib/tasks/cache_maintenance.rake

@@ -0,0 +1,8 @@
+namespace :cartodb do
+  namespace :cache do
+    desc "invalidate Varnish surrogate key"
+    task :invalidate_surrogate_key, [:key] => :environment do |t, args|
+      CartoDB::Varnish.new.purge_surrogate_key(args[:key])
+    end
+  end
+end

lib/tasks/cleanup_importer_tables.rake

@@ -0,0 +1,33 @@
+namespace :cartodb do
+  namespace :db do
+    desc 'Moves left-over tables from failed imports from the public to the cdb_importer schema'
+    task :cleanup_importer_tables => :environment do
+      count = ::User.count
+      ::User.all.each_with_index do |user, index|
+        puts "Cleaning up importer tables for #{user.username}"
+        begin
+          user.in_database.fetch(%Q(
+            SELECT table_name FROM information_schema.tables
+            AS table_name
+            WHERE table_schema = 'public'
+            AND table_name ~ 'importer_\\w{32}'
+          )).map { |record|
+            @table_name = record.fetch(:table_name)
+            user.in_database.run(%Q{
+              DROP TABLE IF EXISTS "cdb_importer"."#{@table_name}"
+            })
+            user.in_database.run(%Q{
+              ALTER TABLE "public"."#{@table_name}"
+              SET SCHEMA "cdb_importer"
+            })
+            puts "------ moved #{@table_name} from user #{user.username}"
+          }
+          printf "OK %-#{20}s (%-#{4}s/%-#{4}s)\n", user.username, index, count
+        rescue => exception
+          printf "FAIL %-#{20}s (%-#{4}s/%-#{4}s) #{exception.message}\n", user.username, index, count
+        end
+        sleep(1.0/5.0)
+      end
+    end
+  end
+end

lib/tasks/common_data.rake

@@ -0,0 +1,10 @@
+namespace :cartodb do
+  namespace :common_data do
+    desc 'Generates datasets assets and upload them to Amazon S3'
+    task :generate_s3_assets, [:all_public] => :environment do |t, args|
+      all_public = args[:all_public].blank? ? false : args[:all_public]
+      common_data = CommonData.new
+      common_data.upload_datasets_to_s3 all_public
+    end
+  end
+end

lib/tasks/connectors_api.rake

@@ -0,0 +1,237 @@
+namespace :cartodb do
+  namespace :connectors do
+
+    desc "Create Connector Providers for Provider Classes"
+    task create_providers: :environment do
+      Carto::Connector.providers(all: true).keys.each do |provider_name|
+        unless Carto::ConnectorProvider.where(name: provider_name).exists?
+          puts "Creating ConnectorProvider #{provider_name}"
+          Carto::ConnectorProvider.create! name: provider_name
+        end
+      end
+      providers = Carto::Connector.providers(all: true).keys.map { |name| "'#{name}'" }
+      Carto::ConnectorProvider.where("name NOT IN (#{providers.join(',')})").each do |provider|
+        puts "Provider #{provider.name} is not configured in the code!"
+      end
+    end
+
+    def find_by_uuid_or_name(id_or_name, klass, name_attribute = :name)
+      include Carto::UUIDHelper
+      if is_uuid?(id_or_name)
+        klass.find(id_or_name)
+      else
+        klass.where(name_attribute => id_or_name).first
+      end
+    end
+
+    def with_provider_user_config(args)
+      provider = find_by_uuid_or_name(args.provider, Carto::ConnectorProvider)
+      user     = find_by_uuid_or_name(args.user, Carto::User, :username)
+      puts "Provider not found: #{args.provider}" if provider.blank?
+      puts "User not found: #{args.user}" if user.blank?
+      if provider.present? && user.present?
+        active_config = user.connector_configuration(provider.name)
+        user_config = Carto::ConnectorConfiguration.where(
+          connector_provider_id: provider.id, user_id: user.id
+        ).first
+        if user.organization.present?
+          org_config = Carto::ConnectorConfiguration.where(
+            connector_provider_id: provider.id, organization_id: user.organization.id
+          ).first
+        end
+        provider_config = Carto::ConnectorConfiguration.default(provider)
+        yield provider, user, active_config, user_config, org_config, provider_config
+      end
+    end
+
+    def with_provider_org_config(args)
+      provider = find_by_uuid_or_name(args.provider, Carto::ConnectorProvider)
+      org      = find_by_uuid_or_name(args.org, Carto::Organization)
+      puts "Provider not found: #{args.provider}" if provider.blank?
+      puts "Organization not found: #{args.org}" if org.blank?
+      if provider.present? && org.present?
+        org_config = Carto::ConnectorConfiguration.where(
+          connector_provider_id: provider.id, organization_id: org.id
+        ).first
+        provider_config = Carto::ConnectorConfiguration.default(provider)
+        yield provider, org, org_config, provider_config
+      end
+    end
+
+    def with_provider_config(args)
+      provider = find_by_uuid_or_name(args.provider, Carto::ConnectorProvider)
+      puts "Provider not found: #{args.provider}" if provider.blank?
+      if provider.present?
+        provider_config = Carto::ConnectorConfiguration.default(provider)
+        yield provider, provider_config
+      end
+    end
+
+    def max_rows_description(max_rows)
+      max_rows.to_i == 0 ? '(unlimited)' : max_rows.to_s
+    end
+
+    desc "Check user connector configuration"
+    task :show_user_config, [:provider, :user] => :environment do |_task, args|
+      with_provider_user_config(args) do |provider, user, config, user_config, org_config, provider_config|
+        puts "Connector #{provider.name} configuration for user #{user.username}:"
+        puts "  Enabled: #{config.enabled?}"
+        puts "  Max. Rows: #{max_rows_description config.max_rows}"
+        if user_config
+          puts "Using user-specific configuration"
+        elsif org_config
+          puts "Using organization #{org_config.organization.name} configuration"
+        elsif provider_config
+          puts "Using #{provider_config.connector_provider.name} defaults"
+        else
+          puts "Using default configuration"
+        end
+      end
+    end
+
+    desc "Set user connector configuration"
+    task :set_user_config, [:provider, :user, :enabled, :max_rows] => :environment do |_task, args|
+      with_provider_user_config(args) do |provider, user, _config, user_config, _org_config, _provider_config|
+        if args.enabled.casecmp('default').zero? && !args.max_rows
+          # rake cartodb:connectors:user_config[provider,user,default] will reset to the default configuration
+          puts "Will reset configuration for #{provider.name} and user #{user.username}"
+          if user_config
+            puts "  Removing existing configuration:"
+            puts "    Enabled: #{user_config.enabled.inspect}"
+            puts "    Max. Rows: #{max_rows_description user_config.max_rows}"
+            user_config.destroy
+          else
+            puts "  User didn't have specific configuration"
+          end
+        else
+          enabled = args.enabled.casecmp('true').zero?
+          max_rows = args.max_rows.to_i
+          max_rows = nil if max_rows == 0
+          if user_config
+            puts "Will update configuration for #{provider.name} and user #{user.username}"
+            puts "  New configuration:"
+            puts "    Enabled: #{enabled.inspect}"
+            puts "    Max. Rows: #{max_rows_description max_rows}"
+            puts "  Existing configuration:"
+            puts "    Enabled: #{user_config.enabled.inspect}"
+            puts "    Max. Rows: #{max_rows_description user_config.max_rows}"
+            user_config.update_attributes! enabled: enabled, max_rows: max_rows
+          else
+            puts "Will create a new configuration for #{provider.name} and user #{user.username}"
+            puts "  New configuration: enabled:"
+            puts "    Enabled: #{enabled.inspect}"
+            puts "    Max. Rows: #{max_rows_description max_rows}"
+            Carto::ConnectorConfiguration.create!(
+              connector_provider: provider,
+              user: user,
+              enabled: enabled,
+              max_rows: max_rows
+            )
+          end
+        end
+      end
+    end
+
+    desc "Check organization connector configuration"
+    task :show_org_config, [:provider, :org] => :environment do |_task, args|
+      with_provider_org_config(args) do |provider, org, org_config, provider_config|
+        puts "Connector #{provider.name} configuration for organization #{org.name}:"
+        if org_config
+          puts "Using organization-specific configuration:"
+          puts "  Enabled: #{org_config.enabled?}"
+          puts "  Max. Rows: #{max_rows_description org_config.max_rows}"
+        elsif provider_config
+          puts "Using #{provider_config.connector_provider.name} defaults:"
+          puts "  Enabled: #{provider_config.enabled?}"
+          puts "  Max. Rows: #{max_rows_description provider_config.max_rows}"
+        else
+          puts "Using default configuration"
+        end
+      end
+    end
+
+    desc "Set organization connector configuration"
+    task :set_org_config, [:provider, :org, :enabled, :max_rows] => :environment do |_task, args|
+      with_provider_org_config(args) do |provider, org, org_config, provider_config|
+        if args.enabled.casecmp('default').zero? && !args.max_rows
+          # rake cartodb:connectors:org_config[provider,org,default] will reset to the default configuration
+          puts "Will reset configuration for #{provider.name} and user #{org.name}"
+          if org_config
+            puts "  Removing existing configuration:"
+            puts "    Enabled: #{org_config.enabled?}"
+            puts "    Max. Rows: #{max_rows_description org_config.max_rows}"
+            org_config.destroy
+          else
+            puts "  Organization didn't have specific configuration"
+          end
+        else
+          enabled = args.enabled.casecmp('true').zero?
+          max_rows = args.max_rows.to_i
+          max_rows = nil if max_rows == 0
+          if org_config
+            puts "Will update configuration for #{provider.name} and org. #{org.name}"
+            puts "  New configuration: enabled:"
+            puts "    Enabled: #{enabled.inspect}"
+            puts "    Max. Rows: #{max_rows_description max_rows}"
+            puts "  Existing configuration: enabled:"
+            puts "    Enabled: #{org_config.enabled?}"
+            puts "    Max. Rows: #{max_rows_description org_config.max_rows}"
+            org_config.update_attributes! enabled: enabled, max_rows: max_rows
+          else
+            puts "Will create a new configuration for #{provider.name} and org. #{org.name}"
+            puts "  New configuration: enabled:"
+            puts "    Enabled: #{enabled.inspect}"
+            puts "    Max. Rows: #{max_rows_description max_rows}"
+            Carto::ConnectorConfiguration.create!(
+              connector_provider: provider,
+              organization: org,
+              enabled: enabled,
+              max_rows: max_rows
+            )
+          end
+        end
+      end
+    end
+
+    desc "Check connector configuration"
+    task :show_config, [:provider] => :environment do |_task, args|
+      with_provider_config(args) do |provider, provider_config|
+        puts "Connector #{provider.name} configuration:"
+        if provider_config
+          puts "Using #{provider_config.connector_provider.name} defaults:"
+          puts "  Enabled: #{provider_config.enabled?}"
+          puts "  Max. Rows: #{max_rows_description provider_config.max_rows}"
+        else
+          puts "Using default configuration"
+        end
+      end
+    end
+
+    desc 'Adapt connector synchronizations to the new API'
+    task adapt_api: :environment do
+
+      def get_ignoring_case(hash, key)
+        _k, v = hash.find { |k, _v| k.to_s.casecmp(key.to_s) == 0 }
+        v
+      end
+
+      # This is meant to be run before deploying the new API (#9674).
+      # Some syncs may fail before deployment is done (because they expect the newer API),
+      # but deployment will be hopefully done before they're retried, so they should succeed then.
+      # No attempt is done to avoid changing 'dead' syncs (left 'created' or retried too many times).
+      # This is idempotent and will do no harm to connectors that use the new API.
+      Carto::Synchronization.where(service_name: 'connector').find_each do |sync|
+        parameters = JSON.load(sync.service_item_id) rescue nil
+        if parameters
+          provider = get_ignoring_case(parameters, 'provider')
+          if !provider
+            puts "Adapting #{sync.id}"
+            parameters['provider'] = 'odbc'
+            sync.service_item_id = parameters.to_json
+            sync.save!
+          end
+        end
+      end
+    end
+  end
+end

lib/tasks/create_importer_schema.rake

@@ -0,0 +1,20 @@
+namespace :cartodb do
+  namespace :db do
+    desc 'Create importer schema and assign privileges to owner'
+    task :create_importer_schema => :environment do
+      count = ::User.count
+      ::User.all.each_with_index do |user, index|
+        begin
+          puts "Creating importer schema for #{user.username}"
+          user.db_service.create_importer_schema
+          user.db_service.set_user_privileges_in_importer_schema
+          printf "OK %-#{20}s (%-#{4}s/%-#{4}s)\n", user.username, index, count
+        rescue => exception
+          printf "FAIL %-#{20}s (%-#{4}s/%-#{4}s) #{exception.message}\n", user.username, index, count
+        end
+        sleep(1.0/5.0)
+      end
+
+    end
+  end
+end

lib/tasks/data_import_log_expiration.rake

@@ -0,0 +1,14 @@
+namespace :dataimport do
+  namespace :log do
+    desc 'Set expiration of DataImport logs to 2 days'
+    task :set_expiration => :environment do
+      redis           = $redis_migrator_logs || Redis.new
+      two_days_secs   = 3600 * 24 * 2
+      block           = lambda { |key| redis.expire(key, two_days_secs) }
+
+      redis.keys("*importer:log:*").each(&block)
+      redis.keys("*importer:entry:*").each(&block)
+    end # set_expiration
+  end # log
+end # dataimport
+

lib/tasks/data_observatory.rake

@@ -0,0 +1,37 @@
+# Datasets CSV example:
+# carto-do-public-data.open_data.geography_usa_state_2015,bq;spanner,999,2020-09-27T08:00:00
+# carto-do-public-data.open_data.demographics_acs_usa_cbsaclipped_2015_yearly_2015,bq,2000,2020-09-27T08:00:00
+namespace :cartodb do
+  namespace :data_observatory do
+    desc "Enables access to DO datasets for a user and saves the metadata in Central and Redis"
+    task :purchase_datasets, [:username, :datasets_csv] => [:environment] do |_, args|
+      username = args[:username]
+      datasets_csv = args[:datasets_csv]
+      usage = 'USAGE: data_observatory:purchase_datasets["username","path/datasets.csv"]'
+      raise usage unless username.present? && datasets_csv.present?
+
+      datasets = []
+      CSV.foreach(args[:datasets_csv]) do |row|
+        available_in = row[1].split(';')
+        dataset = { dataset_id: row[0], available_in: available_in, price: row[2].to_f, expires_at: Time.parse(row[3]) }
+        datasets << dataset
+      end
+
+      Carto::DoLicensingService.new(username).subscribe(datasets)
+
+      puts 'Task finished succesfully!'
+    end
+
+    desc "Removes access to a DO dataset for a user and updates the metadata in Central and Redis"
+    task :remove_purchase, [:username, :dataset_id] => [:environment] do |_, args|
+      username = args[:username]
+      dataset_id = args[:dataset_id]
+      usage = 'USAGE: data_observatory:remove_purchase["username","project.schema.table"]'
+      raise usage unless username.present? && dataset_id.present?
+
+      Carto::DoLicensingService.new(username).unsubscribe(dataset_id)
+
+      puts 'Task finished succesfully!'
+    end
+  end
+end

lib/tasks/dummy.rake

@@ -0,0 +1,5 @@
+namespace :cartodb do
+    task :load_time => :environment do
+      
+    end
+end

lib/tasks/enable_feature.rake

@@ -0,0 +1,174 @@
+namespace :cartodb do
+  namespace :features do
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "enable feature for all users"
+    task :enable_feature_for_all_users, [:feature] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      ::User.all.each do |user|
+        if FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].nil?
+            FeatureFlagsUser.new(feature_flag_id: ff.id, user_id: user.id).save
+        end
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "enable feature for a given user"
+    task :enable_feature_for_user, [:feature, :username] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      user = ::User[username: args[:username]]
+      raise "[ERROR]  User '#{args[:username]}' does not exist" if user.nil?
+
+      if FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].nil?
+        FeatureFlagsUser.new(feature_flag_id: ff.id, user_id: user.id).save
+      else
+        puts "[INFO]  Feature '#{args[:feature]}' was already enabled for user '#{args[:username]}'"
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "enable feature for a given organization"
+    task :enable_feature_for_organization, [:feature, :org_name] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      organization = Organization[name: args[:org_name]]
+      raise "[ERROR]  Organization '#{args[:org_name]}' does not exist" if organization.nil?
+
+      organization.users.each do |user|
+        if FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].nil?
+            FeatureFlagsUser.new(feature_flag_id: ff.id, user_id: user.id).save
+        end
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "disable feature for all users"
+    task :disable_feature_for_all_users, [:feature] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      ffus = FeatureFlagsUser[:feature_flag_id => ff.id]
+      if ffus.nil?
+        puts "[INFO]  No users had feature '#{args[:feature]}' enabled"
+      else
+        ffus.destroy
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "disable feature for a given user"
+    task :disable_feature_for_user, [:feature, :username] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      user = ::User[username: args[:username]]
+      raise "[ERROR]  User '#{args[:username]}' does not exist" if user.nil?
+
+      if FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].nil?
+        puts "[INFO]  Feature '#{args[:feature]}' was already disabled for user '#{args[:username]}'"
+      else
+        FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].destroy
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "disable feature for a given organization"
+    task :disable_feature_for_organization, [:feature, :org_name] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      organization = Organization[name: args[:org_name]]
+      raise "[ERROR]  Organization '#{args[:org_name]}' does not exist" if organization.nil?
+
+      organization.users.each do |user|
+        if FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].nil?
+          puts "[INFO]  Feature '#{args[:feature]}' was already disabled for user '#{args[:username]}'"
+        else
+          FeatureFlagsUser[feature_flag_id: ff.id, user_id: user.id].destroy
+        end
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "add feature flag and optionally set restricted (default is true)"
+    task :add_feature_flag, [:feature, :restricted] => :environment do |_task, args|
+      restricted = args[:restricted] ? args[:restricted].casecmp('false') != 0 : true
+
+      ff = FeatureFlag[name: args[:feature]]
+      if !ff
+        ff = FeatureFlag.new(name: args[:feature], restricted: restricted)
+        ff.id = FeatureFlag.any? ? FeatureFlag.order(:id).last.id + 1 : 0
+        ff.save
+
+        puts "[INFO]\tFeature flag '#{args[:feature]}' created and restricted set to '#{ff.restricted}'"
+      else
+        raise "[ERROR]\tFeature '#{args[:feature]}' already exists and its restricted set to '#{ff.restricted}'"
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "change feature flag to restricted or unrestricted"
+    task :change_feature_restricted, [:feature, :restricted] => :environment do |_task, args|
+      restricted = args[:restricted] ? args[:restricted].casecmp('false') != 0 : true
+
+      ff = FeatureFlag[name: args[:feature]]
+      if ff
+        ff.restricted = restricted
+        ff.save
+
+        puts "[INFO]\tFeature flag '#{args[:feature]}' restricted set to '#{ff.restricted}'"
+      else
+        raise "[ERROR]\tFeature '#{args[:feature]}' doesn't exist"
+      end
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "remove feature flag"
+    task :remove_feature_flag, [:feature] => :environment do |t, args|
+
+      ff = FeatureFlag[:name => args[:feature]]
+      raise "[ERROR]  Feature '#{args[:feature]}' does not exist" if ff.nil?
+
+      ffus = FeatureFlagsUser[:feature_flag_id => ff.id]
+      if ffus.nil?
+        puts "[INFO]  No users had feature '#{args[:feature]}' enabled"
+      else
+        ffus.destroy
+      end
+
+      ff.destroy()
+    end
+
+    # WARNING: For use only at development, opensource and custom installs.
+    # Refer to https://github.com/CartoDB/cartodb-management/wiki/Feature-Flags
+    desc "list all features"
+    task :list_all_features => :environment do
+
+      puts "Available features:"
+      FeatureFlag.all.each do |feature|
+        puts "  - #{feature.name}"
+      end
+    end
+
+  end # Features
+end # CartoDB

lib/tasks/fix.rake

@@ -0,0 +1,30 @@
+# One-time fix tasks
+namespace :carto do
+  namespace :fix do
+    desc "Fix zeroed mapzen quotas for organizations"
+    task :zero_mapzen_quota_orgs => :environment do
+      deploy_time_utc = '2016-10-20 15:32:00'
+      num_orgs = ::Organization.where("updated_at >= '#{deploy_time_utc}' AND mapzen_routing_quota IS NULL").count
+      num_org = 0
+      puts "Fixing #{num_orgs} organizations"
+      ::Organization.where("updated_at >= '#{deploy_time_utc}' AND mapzen_routing_quota IS NULL").each do |org|
+        num_org += 1
+        puts "  fix #{org.name} #{num_org}/#{num_orgs}"
+        org.save_metadata
+      end
+    end
+
+    desc "Fix zeroed mapzen quotas for users"
+    task :zero_mapzen_quota_users => :environment do
+      deploy_time_utc = '2016-10-20 15:32:00'
+      num_users = ::User.where("updated_at >= '#{deploy_time_utc}' AND mapzen_routing_quota IS NULL").count
+      num_user = 0
+      puts "Fixing #{num_users} users"
+      ::User.where("updated_at >= '#{deploy_time_utc}' AND mapzen_routing_quota IS NULL").use_cursor.each do |user|
+        num_user += 1
+        puts "  fix #{user.username} #{num_user}/#{num_users}"
+        user.save_metadata
+      end
+    end
+  end
+end

lib/tasks/fix_unique_legends.rake

@@ -0,0 +1,18 @@
+namespace :cartodb do
+  desc "fixes duplicated unique legends"
+  task fix_unique_legends: :environment do
+    query = 'SELECT layer_id, count(*) from legends group by layer_id having count(*) > 1'
+    ActiveRecord::Base.connection.execute(query).each do |row|
+      legends = Carto::Layer.find(row['layer_id']).legends
+      legend_types = Carto::Legend::LEGEND_TYPES_PER_ATTRIBUTE.keys
+      legends_per_type = legend_types.reduce({}) { |m, o| m.merge(o => []) }
+      legend_per_type = legends.reduce(legends_per_type) do |m, legend|
+        legend_types.reduce(m) do |m2, t|
+          m2[t] << legend if Carto::Legend::LEGEND_TYPES_PER_ATTRIBUTE[t].include?(legend.type)
+          m2
+        end
+      end
+      legend_per_type.each { |_, l| l.sort_by(&:updated_at).slice(0..l.count - 2).each(&:destroy) if l.size > 1 }
+    end
+  end
+end

lib/tasks/fix_unique_overlays.rake

@@ -0,0 +1,17 @@
+namespace :cartodb do
+  desc "fixes duplicated unique overlays"
+  task fix_unique_overlays: :environment do
+    query = <<-SQL
+      select visualization_id, type from overlays
+      where type in ('header', 'search', 'layer_selector', 'share', 'zoom', 'logo', 'loader', 'fullscreen')
+      group by visualization_id, type having count(*) > 1
+    SQL
+    vis_ids = ActiveRecord::Base.connection.execute(query).values
+    overlays = vis_ids.map { |i| Carto::Overlay.where(visualization_id: i[0], type: i[1]) }
+
+    overlays.each do |o|
+      o.to_a.sort! { |l, r| r.order <=> l.order }
+      o.slice(1..o.count).each(&:destroy)
+    end
+  end
+end

lib/tasks/generate_csv.rake

@@ -0,0 +1,24 @@
+require 'csv'
+
+namespace :cartodb do
+  desc 'Generates random CSV files in db/fake_data'
+  task 'generate_random_csv' => :environment do
+    filename = "#{Rails.root}/db/fake_data/csv_file_#{rand(10)}_#{Time.now.to_i}.csv"
+    CSV.open(filename, "wb") do |csv|
+      columns = []
+      100.times do
+        columns << String.random(30)
+      end
+      csv << columns
+      1.upto(100000) do
+        columns = []
+        100.times do
+          columns << String.random(30)
+        end
+        csv << columns
+        putc '.'
+      end
+    end
+    puts "Generated file: #{filename}"
+  end
+end

lib/tasks/google_services.rake

@@ -0,0 +1,189 @@
+namespace :cartodb do
+  namespace :services do
+    namespace :external do
+      namespace :google do
+        def add_feature_flag_if_not_exists(name, restricted = true)
+          ff = Carto::FeatureFlag.where(name: name).first
+
+          if ff.nil?
+            ff = Carto::FeatureFlag.new(name: name, restricted: restricted)
+
+            ff.id = Carto::FeatureFlag.order(:id).last.id + 1
+            ff.restricted = restricted
+          elsif ff.restricted != restricted
+            ff.restricted = restricted
+          end
+
+          ff.save
+        end
+
+        def add_feature_flag_to_user(feature_name, user)
+          ff = Carto::FeatureFlag.where(name: feature_name).first
+          raise "No such feature flag '#{feature_name}'" if ff.nil?
+
+          return if Carto::FeatureFlagsUser.where(feature_flag_id: ff.id, user_id: user.id).exists?
+
+          Carto::FeatureFlagsUser.create(feature_flag_id: ff.id, user_id: user.id)
+        end
+
+        def remove_feature_flag_from_user(feature_name, user)
+          ff = Carto::FeatureFlag.where(name: feature_name).first
+          raise "No such feature flag '#{feature_name}'" if ff.nil?
+
+          ffu = Carto::FeatureFlagsUser.where(feature_flag_id: ff.id, user_id: user.id).first
+
+          if ffu.nil?
+            puts "[BUG] no feature flag user for feature '#{feature_name}' and '#{user.username}'"
+          else
+            ffu.destroy
+          end
+        end
+
+        def remove_feature_flag_if_exists(name)
+          ff = Carto::FeatureFlag.where(name: name).first
+
+          ff.destroy unless ff.nil?
+        end
+
+        def prepend_client_equal_if_needed(key)
+          key.downcase.start_with?('client=') ? key : "client=#{key}"
+        end
+
+        desc 'Enable Google Services for a given user'
+        task :enable_for_user, [:google_maps_key, :google_maps_private_key, :username] => [:environment] do |_, args|
+          add_feature_flag_if_not_exists('google_maps')
+
+          user = ::User.where(username: args[:username]).first
+          raise "No user with username '#{args[:username]}'" if user.nil?
+
+          puts "Enabling 'google_maps' feature flag for user '#{user.username}'"
+          add_feature_flag_to_user('google_maps', user)
+
+          puts "Enabling google services for user '#{user.username}'..."
+          user.google_maps_key = prepend_client_equal_if_needed(args[:google_maps_key])
+          user.google_maps_private_key = args[:google_maps_private_key] unless args[:google_maps_private_key].blank?
+
+          puts user.errors.full_messages unless user.save
+        end
+
+        desc 'Enable Google Services for an organization'
+        task :enable_for_organization,
+             [:google_maps_key, :google_maps_private_key, :org_name] => [:environment] do |_, args|
+          add_feature_flag_if_not_exists('google_maps')
+
+          organization = ::Organization.where(name: args[:org_name]).first
+
+          raise "No organization with name '#{args[:org_name]}'" if organization.nil?
+
+          puts "Enabling 'google_maps' feature flag for users in '#{organization.name}'"
+          organization.users.each do |user|
+            puts "\tenabling 'google_maps' feature flag for user '#{user.username}'"
+            add_feature_flag_to_user('google_maps', user)
+          end
+
+          puts "Enabling google services for organization '#{organization.name}'..."
+          organization.google_maps_key = prepend_client_equal_if_needed(args[:google_maps_key])
+
+          unless args[:google_maps_private_key].blank?
+            organization.google_maps_private_key = args[:google_maps_private_key]
+          end
+
+          puts organization.errors.full_messages unless organization.save
+        end
+
+        desc 'Enable Google Services for all users'
+        task :enable_for_all, [:google_maps_key, :google_maps_private_key] => [:environment] do |_, args|
+          add_feature_flag_if_not_exists('google_maps')
+
+          ::Organization.all.each do |organization|
+            puts "Enabling 'google_maps' feature flag for users in '#{organization.name}'"
+            organization.users.each do |user|
+              puts "\tenabling 'google_maps' feature flag for user '#{user.username}'"
+              add_feature_flag_to_user('google_maps', user)
+            end
+
+            puts "Enabling google services for organization '#{organization.name}'..."
+            organization.google_maps_key = prepend_client_equal_if_needed(args[:google_maps_key])
+
+            unless args[:google_maps_private_key].blank?
+              organization.google_maps_private_key = args[:google_maps_private_key]
+            end
+
+            puts organization.errors.full_messages unless organization.save
+          end
+
+          # For users not in organizations
+          ::User.where(organization_id: nil).each do |user|
+            puts "Enabling 'google_maps' feature flag for user '#{user.username}'"
+            add_feature_flag_to_user('google_maps', user)
+
+            puts "Enabling google services for non-organization user '#{user.username}'..."
+            user.google_maps_key = prepend_client_equal_if_needed(args[:google_maps_key])
+            user.google_maps_private_key = args[:google_maps_private_key] unless args[:google_maps_private_key].blank?
+
+            puts user.errors.full_messages unless user.save
+          end
+        end
+
+        desc 'Disable Google Services for a given user'
+        task :disable_for_user, [:username] => [:environment] do |_, args|
+          user = ::User.where(username: args[:username]).first
+
+          raise "No user with username '#{args[:username]}'" if user.nil?
+
+          puts "Removing 'google_maps' feature flag from user '#{user.username}'"
+          remove_feature_flag_from_user('google_maps', user)
+
+          puts "Disabling google services for user '#{user.username}'..."
+          user.google_maps_key = nil
+          user.google_maps_private_key = nil
+
+          puts user.errors.full_messages unless user.save
+        end
+
+        desc 'Disable Google Services for an organization'
+        task :disable_for_organization, [:org_name] => [:environment] do |_, args|
+          organization = ::Organization.where(name: args[:org_name]).first
+
+          raise "No organization with name '#{args[:org_name]}'" if organization.nil?
+
+          puts "Removing 'google_maps' feature flag from users in '#{organization.name}'"
+          organization.users.each do |user|
+            puts "\tremoving 'google_maps' feature flag from user '#{user.username}'"
+            remove_feature_flag_from_user('google_maps', user)
+          end
+
+          puts "Disabling google services for organization '#{organization.name}'..."
+          organization.google_maps_key = nil
+          organization.google_maps_private_key = nil
+
+          puts organization.errors.full_messages unless organization.save
+        end
+
+        desc 'Disable Google Services for all users'
+        task disable_for_all: [:environment] do |_|
+          remove_feature_flag_if_exists('google_maps')
+
+          ::Organization.all.each do |organization|
+            puts "Disabling google services for organization '#{organization.name}'..."
+
+            organization.google_maps_key = nil
+            organization.google_maps_private_key = nil
+
+            puts organization.errors.full_messages unless organization.save
+          end
+
+          # For users not in organizations
+          ::User.where(organization_id: nil).each do |user|
+            puts "Disabling google services for non-organization user '#{user.username}'..."
+
+            user.google_maps_key = nil
+            user.google_maps_private_key = nil
+
+            puts user.errors.full_messages unless user.save
+          end
+        end
+      end
+    end
+  end
+end

lib/tasks/import.rake

@@ -0,0 +1,93 @@
+require 'fileutils'
+require_relative '../../app/helpers/file_upload'
+
+namespace :cartodb do
+  desc 'Import a file to CartoDB. "times" parameter is there for load test purposes. Defaults to 1.'
+  task :import, [:username, :filepath, :times] => [:environment] do |_task, args|
+    times = (args[:times] || 1).to_i
+    user        = ::User.where(username: args[:username]).first
+    filepath    = File.expand_path(args[:filepath])
+
+    data_import = DataImport.create(
+      :user_id       => user.id,
+      :data_source   => filepath,
+      :updated_at    => Time.now,
+      :append        => false
+    )
+    data_import.values[:data_source] = filepath
+
+    (1..times).each do
+      data_import.run_import!
+      puts data_import.log
+    end
+  end
+
+
+  desc 'Uploads a single data import enqueued file to S3, triggering the normal flow afterwards'
+  task upload_to_s3: [:environment] do
+    data_import_item = DataImport.where(state: DataImport::STATE_ENQUEUED)
+                                 .where(upload_host: Socket.gethostname)
+                                 .order(:created_at)
+                                 .first
+
+    file_upload_helper = CartoDB::FileUpload.new(Cartodb.config[:importer].fetch("uploads_path", nil))
+
+    unless data_import_item.nil?
+      # be 100% safe in rescue blocks when trying to log the failed id
+      data_import_item_id = data_import_item.id rescue nil
+
+      begin
+        puts "Retrieved #{data_import_item.id}"
+
+        data_import_item.state = DataImport::STATE_PENDING
+        data_import_item.save
+
+        filepath = File.realpath data_import_item.data_source
+        filename = File.basename data_import_item.data_source
+
+        uploads_path = File.realpath file_upload_helper.get_uploads_path
+
+        # Files are temp stored in "/%{uploads_path}/token/basename.ext"
+        token = File.basename File.dirname filepath
+
+        file_uri = file_upload_helper.upload_file_to_s3(filepath, filename, token, Cartodb.config[:importer]['s3'])
+        begin
+          File.delete(filepath)
+          folder = filepath.slice(0, filepath.rindex('/')).gsub('..', '')
+          FileUtils.rm_rf(folder) unless folder.length < uploads_path.to_s.length
+        rescue => exception
+          puts "Errored #{data_import_item_id} : #{exception}"
+          CartoDB::notify_error(
+            exception,
+            task: 'cartodb:upload_to_s3 Rake',
+            import_id: data_import_item_id,
+            item: data_import_item.inspect
+          )
+        end
+
+        data_import_item.data_source = file_uri
+        data_import_item.save
+
+        Resque.enqueue(Resque::ImporterJobs, job_id: data_import_item.id)
+
+        puts "Uploaded #{data_import_item.id}"
+      rescue => exception
+        puts "Errored #{data_import_item_id} : #{exception}"
+
+        CartoDB::notify_error(
+          exception,
+          task: 'cartodb:upload_to_s3 Rake',
+          import_id: data_import_item_id,
+          item: data_import_item.inspect
+        )
+
+        data_import_item.state = DataImport::STATE_FAILURE
+        data_import_item.save
+
+        File.delete(filepath)
+        folder = filepath.slice(0, filepath.rindex('/')).gsub('..', '')
+        FileUtils.rm_rf(folder) unless folder.length < file_upload_helper.get_uploads_path.to_s.length
+      end
+    end
+  end
+end

lib/tasks/layers.rake

@@ -0,0 +1,204 @@
+require 'carto/mapcapped_visualization_updater'
+
+namespace :carto do
+  namespace :db do
+    desc "get modified layers"
+    task :get_modified_layers => :environment do
+      if ENV['DATE_AFTER'].blank?
+        raise "Missing DATE_AFTER var"
+      else
+        date_after = ENV['DATE_AFTER']
+      end
+      if ENV['DATE_BEFORE'].blank?
+        raise "Missing DATE_BEFORE var"
+      else
+        date_before = ENV['DATE_BEFORE']
+      end
+
+      layers = Layer.where("updated_at BETWEEN timestamp with time zone '#{date_after}' and timestamp with time zone '#{date_before}'").all
+      final_layers = layers.select do |l|
+        options = JSON.parse(l[:options])
+        options["tile_style_custom"] == false && ["category", "bubble", "torque", "choropleth"].include?(options["wizard_properties"]["type"])
+      end
+      affected_visualizations = []
+      final_layers.each do |l|
+      end
+      affected_visualizations.uniq!
+      puts "Affected layers"
+      puts "---------------"
+      final_layers.each do |l|
+        begin
+          v = l.maps.first.visualizations.first
+          owner = l.maps.first.user
+          affected_visualizations << "#{v.id} | #{v.name} | updated: #{v.updated_at} | user: #{owner.username} | type: #{v.type}"
+          puts "id: #{l.id} | visualization: #{v.id} | vis_type: #{v.type} | updated_at: #{l.updated_at} | user: #{owner.username}"
+        rescue
+          puts "Ignoring orphan layer #{l.id}"
+        end
+      end
+      puts "TOTAL: #{final_layers.length}"
+      puts ""
+      puts "Affected visualizations"
+      puts "-----------------------"
+      affected_visualizations.each do |v|
+        puts v
+      end
+      puts "TOTAL: #{affected_visualizations.length}"
+    end
+
+    desc "CARTO rebranding attribution change"
+    task set_carto_attribution: :environment do
+      puts "Updating layer attributions"
+      ActiveRecord::Base.logger = nil
+
+      total = Carto::Layer.count
+      acc = 0
+      errors = 0
+
+      Carto::Layer.find_each do |layer|
+        acc += 1
+        puts "#{acc} / #{total}" if acc % 100 == 0
+        begin
+          attribution = layer.options['attribution']
+          if attribution.present?
+            attribution.gsub!('CartoDB', 'CARTO')
+            attribution.gsub!('cartodb.com', 'carto.com')
+            attribution.gsub!('http://carto', 'https://carto')
+            attribution.gsub!(
+              'OpenStreetMap</a> contributors &copy; <a href=\"https://carto.com/attributions\">CARTO</a>',
+              'OpenStreetMap</a> contributors')
+          end
+          category = layer.options['category']
+          if category.present? && category == 'CartoDB'
+            layer.options['category'] = 'CARTO'
+          end
+          layer.save
+        rescue => e
+          errors += 1
+          STDERR.puts "Error updating layer #{layer.id}: #{e.inspect}. #{e.backtrace.join(',')}"
+        end
+      end
+
+      puts "Finished. Total: #{total}. Errors: #{errors}"
+    end
+
+    desc "Nokia -> HERE layer update (platform #2815)"
+    task update_nokia_layers: :environment do
+      include Carto::MapcappedVisualizationUpdater
+
+      basemaps = Cartodb.get_config(:basemaps, 'Here')
+
+      puts "Updating base layer urls"
+      layer_dataset = Carto::Layer.where(kind: 'tiled')
+                                  .where("options LIKE '{%' AND options::json->>'className' IN (?)", basemaps.keys)
+      total = layer_dataset.count
+      acc = 0
+      errors = 0
+
+      puts "Updating #{total} layers"
+      layer_dataset.find_each do |layer|
+        acc += 1
+        puts "#{acc} / #{total}" if (acc % 100).zero?
+        begin
+          visualization = layer.visualization
+          next unless visualization
+
+          success = update_visualization_and_mapcap(visualization) do |vis, persisted|
+            vis.user_layers.each do |l|
+              layer_class = l.options[:className]
+              if basemaps.keys.include?(layer_class)
+                l.options[:urlTemplate] = basemaps[layer_class.to_s]['url']
+                l.options[:name] = basemaps[layer_class.to_s]['name']
+                l.save if persisted
+              end
+            end
+          end
+
+          raise 'MapcappedVisualizationUpdater returned false' unless success
+        rescue => e
+          errors += 1
+          STDERR.puts "Error updating layer #{layer.id}: #{e.inspect}. #{e.backtrace.join(',')}"
+        end
+      end
+
+      puts "Finished. Total: #{total}. Errors: #{errors}"
+    end
+
+    module LayersRake
+      def self.update_layer(layer, options, persisted)
+        layer.options = options
+        layer.user.viewer = false # To avoid validation issues
+        layer.save! if persisted
+      end
+    end
+
+    desc "Syncs all layers with the configuration from app_config"
+    task sync_basemaps_from_app_config: :environment do
+      include Carto::MapcappedVisualizationUpdater
+
+      basemaps = Cartodb.get_config(:basemaps)
+
+      basemaps_by_class_name = basemaps.flat_map { |category, classes|
+        if category != 'GMaps'
+          classes.map { |_, attributes|
+            class_name = attributes['className']
+            [class_name, attributes] if class_name.present?
+          }.compact
+        else
+          []
+        end
+      }.compact.to_h
+
+      puts "Updating base layer urls"
+      viz_dataset = Carto::Visualization.where(type: ['table', 'derived'])
+      total = viz_dataset.count
+      acc = 0
+      updated = 0
+      errors = 0
+
+      puts "Updating #{total} visualizations"
+      viz_dataset.eager_load(map: :layers).eager_load(:user).find_each do |visualization|
+        acc += 1
+        puts "#{acc} / #{total}" if (acc % 100).zero?
+        next unless visualization.user
+        begin
+          success = update_visualization_and_mapcap(visualization) do |vis, persisted|
+            # Find visualization tiled base layers
+            base_layers = vis.layers.select(&:tiled?)
+            next true unless base_layers.count.between?(1, 2) # Other kind of basemaps (e.g: plain color), skip
+            bottom_layer, labels_layer = base_layers.sort_by(&:order)
+
+            # Find basemap in configuration
+            class_name = bottom_layer.options['className']
+            attributes = basemaps_by_class_name[class_name]
+            next true unless attributes # Unknown basemap class: do nothing (e.g: custom basemap)
+
+            # Update bottom layer
+            LayersRake.update_layer(bottom_layer, attributes.except('default'), persisted)
+
+            # Update top layer (if present)
+            if labels_layer && attributes['labels']
+              default_labels_layer = Carto::LayerFactory.build_default_labels_layer(bottom_layer)
+              LayersRake.update_layer(labels_layer, default_labels_layer.options, persisted)
+            elsif labels_layer
+              STDERR.puts "WARN: Visualization #{vis.id} has label but basemap class #{class_name} does not"
+            elsif attributes['labels']
+              STDERR.puts "WARN: Basemap class #{class_name} has label but visualization #{vis.id} does not"
+            end
+
+            updated += 1
+
+            true
+          end
+
+          raise 'MapcappedVisualizationUpdater returned false' unless success
+        rescue StandardError => e
+          errors += 1
+          STDERR.puts "Error updating visualization #{visualization.id}: #{e.inspect}. #{e.backtrace.join(',')}"
+        end
+      end
+
+      puts "Finished. Total: #{total}. Updated: #{updated}. Errors: #{errors}"
+    end
+  end
+end

lib/tasks/ldap.rake

@@ -0,0 +1,163 @@
+namespace :cartodb do
+  namespace :ldap do
+    desc "Tests an LDAP connection. Returns a summary of all tests ran, with success status and error messages."
+    task :test_ldap_connection, [:from_database] => :environment do |_t, args|
+      args.with_defaults(from_database: false)
+
+      test_user     = ENV['TEST_USER']
+      test_password = ENV['TEST_PASSWORD']
+      test_search   = ENV['TEST_SEARCH']
+      ldap = if args.from_database
+               Carto::Ldap::Configuration.first
+             else
+               ldap_configuration_from_environment(Carto::Organization.new)
+             end
+
+      # Test configuration
+      config_result = if ldap.valid?
+                        { success: true }
+                      else
+                        { success: false, error: ldap.errors.to_h }
+                      end
+
+      result = { config: config_result }
+
+      REQUIRED_FIELDS = [:host, :port, :connection_user, :connection_password].freeze
+      if !config_result[:success] && config_result[:error].keys.any? { |k| REQUIRED_FIELDS.include?(k) }
+        result[:connection] = { success: false, error: 'Configuration not valid' }
+      end
+
+      # Test connection
+      result[:connection] = ldap.test_connection unless result[:connection]
+
+      CONNECTION_DEPENDENT_TESTS = [:login, :user_search, :group_search, :search].freeze
+      if result[:connection][:success]
+        result[:connection].delete(:connection)
+        if ldap.domain_bases.present?
+          result[:login] = if test_user.present? && test_password.present?
+                            test_ldap_user(ldap, test_user, test_password)
+                           else
+                             { success: false, error: 'Test credentials not provided' }
+                           end
+
+          result[:user_search] = if ldap.user_object_class.present?
+                                   user_count = ldap.users.count
+                                   { success: user_count > 0, count: user_count }
+                                 else
+                                   { success: false, error: 'User class not provided' }
+                                 end
+
+          result[:group_search] = if ldap.group_object_class.present?
+                                    group_count = ldap.groups.count
+                                    { success: group_count > 0, count: group_count }
+                                  else
+                                    { success: false, error: 'Group class not provided' }
+                                  end
+
+          result[:search] = if test_search.present?
+                              object_count = ldap.send(:search_in_domain_bases, Net::LDAP::Filter.eq('objectClass', test_search)).count
+                              { success: true, count: object_count }
+                            else
+                              { success: false, error: 'Object class not provided' }
+                            end
+        else
+          CONNECTION_DEPENDENT_TESTS.each do |test|
+            result[test] = { success: false, error: 'Domain bases not provided' }
+          end
+        end
+      else
+        CONNECTION_DEPENDENT_TESTS.each do |test|
+          result[test] = { success: false, error: 'Connection failed' }
+        end
+      end
+      puts JSON.pretty_generate(result)
+    end
+
+    # INFO: Separate multiple domain names by commas
+    desc "Creates an LDAP Configuration entry"
+    task :create_ldap_configuration, [] => :environment do |_t, _args|
+
+      if ENV['ORGANIZATION_ID'].blank?
+        if ENV['ORGANIZATION_NAME'].blank?
+          raise "Missing ORGANIZATION_ID and ORGANIZATION_NAME. Must provide one of both"
+        else
+          organization = Carto::Organization.where(name: ENV['ORGANIZATION_NAME']).first
+        end
+      else
+        organization = Carto::Organization.find(ENV['ORGANIZATION_ID'])
+      end
+      ldap = ldap_configuration_from_environment(organization)
+
+      unless ldap.valid?
+        missing = ldap.errors.keys.reject { |k| k == :domain_bases_list }
+        raise "Missing: " + missing.join(', ').upcase
+      end
+
+      if ldap.save
+        puts "LDAP configuration created with id: #{ldap.id}"
+      else
+        puts "Error saving LDAP configuration"
+      end
+    end
+
+    desc "Deletes existing LDAP Configuration entries"
+    task :reset_ldap_configuration, [] => :environment do |_t, _args|
+      Carto::Ldap::Configuration.delete_all
+    end
+  end
+
+  private
+
+  def ldap_configuration_from_environment(organization)
+    # Mandatory: connection parameters
+    host = ENV['HOST']
+    port = ENV['PORT']
+    ssl_version = ENV['SSL_VERSION'].blank? ? nil : ENV['SSL_VERSION']
+    encryption = ENV['ENCRYPTION'].blank? ? nil : ENV['ENCRYPTION']
+    connection_user = ENV['CONNECTION_USER']
+    connection_password = ENV['CONNECTION_PASSWORD']
+
+    # Optional: for testing auth/searches
+    user_id_field            = ENV['USER_ID_FIELD']
+    domain_bases             = ENV['DOMAIN_BASES']
+    additional_search_filter = ENV['ADDITIONAL_SEARCH_FILTER']
+
+    # Optional: for testing searches
+    username_field     = ENV['USERNAME_FIELD']
+    email_field        = ENV['EMAIL_FIELD']
+    user_object_class  = ENV['USER_OBJECT_CLASS']
+    group_object_class = ENV['GROUP_OBJECT_CLASS']
+
+    Carto::Ldap::Configuration.new(
+      organization:             organization,
+      host:                     host,
+      port:                     port,
+      encryption:               encryption,
+      ssl_version:              ssl_version,
+      connection_user:          connection_user,
+      connection_password:      connection_password,
+      user_id_field:            user_id_field,
+      username_field:           username_field,
+      additional_search_filter: additional_search_filter,
+      email_field:              email_field,
+      domain_bases:             domain_bases,
+      user_object_class:        user_object_class,
+      group_object_class:       group_object_class
+    )
+  end
+
+  def test_ldap_user(ldap, test_user, test_password)
+    unless ldap.user_id_field.present? && ldap.username_field.present? && ldap.email_field.present?
+      return { success: false, error: 'User id, username or email attribute names not specified.' }
+    end
+
+    entry = ldap.authenticate(test_user, test_password)
+    return { success: false, error: 'Cannot login with test credentials' } unless entry
+
+    if entry.email && entry.username
+      { success: true, email: entry.email, username: entry.username }
+    else
+      { success: false, error: 'Cannot retrieve user attributes. Check the email and username attribute names' }
+    end
+  end
+end

lib/tasks/legends.rake

@@ -0,0 +1,100 @@
+namespace :carto do
+  namespace :legends do
+    namespace :migration do
+      desc 'Migrate html type legends to custom'
+      task migrate_html_to_custom: :environment do
+        html_legends = Carto::Legend.where(type: 'html')
+        html_legends_count = html_legends.count
+
+        errored = []
+        puts "#{html_legends_count} html type legends..."
+        html_legends.each_with_index do |legend, index|
+          if Carto::Layer.exists?(legend.layer_id) && !legend.update_attributes(type: 'custom')
+            errored << legend
+          end
+
+          printf "(#{index}/#{html_legends_count})"
+        end
+
+        2.times { puts }
+
+        unless errored.empty?
+          puts "Errored legend migrations (#{errored.count}):"
+          errored.each do |legend|
+            puts "#{legend.id}: #{legend.errors.full_messages.join(',')}"
+          end
+        end
+      end
+
+      desc 'Fix custom legends with images'
+      task migrate_custom_image: :environment do
+        include Carto::MapcappedVisualizationUpdater
+
+        CSS_URL_REGEX = /^(?:url\(['"]?)(.*?)(?:['"]?\))$/
+        STATIC_ASSETS_REGEX = /http:\/\/com.cartodb.users-assets.production.s3.amazonaws.com(.*)/
+
+        puts "Updating base layer urls"
+        layers = Carto::Layer.joins(:legends).where(legends: { type: 'custom' })
+
+        total = layers.count
+        acc = 0
+        errors = 0
+
+        puts "Updating #{total} layers"
+
+        layers.find_each do |layer|
+          acc += 1
+
+          puts "#{acc} / #{total}" if (acc % 100).zero?
+
+          begin
+            visualization = layer.visualization
+            next unless visualization
+
+            success = update_visualization_and_mapcap(visualization) do |vis, persisted|
+              vis.data_layers.each do |l|
+                acc_legend = 0
+
+                l.legends.each do |legend|
+                  acc_category = 0
+                  categories = legend[:definition][:categories]
+
+                  categories && categories.each do |category|
+                    icon = category[:icon]
+
+                    css_url_match = icon && CSS_URL_REGEX.match(icon)
+                    if css_url_match
+                      icon = css_url_match[1]
+
+                      category[:icon] = icon
+                    end
+
+                    static_assets_match = icon && STATIC_ASSETS_REGEX.match(icon)
+                    if static_assets_match
+                      icon = "https://s3.amazonaws.com/com.cartodb.users-assets.production#{static_assets_match[1]}"
+
+                      category[:icon] = icon
+                    end
+
+                    acc_category += 1
+                  end
+
+                  legend.save if persisted
+
+                  acc_legend += 1
+                end
+              end
+            end
+
+            raise 'MapcappedVisualizationUpdater returned false' unless success
+          rescue => e
+            errors += 1
+            STDERR.puts "Error updating layer #{layer.id}: #{e.inspect}. #{e.backtrace.join(',')}"
+          end
+        end
+
+        puts "Finished. Total: #{total}. Errors: #{errors}"
+      end
+    end
+  end
+end

lib/tasks/map_views.rake

@@ -0,0 +1,45 @@
+namespace :cartodb do
+  namespace :users do
+    desc 'Get map views from remote service for every user'
+    task :update_remote_map_views => :environment do
+      puts "Updating local map views cache for every user..."
+      ::User.all.each do |u|
+        print "  - Update #{u.username}"
+        u.set_old_api_calls # updates map views stats older than 3 hours
+        print " OK\n"
+      end
+    end
+  end
+
+  namespace :mapviews do
+    desc 'Get the sum of all the torque mapviews'
+    task :torque_total => :environment do
+      total_torque_mapviews = 0
+      total_visualizations = 0
+      offset = 0
+      stats_manager = CartoDB::Stats::APICalls.new
+      while (torque_vis = SequelRails.connection.fetch(get_torque_visualization(offset, 1000)).all).length > 0
+        torque_vis.each do |vis|
+          total_visualizations += 1
+          total_torque_mapviews += get_total_vis_mapviews(vis, stats_manager)
+        end
+        offset += torque_vis.length
+      end
+      puts "Total torque mapviews: #{total_torque_mapviews}"
+      puts "Total visualizations with torque layers: #{total_visualizations}"
+    end
+
+    def get_torque_visualization(offset, pagesize)
+      %[SELECT v.id, u.username FROM visualizations v
+        INNER JOIN users u ON v.user_id = u.id
+        INNER JOIN layers_maps lm ON v.map_id = lm.map_id
+        INNER JOIN layers l ON lm.layer_id = l.id
+        WHERE l.kind = 'torque'
+        LIMIT #{pagesize} OFFSET #{offset}]
+    end
+
+    def get_total_vis_mapviews(visualization, stats_manager)
+        stats_manager.get_total_api_calls_from_redis(visualization[:username], visualization[:id])
+    end
+  end
+end

lib/tasks/maps.rake

@@ -0,0 +1,12 @@
+require 'carto/export/map_statistics'
+
+namespace :cartodb do
+  namespace :maps do
+    desc 'Get CSV with all maps info'
+    task map_info_csv: :environment do
+      ms = Carto::Export::MapStatistics.new
+      ms.run!
+      puts "Here's your file: #{ms.filepath}\nThanks for coming :)"
+    end
+  end
+end

lib/tasks/metrics.rake

@@ -0,0 +1,95 @@
+namespace :cartodb do
+  namespace :metrics do
+    # service provider names
+    PROVIDERS = ['heremaps', 'google', 'mapzen', 'mapbox', 'tomtom'].freeze
+    SERVICES = {
+      geocoder: { method: :get_geocoding_calls, column: 'geocoder_provider' },
+      isolines: { method: :get_here_isolines_calls, column: 'isolines_provider' },
+      routing: { method: :get_mapzen_routing_calls, column: 'routing_provider' }
+    }.freeze
+
+    # e.g. bundle exec rake cartodb:metrics:ds_provider_metrics['heremaps','2018-01-01','2018-01-31']
+    #      bundle exec rake cartodb:metrics:ds_provider_metrics['heremaps','2018-01-01','2018-01-31','/tmp/test.csv']
+    desc 'Get DS provider metrics in a defined period of time'
+    task :ds_provider_metrics, [:provider, :from, :to, :output_file] => :environment do |_t, args|
+      provider = args[:provider]
+      from = args[:from].blank? ? nil : args[:from].to_date
+      to = args[:to].blank? ? nil : args[:to].to_date
+      if provider.nil? || from.nil? || to.nil?
+        raise "Either the provider or one of the dates is not provided as argument"
+      end
+      unless PROVIDERS.include?(provider)
+        raise "The provider passed as argument is not correct. The accepted values are #{DS_PROVIDERS}"
+      end
+      default_output_file = "/tmp/ds_metrics_#{provider}_#{from.strftime('%Y%m%d')}_#{to.strftime('%Y%m%d')}.csv"
+      output_file = args[:output_file].blank? ? default_output_file : args[:output_file]
+      date_from = from.strftime('%Y-%m-%d')
+      date_to = to.strftime('%Y-%m-%d')
+      CSV.open(output_file, "wb") do |csv|
+        SERVICES.each do |service, data|
+          Carto::User.where(data[:column] => provider).find_each do |user|
+            usage = nil
+            organization_name = nil
+            if user.organization_owner?
+              usage = user.organization.public_send(data[:method], to: to, from: from)
+              organization_name = user.organization.name
+            elsif not user.organization_user?
+              usage = user.public_send(data[:method], {to: to, from: from})
+            end
+            if !usage.nil? and usage > 0
+              csv << [provider, service, date_from, date_to, organization_name, user.username, usage]
+            end
+          end
+        end
+      end
+    end
+
+    # e.g. bundle exec rake cartodb:metrics:ds_user_metrics['rtorre','2019-01-01','2019-01-31']
+    #      bundle exec rake cartodb:metrics:ds_user_metrics['rtorre','2019-01-01','2019-01-31','/tmp/test.csv']
+    desc 'Get DS daily usage metrics for a user within a period of time'
+    task :ds_user_metrics, [:username, :from, :to, :output_file] => :environment do |_t, args|
+      username = args[:username]
+      from = args.from.to_date
+      to = args.to.to_date
+      default_output_file = "/tmp/ds_metrics_#{username}_#{from.strftime('%Y%m%d')}_#{to.strftime('%Y%m%d')}.csv"
+      args.with_defaults(output_file: default_output_file)
+      user = Carto::User.where(username: username).first
+      CSV.open(args.output_file, "wb") do |csv|
+        SERVICES.each do |service, data|
+          provider = user[data[:column]]
+          from.upto(to) do |date|
+            usage = user.public_send(data[:method], from: date, to: date, orgwise: false)
+            if !usage.nil? && usage > 0
+              csv << [username, service, provider, date.strftime('%Y-%m-%d'), usage]
+            end
+          end
+        end
+      end
+    end
+
+    # e.g. bundle exec rake cartodb:metrics:ds_org_metrics['team','2019-01-01','2019-01-31']
+    #      bundle exec rake cartodb:metrics:ds_org_metrics['team','2019-01-01','2019-01-31','/tmp/test.csv']
+    desc 'Get DS daily usage metrics for a user within a period of time'
+    task :ds_org_metrics, [:orgname, :from, :to, :output_file] => :environment do |_t, args|
+      orgname = args[:orgname]
+      from = args.from.to_date
+      to = args.to.to_date
+      default_output_file = "/tmp/ds_metrics_#{orgname}_#{from.strftime('%Y%m%d')}_#{to.strftime('%Y%m%d')}.csv"
+      args.with_defaults(output_file: default_output_file)
+      organization_id = Carto::Organization.where(name: orgname).first.id
+      CSV.open(args.output_file, "wb") do |csv|
+        Carto::User.where(organization_id: organization_id).find_each do |user|
+          SERVICES.each do |service, data|
+            provider = user[data[:column]]
+            from.upto(to) do |date|
+              usage = user.public_send(data[:method], from: date, to: date, orgwise: false)
+              if !usage.nil? && usage > 0
+                csv << [user.username, service, provider, date.strftime('%Y-%m-%d'), usage]
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/tasks/notify_trending_maps.rake

@@ -0,0 +1,39 @@
+require_relative '../../lib/cartodb/trending_maps'
+require_relative '../../lib/static_maps_url_helper'
+
+namespace :cartodb do
+  namespace :trending_maps do
+    desc "Search for yesterday's trending maps to notify"
+    task :notify, [:simulation] => :environment do |_task, args|
+      simulation = (args[:simulation] == 'true') ? true : false
+
+      puts "SIMULATION MODE!" if simulation
+      trending_maps_lib = CartoDB::TrendingMaps.new
+      trending_maps = trending_maps_lib.get_trending_maps
+
+      trending_maps.each do |visualization_id, data|
+        views = data[:mapviews]
+
+        puts "Notifying trending map #{visualization_id} with a total of #{views} mapviews"
+        visualization = Carto::Visualization.find(visualization_id)
+        preview_image = Carto::StaticMapsURLHelper.new.url_for_static_map_without_request(data[:user],
+                                                                                          'http',
+                                                                                          visualization,
+                                                                                          600,
+                                                                                          300)
+        unless simulation
+          trending_maps_lib.notify_trending_map(visualization_id, views, preview_image)
+
+          user_id = visualization.user.id
+          Carto::Tracking::Events::ScoredTrendingMap.new(user_id,
+                                                         user_id: user_id,
+                                                         visualization_id: visualization.id,
+                                                         mapviews: views).report
+        end
+      end
+
+      trending_maps_lib.send_trending_map_report(trending_maps) unless simulation
+    end
+
+  end
+end

lib/tasks/oauth.rake

@@ -0,0 +1,89 @@
+require_relative '../../lib/cartodb/trending_maps'
+require_relative '../../lib/static_maps_url_helper'
+
+namespace :cartodb do
+  namespace :oauth do
+    desc 'Delete expired oauth access tokens'
+    task destroy_expired_access_tokens: :environment do
+      Carto::OauthAccessToken.expired.find_each do |token|
+        begin
+          token.destroy!
+        rescue StandardError => e
+          CartoDB::Logger.error(message: 'Could not destroy expired access token', exception: e)
+        end
+      end
+    end
+
+    desc 'Delete expired oauth authorization codes'
+    task destroy_expired_authorization_codes: :environment do
+      Carto::OauthAuthorizationCode.expired.find_each do |code|
+        begin
+          code.destroy!
+        rescue StandardError => e
+          CartoDB::Logger.error(message: 'Could not destroy expired authorization code', exception: e)
+        end
+      end
+    end
+
+    desc 'Delete expired oauth refresh tokens'
+    task destroy_expired_refresh_tokens: :environment do
+      Carto::OauthRefreshToken.expired.find_each do |code|
+        begin
+          code.destroy!
+        rescue StandardError => e
+          CartoDB::Logger.error(message: 'Could not destroy expired refresh token', exception: e)
+        end
+      end
+    end
+
+    desc 'Delete all expired oauth objects'
+    task destroy_expired_oauth_keys: [
+      :destroy_expired_access_tokens,
+      :destroy_expired_refresh_tokens,
+      :destroy_expired_authorization_codes
+    ]
+
+    # Since ownership_role creation is already handled in oauth_app_user
+    # this task is meant to run just once for existing oauth_app_users in every cloud
+    desc 'Create ownership roles for oauth_app_users missing it.'
+    task create_ownership_role: :environment do
+      Carto::OauthAppUser.find_each do |oau|
+        begin
+          next if oau.exists_ownership_role?
+
+          oau.create_ownership_role
+          oau.grant_ownership_role_privileges
+        rescue StandardError => e
+          CartoDB::Logger.error(message: 'Could not create ownership role', exception: e)
+        end
+      end
+    end
+
+    desc 'Track existing OauthApps and OauthAppUsers in Segment'
+    task track_oauth_apps: :environment do
+      Carto::OauthApp.find_each do |app|
+        begin
+          app_properties = {
+            user_id: app.user_id,
+            app_id: app.id,
+            app_name: app.name,
+            timestamp: app.created_at
+          }
+          Carto::Tracking::Events::CreatedOauthApp.new(app.user_id, app_properties).report
+
+          app.oauth_app_users.find_each do |app_user|
+            app_user_properties = {
+              user_id: app_user.user_id,
+              app_id: app.id,
+              app_name: app.name,
+              timestamp: app_user.created_at
+            }
+            Carto::Tracking::Events::CreatedOauthAppUser.new(app_user.user_id, app_user_properties).report
+          end
+        rescue StandardError => e
+          CartoDB::Logger.error(message: 'Could not track event', exception: e)
+        end
+      end
+    end
+  end
+end

lib/tasks/permissions.rake

@@ -0,0 +1,87 @@
+namespace :cartodb do
+  namespace :permissions do
+    desc 'setup default permissions for visualizations without them'
+    task :fill_missing_permissions, [:auto_fix] => :environment do |_, args|
+      File.open('log/fill_missing_permissions.log', 'a') do |log_file|
+        auto_fix = args[:auto_fix].present?
+
+        # This uses the new models to avoid triggering side-effects
+        viz = Carto::Visualization.select('visualizations.*')
+                                  .joins('LEFT JOIN permissions ON visualizations.permission_id = permissions.id')
+                                  .where('permissions.id IS NULL OR permissions.entity_id != visualizations.id').all
+        puts "*** Updating permissions for #{viz.count} visualization"
+        puts "*** Automatically fixing conflicts" if auto_fix
+        sleep 5
+        failed_ids = []
+
+        viz.each do |v|
+          begin
+            puts '*** Updating permissions for visualization ' + v.id
+            # Tries to find existing permission
+            perms = Carto::Permission.where(entity_id: v.id).all
+            if perms.empty?
+              puts '  - Creating new default permission'
+
+              user_id = v.user_id ? v.user_id : '00000000-0000-0000-0000-000000000000'
+              username = v.user ? v.user.username : 'cdb_unknown' # Workaround for invalid users
+              perm = Carto::Permission.create(
+                owner_id:       user_id,
+                owner_username: username
+              )
+
+              log_file.puts "Visualization #{v.id}, permission changed from '#{v.permission_id}' to '#{perm.id}' (default)"
+              v.update_column(:permission_id, perm.id)
+            elsif perms.count == 1 || auto_fix
+              puts '  - Reusing existing permission'
+              log_file.puts "Visualization #{v.id}, permission changed from '#{v.permission_id}' to '#{perms.first.id}'"
+              v.update_column(:permission_id, perms.first.id)
+            else
+              puts '  - Multiple permissions, skipping'
+              failed_ids << v.id
+            end
+            sleep 0.2
+          rescue => e
+            puts 'ERROR ' + e.message
+            puts e.backtrace.join("\n")
+            failed_ids << v.id
+          end
+        end
+        unless failed_ids.empty?
+          puts '*** Failed visualizations:'
+          puts failed_ids.join('\n')
+        end
+      end
+    end
+
+    desc 'fix permission pointing to non-existing entities'
+    task :fix_permission_acl => :environment do
+      permission_query = Carto::Permission.where("access_control_list != '[]'")
+
+      # Load the full list of user and group ids into memory for performance
+      uids = Set.new(Carto::User.select(:id).map(&:id))
+      gids = Set.new(Carto::Group.select(:id).map(&:id))
+
+      total = permission_query.count
+      i = 0
+      permission_query.find_each do |p|
+        begin
+          new_acl = p.acl.reject do |acl|
+            acl[:type] == 'user' && !uids.include?(acl[:id]) ||
+              acl[:type] == 'group' && !gids.include?(acl[:id])
+          end
+
+          if new_acl != p.acl
+            puts "Fixing #{p.id}. From #{p.acl} to #{new_acl}"
+            p.update_column(:access_control_list, JSON.dump(new_acl))
+            p.save!
+          end
+
+          i += 1
+          puts "#{i} / #{total}" if (i % 100).zero?
+        rescue => e
+          puts "Unable to update Permission: #{e}"
+        end
+      end
+    end
+  end
+end

lib/tasks/platform_limits.rake

@@ -0,0 +1,92 @@
+namespace :cartodb do
+  desc "Adapt max_import_file_size according to disk quota"
+  task :setup_max_import_file_size_based_on_disk_quota => :environment do
+    mid_size = 500*1024*1024
+    big_size = 1000*1024*1024
+
+    ::User.all.each do |user|
+      quota_in_mb = user.quota_in_bytes/1024/1024
+      if quota_in_mb >= 450 && quota_in_mb < 1500
+        user.max_import_file_size = mid_size
+        user.save
+        print "M"
+      elsif quota_in_mb >= 1500
+        user.max_import_file_size = big_size
+        user.save
+        print "B"
+      else
+        print "."
+      end
+    end
+    puts "\n"
+  end
+
+  desc "Adapt max_import_table_row_count according to disk quota"
+  task :setup_max_import_table_row_count_based_on_disk_quota => :environment do
+    mid_count = 1000000
+    big_count = 5000000
+
+    ::User.all.each do |user|
+      quota_in_mb = user.quota_in_bytes/1024/1024
+      if quota_in_mb >= 50 && quota_in_mb < 1000
+        user.max_import_table_row_count = mid_count
+        user.save
+        print "M"
+      elsif quota_in_mb >= 1000
+        user.max_import_table_row_count = big_count
+        user.save
+        print "B"
+      else
+        print "."
+      end
+    end
+    puts "\n"
+  end
+
+  desc "Increase limits for twitter import users"
+  task :increase_limits_for_twitter_import_users => :environment do
+    file_size_quota = 1500*1024*1024
+    row_count_quota = 5000000
+
+    ::User.where(twitter_datasource_enabled: true).each do |user|
+      # Only increase, don't decrease
+      user.max_import_file_size = file_size_quota if file_size_quota > user.max_import_file_size
+      user.max_import_table_row_count = row_count_quota if row_count_quota > user.max_import_table_row_count
+      user.save
+      puts "#{user.username}"
+    end
+  end
+
+  desc "Set custom platform limits for a user"
+  task :set_custom_limits_for_user, [:username, :import_file_size, :table_row_count, :concurrent_imports] => :environment do |task_name, args|
+
+    raise "Invalid username supplied" if args[:username].nil?
+    raise "Invalid import size" if args[:import_file_size].nil? || args[:import_file_size].to_i <= 0
+    raise "Invalid tabel row count" if args[:table_row_count].nil? || args[:table_row_count].to_i <= 0
+    raise "Invalid concurrent imports" if args[:concurrent_imports].nil? || args[:concurrent_imports].to_i <= 0
+
+    user = ::User.where(username: args[:username]).first
+
+    raise "User not found" if user.nil?
+
+    user.max_import_file_size = args[:import_file_size].to_i
+    user.max_import_table_row_count = args[:table_row_count].to_i
+    user.max_concurrent_import_count = args[:concurrent_imports].to_i
+    user.save
+  end
+
+  desc 'Set max_import_file_size, max_import_table_row_count'
+  task :set_import_limits, [:username, :max_import_file_size, :max_import_table_row_count] => :environment do |_task, args|
+    username = args[:username]
+    raise 'username needed' unless username
+    user = ::User.where(username: username).first
+    raise "user #{username} not found" unless user
+
+
+    user.max_import_file_size = args[:max_import_file_size] if args[:max_import_file_size].present?
+    user.max_import_table_row_count = args[:max_import_table_row_count] if args[:max_import_table_row_count].present?
+
+    user.save
+  end
+
+end

lib/tasks/post_process_requests.rake

@@ -0,0 +1,19 @@
+namespace :cartodb do
+  namespace :log do
+    desc "Take an input with the format 'verb,request,respnse_time' and outputs 'verb,controller,action,response_time'"
+    task :post_process_requests, [:input_file] => :environment do |task, args|
+      input_file = args[:input_file]
+      raise "Error: need to provide an input file. E.g: rake cartodb:log:post_process_requests['input_file.csv']" if input_file.nil?
+
+      routes = Rails.application.routes
+      CSV.foreach(input_file) do |row|
+        verb, request, response_time = row
+        uri = URI.parse(request)
+        r = routes.recognize_path(uri.path, method: verb) rescue next
+        controller = r[:controller]
+        action = r[:action]
+        puts "#{verb},#{controller},#{action},#{response_time}"
+      end
+    end
+  end
+end

lib/tasks/prototypes_tasks.rake

@@ -0,0 +1,18 @@
+namespace :cartodb do
+  namespace :prototypes do
+    desc 'Generates data for Madrid Clubbing prototype'
+    task :import_madrid_clubbing => :environment do
+      puts "Creating table 'Clubbing Madrid'"
+      puts "Importing the data..."
+      user = ::User[1]
+      table = Table.new :privacy => Table::PRIVACY_PUBLIC, :name => 'Clubbing Madrid',
+                        :tags => 'bars, madrid'
+      table.user_id = user.id
+      table.import_from_file = File.open("#{Rails.root}/db/fake_data/clubbing.csv", "r")
+      table.save
+      puts "Geocoding all this data..."
+      puts "Go go go!"
+      table.set_address_column!(:direccion_completa)
+    end
+  end
+end

lib/tasks/query_api_key.rake

@@ -0,0 +1,8 @@
+namespace :user do
+  desc 'Get the API key from a username'
+  task :query_api_key, [:username] => [:environment] do |task, args|
+    user = ::User.where(username: args[:username]).first
+    puts user.api_key
+  end
+end
+

lib/tasks/random_data.rake

@@ -0,0 +1,41 @@
+require Rails.root.join('spec/support/factories/tables')
+
+namespace :cartodb do
+  desc "Create a table with thousands of random rows"
+  task :random_table_with_lots_of_rows => :environment do
+    include CartoDB::Factories
+
+    user = ::User[:username => 'admin']
+    puts '========================'
+    puts 'Creating random table...'
+    table = create_table :user_id => user.id
+    puts '...done!'
+
+    puts 'Inserting random rows into the created table...'
+    user.in_database do |user_database|
+      200_000.times do
+        user_database.run("INSERT INTO \"#{table.name}\" (Name,Latitude,Longitude,Description) VALUES
+                              ('#{String.random(10)}', #{Float.random_latitude}, #{Float.random_longitude}, '#{String.random(100)}')")
+      end
+    end
+    puts '...done!'
+    puts "Created random table with name #{table.name}"
+    puts '========================'
+  end
+
+  desc "Imports a multi-table file"
+  task :import_multitable_file => :environment do
+    require_relative '../../spec/support/helpers'
+    include HelperMethods
+
+      Resque::ImporterJobs.enqueue(::User.first.id,
+        'wadus',
+        '/uploads/0b4186c852395c0c16b2/ESP_adm.zip',
+        nil,
+        nil,
+        nil,
+        nil,
+        nil)
+  end
+
+end

lib/tasks/redis_keys.rake

@@ -0,0 +1,158 @@
+namespace :cartodb do
+  namespace :redis_keys do
+    module ExportRedisKeys
+      def get_database(number)
+        db_numbers[number] || raise('Invalid redis db')
+      end
+
+      def db_numbers
+        @db_numbers || {
+          '0' => $tables_metadata,
+          '3' => $api_credentials,
+          '5' => $users_metadata,
+          '6' => $redis_migrator_logs,
+          '8' => $limits_metadata
+        }.freeze
+      end
+
+      def export_key(rdb, key)
+        ttl = rdb.ttl(key)
+        {
+          ttl: ttl < 0 ? 0 : ttl,
+          value: rdb.dump(key)
+        }
+      end
+
+      def import_key(rdb, key, value)
+        rdb.restore(key, value[:ttl], value[:value])
+      end
+    end
+
+    desc 'export keys in file'
+    task :export, [:filename, :db] => :environment do |_, args|
+      include ExportRedisKeys
+
+      if args[:filename].nil?
+        puts "usage: bundle exec rake cartodb:redis_keys:export[filter_file]\n
+              you must pass a file that contains the redis keys you want to export one per line"
+        exit 1
+      end
+
+      rdb = get_database(args[:db])
+
+      result = {}
+      File.foreach(args[:filename]) do |line|
+        line.strip!
+        result[line] = export_key(rdb, line)
+      end
+
+      File.open('redis_export.json', 'w') { |file| file.write(result.to_json) }
+    end
+
+    desc 'import keys in given json file to redis db'
+    task :import, [:filename, :db, :overwrite] => :environment do |_, args|
+      include ExportRedisKeys
+
+      if args[:filename].nil?
+        puts "usage: bundle exec rake cartodb:redis_keys:import[file.json,redis_db,overwrite]\n
+              File containing a json that will be used for restoring keys. Overwrite defaults to false"
+        exit 1
+      end
+      args.with_defaults(overwrite: 'false')
+      overwrite = args[:overwrite] == 'true'
+
+      redis_hash = JSON.parse(File.read(args[:filename])).deep_symbolize_keys
+
+      rdb = get_database(args[:db])
+      redis_hash.each do |key, value|
+        rdb.del(key) if overwrite && rdb.exists(key)
+        if overwrite || !rdb.exists(key)
+          puts "Importing #{key}"
+          import_key(rdb, key, value)
+        end
+      end
+    end
+
+    desc 'export named_maps key'
+    task :export_named_maps, [:file, :batch_size] => :environment do |_task, args|
+      args.with_defaults(batch_size: 100)
+
+      if args[:file].nil?
+        puts "usage: bundle exec rake cartodb:redis_keys:export_named_maps[filter]\n
+        you must pass a file that contains a filter to select users such as:\n
+        username in ['alex', 'lemmy']"
+        exit 1
+      end
+
+      module ExportNamedMaps
+        def export_users_json_hash(users, batch_size)
+          {
+            redis: export_users(users, batch_size)
+          }
+        end
+
+        def export_users(users, batch_size)
+          export_users_hash = { tables_metadata: {} }
+          users.use_cursor(rows_per_fetch: batch_size).each do |u|
+            puts("Exporting named maps for user #{u.username}")
+            export_users_hash[:tables_metadata].merge!(export_dataservices($tables_metadata, "map_tpl|#{u.username}"))
+          end
+          export_users_hash
+        end
+
+        def export_dataservices(rdb, prefix)
+          rdb.keys(prefix.to_s).map { |key|
+            export_key(rdb, key)
+          }.reduce({}, &:merge)
+        end
+
+        def export_key(redis_db, key)
+          {
+            key => {
+              ttl: [0, redis_db.pttl(key)].max, # PTTL returns -1 if not set, clamp to 0
+              value: redis_db.hgetall(key)
+            }
+          }
+        end
+      end
+
+      include ExportNamedMaps
+
+      users = User.where(File.read(args[:file]).to_s)
+      File.open('redis_export.json', 'w') { |file| file.write(export_users_json_hash(users, args[:batch_size]).to_json) }
+    end
+
+    desc 'import named_maps key '
+    task :import_named_maps, [:filename] => :environment do |_task, args|
+
+      if args[:filename].nil?
+        puts "usage: bundle exec rake cartodb:redis_keys:import_named_maps['redis_export.json']\n
+         you must pass the name of the file with the keys you want to import"
+        exit 1
+      end
+
+      module ImportNamedMaps
+        def restore_redis_from_hash_export(exported_hash)
+          restore_redis(exported_hash[:redis])
+        end
+
+        def restore_redis(redis_export)
+          restore_keys($tables_metadata, redis_export[:tables_metadata])
+        end
+
+        def restore_keys(redis_db, redis_keys)
+          redis_keys.each do |key, value|
+            value[:value].each do |k, v|
+              redis_db.hset(key, k, v) unless redis_db.hexists(key, k)
+            end
+          end
+        end
+      end
+
+      include ImportNamedMaps
+
+      exported_json_string = File.read(Dir[args[:filename]].first)
+      restore_redis_from_hash_export(JSON.parse(exported_json_string).deep_symbolize_keys)
+    end
+  end
+end

lib/tasks/redis_maintainance.rake

@@ -0,0 +1,17 @@
+namespace :cartodb do
+  namespace :redis do
+    desc "update redis with metadata for every user"
+    task :user_metadata => :environment do
+      ::User.all.each do |user|
+        user.save_metadata
+      end
+    end
+
+    desc "purge redis vizjson cache"
+    task :purge_vizjson => :environment do
+      ::User.each do |user|
+        user.purge_redis_vizjson_cache
+      end
+    end
+  end
+end

lib/tasks/remote_tables_maintenance.rake

@@ -0,0 +1,227 @@
+require 'date'
+require_relative '../../app/services/carto/data_library_service'
+require_relative '../../lib/carto_api/json_client'
+
+namespace :cartodb do
+
+  namespace :remotes do
+
+    task :clear, [:username] => [:environment] do |t, args|
+      username = args[:username]
+      raise 'username required' unless username.present?
+
+      u = ::User.where(username: username).first
+      require_relative '../../app/services/visualization/common_data_service'
+      deleted = CartoDB::Visualization::CommonDataService.new.delete_common_data_for_user(u)
+      puts "Deleted #{deleted} remote visualizations"
+    end
+
+    task :clear_org, [:org_name] => [:environment] do |t, args|
+      org_name = args[:org_name]
+      raise 'organization name required' unless org_name.present?
+
+      require_relative '../../app/services/visualization/common_data_service'
+      common_data_service = CartoDB::Visualization::CommonDataService.new
+      o = Organization.where(name: org_name).first
+      o.users.each { |u|
+        common_data_service.delete_common_data_for_user(u)
+      }
+    end
+
+    desc 'Load common data account remotes. Pass username as first argument. Example: `rake cartodb:remotes:reload[development]`'
+    task :reload, [:username] => [:environment] do |t, args|
+      username = args[:username]
+      raise 'username required' unless username.present?
+
+      u = ::User.where(username: username).first
+      require_relative '../../app/services/visualization/common_data_service'
+      vis_api_url = get_visualizations_api_url
+      CartoDB::Visualization::CommonDataService.new.load_common_data_for_user(u, vis_api_url)
+    end
+
+    desc 'Load common data account remotes for a whole organization. Pass organization name as first argument. Example: `rake cartodb:remotes:reload[my_team]`'
+    task :reload_org, [:org_name] => [:environment] do |t, args|
+      org_name = args[:org_name]
+      raise 'organization name required' unless org_name.present?
+
+      require_relative '../../app/services/visualization/common_data_service'
+      common_data_service = CartoDB::Visualization::CommonDataService.new
+      vis_api_url = get_visualizations_api_url
+      o = Organization.where(name: org_name).first
+      o.users.each {|u|
+        common_data_service.load_common_data_for_user(u, vis_api_url)
+      }
+    end
+
+    desc 'Load common data account remotes for multiple users, in alphabetical order. If you pass a username, it will do it beginning in the next username'
+    task :load_all, [:from_username] => [:environment] do |t, args|
+      require_relative '../../app/services/visualization/common_data_service'
+      common_data_service = CartoDB::Visualization::CommonDataService.new
+      vis_api_url = get_visualizations_api_url
+      puts DateTime.now
+      # TODO: batch
+      users = ::User.order_by(:username)
+      users = users.where("username > '#{args[:from_username]}'") unless args[:from_username].nil?
+      users.all.each do |user|
+        added, updated, not_modified, removed, failed = common_data_service.load_common_data_for_user(user, vis_api_url)
+        printf("%20s: +%03d; *%03d; =%03d; -%03d; e%03d\n", user.username, added, updated, not_modified, removed, failed)
+      end
+      puts DateTime.now
+    end
+
+    load_in_data_library_mandatory_args = [:scheme, :base_domain, :port, :source_username,
+                                           :source_api_key, :target_username, :granted_api_key]
+    load_in_data_library_optional_args = [:format]
+    load_in_data_library_from_api_key_args = load_in_data_library_mandatory_args + load_in_data_library_optional_args
+    # Example: rake cartodb:remotes:load_in_data_library_from_api_key[https,carto.com,80,s_user,s_user_key,t_user,g_key,[format]]
+    desc 'Loads all datasets for a API key in a Data Library. `granted_api_key` will be stored for importing.'
+    task :load_in_data_library_from_api_key, load_in_data_library_from_api_key_args => [:environment] do |_, args|
+      raise "All arguments are mandatory" unless load_in_data_library_mandatory_args.all? { |a| args[a].present? }
+      format_arg = args[:format] || 'gpkg'
+
+      client = CartoAPI::JsonClient.new(
+        scheme: args['scheme'], base_domain: args['base_domain'], port: args['port'].to_i
+      )
+      service = Carto::DataLibraryService.new
+      service.load_datasets!(client,
+                             source_username: args['source_username'], source_api_key: args['source_api_key'],
+                             target_username: args['target_username'], granted_api_key: args['granted_api_key'],
+                             format: format_arg)
+    end
+
+    load_in_data_library_mandatory_with_source_args = load_in_data_library_mandatory_args + ['source_dataset']
+    load_in_data_library_args = load_in_data_library_mandatory_with_source_args + load_in_data_library_optional_args
+    # Example: rake cartodb:remotes:load_in_data_library[https,carto.com,80,s_data,s_user,s_user_key,t_user,g_key, source_dataset, [format]]
+    desc 'Loads a dataset in a Data Library. `granted_api_key` is the key that will be stored for importing.'
+    task :load_in_data_library, load_in_data_library_args => [:environment] do |_, args|
+      raise "All arguments except format are mandatory" unless load_in_data_library_mandatory_with_source_args.all? { |a| args[a].present? }
+      format_arg = args[:format] || 'gpkg'
+
+      client = CartoAPI::JsonClient.new(
+        scheme: args['scheme'], base_domain: args['base_domain'], port: args['port'].to_i
+      )
+      service = Carto::DataLibraryService.new
+      service.load_dataset!(client,
+                            source_dataset: args['source_dataset'], source_username: args['source_username'],
+                            source_api_key: args['source_api_key'],
+                            target_username: args['target_username'], granted_api_key: args['granted_api_key'],
+                            format: format_arg)
+    end
+
+    desc 'Removes a Data Library entry from a user'
+    task :remove_from_data_library, [:username, :visualization_name] => [:environment] do |_, args|
+      username = args['username']
+      name = args['visualization_name']
+      raise 'All Arguments except format are mandatory' unless username.present? && name.present?
+
+      user = Carto::User.find_by_username(username)
+      raise 'User not found' unless user
+
+      visualization = user.visualizations.remotes.find_by_name(name)
+      raise 'Visualization not found' unless visualization
+
+      visualization.destroy
+    end
+
+    desc "Invalidate user's date flag and make them refresh data library"
+    task :invalidate_common_data => [:environment] do
+      require_relative '../../app/helpers/common_data_redis_cache'
+      require_relative '../../app/services/visualization/common_data_service'
+
+      invalidate_sql = %Q[
+          UPDATE users
+          SET last_common_data_update_date = null
+          WHERE last_common_data_update_date >= now() - '#{::User::COMMON_DATA_ACTIVE_DAYS} day'::interval;
+        ]
+      updated_rows = SequelRails.connection.fetch(invalidate_sql).update
+      CommonDataRedisCache.new.invalidate
+      puts "#{updated_rows} users invalidated"
+
+      # Now we try to add the new common-data request to the cache using the common_data user
+      common_data_user = ::User.where(username: Cartodb.config[:common_data]["username"]).first
+      if !common_data_user.nil?
+        vis_api_url = get_visualizations_api_url
+        CartoDB::Visualization::CommonDataService.new.load_common_data_for_user(common_data_user, vis_api_url)
+      end
+    end
+
+    def get_visualizations_api_url
+      common_data_config = Cartodb.config[:common_data]
+      username = common_data_config["username"]
+      base_url = common_data_config["base_url"].nil? ? CartoDB.base_url(username) : common_data_config["base_url"]
+      base_url + "/api/v1/viz?type=table&privacy=public"
+    end
+
+    module RemoteTablesMaintenanceRake
+      def self.delete_remote_visualizations(user)
+        user.update_column(:last_common_data_update_date, nil)
+
+        user.visualizations.where(type: 'remote').each do |v|
+          begin
+            unless v.external_source
+              puts "  Remote visualization #{v.id} does not have a external source. Skipping..."
+              next
+            end
+            if v.external_source.external_data_imports.any?
+              puts "  Remote visualization #{v.id} has been previously imported. Skipping..."
+              next
+            end
+
+            v.external_source.delete
+            v.delete
+          rescue => e
+            puts "  Error deleting visualization #{v.id}: #{e.message}"
+          end
+        end
+      end
+    end
+
+    # Removes common data visualizations from users which have not seen activity in some time
+    # e.g: rake cartodb:remotes:remove_from_inactive_users[365,90] will affect all users
+    # whose last activity was between 1 year and 3 months ago
+    desc 'Remove common data visualizations from inactive users'
+    task :remove_from_inactive_users, [:start_days_ago, :end_days_ago] => :environment do |_t, args|
+      start_days_ago = args[:start_days_ago].try(:to_i) || 365 * 2
+      end_days_ago = args[:end_days_ago].try(:to_i) || 30 * 3
+      raise 'Invalid date interval' unless start_days_ago > end_days_ago && end_days_ago > 0
+      start_date = DateTime.now - start_days_ago
+      end_date = DateTime.now - end_days_ago
+
+      puts "Removing common data visualizations for users with last activity between #{start_date} and #{end_date}"
+      query = Carto::User.where("COALESCE(dashboard_viewed_at, created_at) BETWEEN '#{start_date}' AND '#{end_date}'")
+                         .where(account_type: 'FREE')
+      user_count = query.count
+      puts "#{user_count} users will be affected. Starting in 10 seconds unless canceled (ctrl+C)"
+      sleep 10
+
+      processed = 0
+      query.find_each do |user|
+        processed += 1
+        puts "#{user.username} (#{processed} / #{user_count})"
+        RemoteTablesMaintenanceRake.delete_remote_visualizations(user)
+      end
+    end
+
+    # Removes common data visualizations from a specific user
+    desc 'Remove common data visualizations from user'
+    task :remove_from_user, [:username] => :environment do |_t, args|
+      RemoteTablesMaintenanceRake.delete_remote_visualizations(Carto::User.find_by_username(args[:username]))
+    end
+
+    # Removes common data visualizations from a specific organization
+    desc 'Remove common data visualizations from user'
+    task :remove_from_organization, [:orgname] => :environment do |_t, args|
+      query = Carto::User.where(organization_id: Carto::Organization.find_by_name(args[:orgname]).id)
+      user_count = query.count
+      puts "#{user_count} users will be affected."
+
+      processed = 0
+      query.find_each do |user|
+        processed += 1
+        puts "#{user.username} (#{processed} / #{user_count})"
+        RemoteTablesMaintenanceRake.delete_remote_visualizations(user)
+      end
+    end
+  end
+
+end

lib/tasks/resque.rake

@@ -0,0 +1,44 @@
+namespace :resque do
+
+  task "setup" => :environment do
+    Resque.before_fork do |job|
+      #we disconnect the worker so it reconnects on each job
+      SequelRails.connection.disconnect 
+      Resque::Metrics.before_fork.call(job)
+    end
+    Resque.after_fork = Resque::Metrics.after_fork
+  end
+
+  desc "Quit running workers"
+  task :stop_workers => :environment do
+    pids = []
+    Resque.workers.each do |worker|
+      pids << worker.id.split(':')[1]
+    end
+    if pids.empty?
+      puts "No workers to kill"
+    else
+      syscmd = "kill -s QUIT #{pids.join(' ')}"
+      system(syscmd)
+    end
+  end
+  
+  desc "Clear pending tasks"
+  task :clear => :environment do
+    queues = Resque.queues
+    queues.each do |queue_name|
+      puts "Clearing #{queue_name}..."
+      Resque.redis.del "queue:#{queue_name}"
+    end
+    
+    puts "Clearing delayed..." # in case of scheduler - doesn't break if no scheduler module is installed
+    Resque.redis.keys("delayed:*").each do |key|
+      Resque.redis.del "#{key}"
+    end
+    Resque.redis.del "delayed_queue_schedule"
+    
+    puts "Clearing stats..."
+    Resque.redis.set "stat:failed", 0 
+    Resque.redis.set "stat:processed", 0
+  end
+end

lib/tasks/saml.rake

@@ -0,0 +1,75 @@
+namespace :cartodb do
+  namespace :saml do
+    # Sets (inserts or overrides) SAML configuration for an organization. The following environments variables are needed:
+    # ORGANIZATION_NAME: name of the organization. Example: 'orgname'.
+    # SAML_ISSUER: [OPTIONAL] Name of the service provider in the SAML server. Example: 'CARTO_SAML_Test'.
+    #   Default: URL similar to `http://192.168.20.2/user/orgname/saml/metadata` or `http://orgname.domain.com/saml/metadata`, depending on your configuration.
+    # SAML_EMAIL_ATTRIBUTE: attribute with the user email. Example: 'email'.
+    # SAML_ASSERTION_CONSUMER_SERVICE_URL: [OPTIONAL] CARTO URL for SAML, including organization name. Examples: 'http://192.168.20.2/user/orgname/saml/finalize'. Defaults to the URL built from configuration and organization name
+    # SAML_SINGLE_LOGOUT_SERVICE_URL: [OPTIONAL] CARTO URL for SAML logout, including organization name. Examples: 'http://192.168.20.2/user/orgname/logout'. Defaults to the URL built from configuration and organization name
+    # SAML_SLO_DIGEST_METHOD: [OPTIONAL] Digest method to use in signed logout request. By default: 'http://www.w3.org/2001/04/xmlenc#sha256'
+    # SAML_SLO_SIGNATURE_METHOD: [OPTIONAL] Signature method to use in signed logout requests. By default: 'http://www.w3.org/2001/04/xmldsig-more#rsa-sha256'
+    # SAML_SP_PRIVATE_KEY_FILE: [OPTIONAL] Private key file used for signed logout requests
+    # SAML_SP_CERTIFICATE_FILE: [OPTIONAL] Certificate file used for signed logout requests, in .pem format
+    #
+    # Option 1. Manual configuration
+    # SAML_IDP_SSO_TARGET_URL: SAML Identity Provider login URL. Example: 'http://192.168.20.2/simplesaml/saml2/idp/SSOService.php'.
+    # SAML_IDP_SLO_TARGET_URL: [OPTIONAL] SAML Identity Provider logout URL. Example: 'http://192.168.20.2/simplesaml/saml2/idp/SingleLogoutService.php'.
+    # SAML_IDP_CERT_FINGERPRINT: SAML server certificate fingerprint. Command: `openssl x509 -noout -fingerprint -in "./cert/server.crt`. Example: '8C:47:97:B1:E2:E4:6C:06:B5:56:11:8A:5A:8B:53:5C:01:05:CB:05'.
+    # SAML_NAME_IDENTIFIER_FORMAT: [OPTIONAL] Format of the name identifier parameter. Example: 'urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified'. Defaults to 'urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress'.
+    #
+    # Option 2. Metadata parsing
+    # SAML_IDP_METADATA_FILE: Url or file that contains metadata about the IdP. Example: 'http://192.168.20.2/saml2/idp/metadata.php'.
+    task :create_saml_configuration, [] => :environment do |_t, _args|
+      organization = Carto::Organization.where(name: ENV['ORGANIZATION_NAME']).first
+      raise "Organization not found: #{ENV['ORGANIZATION_NAME']}" unless organization
+
+      configuration = if ENV['SAML_IDP_METADATA_FILE'].present?
+                        idp_metadata_parser = OneLogin::RubySaml::IdpMetadataParser.new
+                        settings = idp_metadata_parser.parse_remote(ENV['SAML_IDP_METADATA_FILE'])
+
+                        {
+                          idp_sso_target_url: settings.idp_sso_target_url,
+                          idp_slo_target_url: settings.idp_slo_target_url,
+                          idp_cert_fingerprint: settings.idp_cert_fingerprint,
+                          name_identifier_format: settings.name_identifier_format
+                        }
+                      else
+                        config = {
+                          idp_sso_target_url: ENV['SAML_IDP_SSO_TARGET_URL'],
+                          idp_cert_fingerprint: ENV['SAML_IDP_CERT_FINGERPRINT'],
+                          name_identifier_format: ENV['SAML_NAME_IDENTIFIER_FORMAT']
+                        }
+                        config[:idp_slo_target_url] = ENV['SAML_IDP_SLO_TARGET_URL'] if ENV['SAML_IDP_SLO_TARGET_URL'].present?
+
+                        config
+                      end
+
+      if ENV['SAML_SP_PRIVATE_KEY_FILE'].present? && ENV['SAML_SP_CERTIFICATE_FILE'].present? &&
+        configuration[:name_identifier_format].present? && configuration[:idp_slo_target_url].present?
+        configuration[:security] = {
+          logout_requests_signed: true,
+          logout_responses_signed: true,
+          digest_method: ENV['SAML_SLO_DIGEST_METHOD'] || XMLSecurity::Document::SHA256,
+          signature_method: ENV['SAML_SLO_SIGNATURE_METHOD'] || XMLSecurity::Document::RSA_SHA256
+        }
+
+        configuration[:private_key] = File.read(ENV['SAML_SP_PRIVATE_KEY_FILE'])
+        configuration[:certificate] = File.read(ENV['SAML_SP_CERTIFICATE_FILE'])
+      end
+
+      base_url = CartoDB.base_url(organization.name)
+      configuration[:issuer] = ENV['SAML_ISSUER'] || base_url + '/saml/metadata'
+      configuration[:email_attribute] = ENV['SAML_EMAIL_ATTRIBUTE']
+      configuration[:assertion_consumer_service_url] = ENV['SAML_ASSERTION_CONSUMER_SERVICE_URL'] || base_url + '/saml/finalize'
+      configuration[:single_logout_service_url] = ENV['SAML_SINGLE_LOGOUT_SERVICE_URL'] || base_url + '/logout'
+      configuration[:name_identifier_format] ||= 'urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress'
+
+      raise "Missing parameter: #{configuration}" unless configuration.values.all?(&:present?)
+
+      organization.update_attribute(:auth_saml_configuration, configuration)
+
+      puts "Configuration metadata is available at #{base_url}/saml/metadata"
+    end
+  end
+end

lib/tasks/services.rake

@@ -0,0 +1,135 @@
+namespace :cartodb do
+  namespace :services do
+
+    # Prefixes corresponding to User columns names/REDIS keys that store providers
+    DS_PROVIDED_SERVICES = ['geocoder', 'routing', 'isolines'].freeze
+
+    # service provider names
+    DS_PROVIDERS = ['heremaps', 'google', 'mapzen', 'mapbox', 'tomtom'].freeze
+
+    # Prefixes/infixes corresponding to User columns names/REDIS keys that store quotas / soft limits
+    # note that there's a couple of naming incosistence due to historical reasons:
+    # * here_isolines refers to isolines in general, not only those provided by here
+    # * mapzen_routing refers to routing in general
+    DS_SERVICES = ['geocoding', 'here_isolines', 'obs_snapshot', 'obs_general', 'mapzen_routing'].freeze
+
+    def assert_valid_arg(args, parameter, accepted_values: nil)
+      value = args[parameter]
+      if value.blank?
+        raise "Please specify the #{parameter}"
+      end
+      case accepted_values
+      when Array
+        if !accepted_values.include?(value)
+          values_sentence = accepted_values.to_sentence(last_word_connector: ' or ', two_words_connector: ' or ')
+          raise "Unknown #{parameter}: #{value.inspect}. Please use one of #{values_sentence}"
+        end
+      when Proc, Hash
+        if !accepted_values[value]
+          raise "Invalid #{parameter} value: #{value.inspect}."
+        end
+      end
+    end
+
+    # usage example:
+    #   bundle exec rake cartodb:services:set_user_provider['username','geocoder','mapzen']
+    desc 'Assign the provider for a service to a user'
+    task :set_user_provider, [:username, :service, :provider] => [:environment] do |_task, args|
+      username = args[:username]
+      service = args[:service]
+      provider = args[:provider]
+      user = username && ::User.find(username: username)
+
+      assert_valid_arg args, :username, accepted_values: proc { user.present? }
+      assert_valid_arg args, :service,  accepted_values: DS_PROVIDED_SERVICES
+      assert_valid_arg args, :provider, accepted_values: DS_PROVIDERS
+
+      service_key = "#{service}_provider="
+      user.send(service_key, provider)
+      user.save
+
+      puts "Changed the user service provider for #{service} to #{provider}."
+    end
+
+    # usage example:
+    #   bundle exec rake cartodb:services:set_organization_provider['orgname','geocoder','mapzen']
+    desc 'Assign the provider for a service to an organization'
+    task :set_organization_provider, [:orgname, :service, :provider] => [:environment] do |_task, args|
+      orgname = args[:orgname]
+      service = args[:service]
+      provider = args[:provider]
+      org = orgname && ::Organization.find(name: orgname)
+
+      assert_valid_arg args, :orgname,  accepted_values: proc { org.present? }
+      assert_valid_arg args, :service,  accepted_values: DS_PROVIDED_SERVICES
+      assert_valid_arg args, :provider, accepted_values: DS_PROVIDERS
+
+      service_key = "#{service}_provider="
+      org.send(service_key, provider)
+      org.save
+
+      puts "Changed the organization service provider for #{service} to #{provider}."
+    end
+
+
+    # usage example:
+    #   bundle exec rake cartodb:services:set_user_quota['username','geocoding',900]
+    desc 'Assign the quota for a service to a user'
+    task :set_user_quota, [:username, :service, :quota] => [:environment] do |_task, args|
+      username = args[:username]
+      service = args[:service]
+      quota = args[:quota]
+      user = username && ::User.find(username: username)
+
+      assert_valid_arg args, :username, accepted_values: proc { user.present? }
+      assert_valid_arg args, :service,  accepted_values: DS_SERVICES
+      assert_valid_arg args, :quota,    accepted_values: ->(value) { value.to_i >= 0 }
+
+      service_quota_key = "#{service}_quota="
+      user.send(service_quota_key, quota)
+      user.save
+
+      puts "Changed the user quota for service #{service} to #{quota}."
+    end
+
+    # usage example:
+    #   bundle exec rake cartodb:services:set_org_quota['orgname','geocoding',900]
+    desc 'Assign the quota for a service to an organization'
+    task :set_org_quota, [:orgname, :service, :quota] => [:environment] do |_task, args|
+      orgname = args[:orgname]
+      service = args[:service]
+      quota = args[:quota]
+      org = orgname && ::Organization.find(name: orgname)
+
+      assert_valid_arg args, :orgname, accepted_values: proc { org.present? }
+      assert_valid_arg args, :service, accepted_values: DS_SERVICES
+      assert_valid_arg args, :quota,   accepted_values: ->(value) { value.to_i >= 0 }
+
+      service_quota_key = "#{service}_quota="
+      org.send(service_quota_key, quota)
+      org.save
+
+      puts "Changed the organization quota for service #{service} to #{quota}."
+    end
+
+    # usage example: (valid values are true or false)
+    #   bundle exec rake cartodb:services:set_user_soft_limit['username','geocoding',true]
+    desc 'Assign the soft limit flag for a service to a user'
+    task :set_user_soft_limit, [:username, :service, :soft_limit] => [:environment] do |_task, args|
+      username = args[:username]
+      service = args[:service]
+      soft_limit = args[:soft_limit] == 'false' ? false : true
+      user = username && ::User.find(username: username)
+
+      assert_valid_arg args, :username,   accepted_values: proc { user.present? }
+      assert_valid_arg args, :service,    accepted_values: DS_SERVICES
+      assert_valid_arg args, :soft_limit, accepted_values: ['true', 'false']
+
+      service_quota_key = "soft_#{service}_limit="
+      user.send(service_quota_key, soft_limit)
+      user.save
+
+      puts "Changed the user soft limit for service #{service} to #{soft_limit}."
+    end
+  end
+end

lib/tasks/set_table_oids.rake

@@ -0,0 +1,76 @@
+namespace :cartodb do
+  namespace :db do
+    desc 'Sets table_ids for all table metadata recors'
+    task :set_table_oids => :environment do
+      count = ::User.count
+      ::User.all.each_with_index do |user, index|
+        puts "Setting table oids for #{user.username}"
+        begin
+          if %w(development staging).include?(ENV['RAILS_ENV'])
+            entries_in_database = Hash[
+              user.in_database.fetch(%Q(
+                SELECT table_name AS name, table_name::regclass::oid AS oid
+                FROM information_schema.tables
+                WHERE table_type  = 'BASE TABLE'
+                AND table_catalog = 'cartodb_#{ENV['RAILS_ENV']}_user_#{user.id}_db'
+                AND table_schema  = 'public';
+              )).all.map { |row| [row.fetch(:name), row.fetch(:oid)] }
+            ]
+          else
+            entries_in_database = Hash[
+              user.in_database.fetch(%Q(
+                SELECT table_name AS name, table_name::regclass::oid AS oid
+                FROM information_schema.tables
+                WHERE table_type  = 'BASE TABLE'
+                AND table_catalog = 'cartodb_user_#{user.id}_db'
+                AND table_schema  = 'public';
+              )).all.map { |row| [row.fetch(:name), row.fetch(:oid)] }
+            ]
+          end
+
+          raise 'No tables found!!!!' if entries_in_database.empty?
+          entries_in_metadata = Hash[
+            Table.fetch(%Q(
+              SELECT name, table_id
+              FROM user_tables
+              WHERE user_id = #{user.id}
+            )).map { |table| [table.name, table.table_id] }
+          ]
+          entries_in_metadata
+            .select { |name, oid| entries_in_database[name] != oid }
+            .each { |name_in_metadata, oid_in_metadata|
+              oid = entries_in_database[name_in_metadata]
+              next if oid.nil? || oid.to_s.empty?
+              puts "Setting table oid for #{name_in_metadata}"
+              Table.db.run(%Q(
+                UPDATE user_tables
+                SET table_id = #{oid}
+                WHERE name = '#{name_in_metadata}'
+                AND user_id = #{user.id}
+              ))
+            }
+          entries_in_database_by_oid = entries_in_database
+                                        .invert.delete_if { |k, v| k.nil? }
+          entries_in_metadata_by_oid = entries_in_metadata
+                                        .invert.delete_if { |k, v| k.nil? }
+          entries_in_metadata_by_oid
+            .select { |oid, name| entries_in_database_by_oid[oid] != name }
+            .each { |oid, name|
+              name_in_database = entries_in_database_by_oid[oid]
+              next unless name_in_database && name
+              puts "Renaming #{name} to #{name_in_database}"
+              table = Table.find_by_identifier(user.id, name)
+              table.name = name_in_database
+              table.save
+            }
+
+          printf "OK %-#{20}s (%-#{4}s/%-#{4}s)\n", user.username, index, count
+        rescue => exception
+          puts exception.backtrace.join("\n")
+          printf "FAIL %-#{20}s (%-#{4}s/%-#{4}s) #{exception.message}\n", user.username, index, count
+        end
+        sleep(1.0/5.0)
+      end
+    end
+  end
+end

lib/tasks/setup.rake

@@ -0,0 +1,204 @@
+namespace :cartodb do
+  namespace :db do
+    task :create, [:env] do |_t, args|
+      # When the db:create task runs, there is no prior database. Since we use models in our initializers, we need to
+      # skip the :environment dependency or Sequel will error out when trying to load the schema for the model table.
+      # This task overrides the default Sequel task, loading the bare minimum (db config) instead of all initializers
+      args.with_defaults(env: Rails.env)
+
+      config = SequelRails::Configuration.for(Rails.root, Carto::Conf.new.db_config)
+
+      unless SequelRails::Storage.create_environment(config.environment_for(args.env))
+        abort "Could not create database for #{args.env}."
+      end
+    end
+  end
+
+  namespace :test do
+    task :prepare do
+      if (ENV['RAILS_ENV'] == "test")
+        begin
+          Rake::Task['db:drop'].invoke
+        rescue Sequel::DatabaseConnectionError
+          puts "Not dropping DB (did not exist)"
+        end
+        Rake::Task['db:create'].invoke &&
+        Rake::Task['db:migrate'].invoke &&
+        Rake::Task['cartodb:db:create_publicuser'].invoke
+      else
+        system("rake cartodb:test:prepare RAILS_ENV=test") || raise("Something went wrong")
+      end
+    end
+  end
+  namespace :db do
+    desc <<-DESC
+Setup cartodb database and creates a new user from environment variables:
+  - ENV['EMAIL']: user e-mail
+  - ENV['PASSWORD']: user password
+  - ENV['SUBDOMAIN']: user subdomain
+DESC
+    task :setup => ["rake:db:create", "rake:db:migrate", "cartodb:db:create_publicuser"] do
+    end
+
+    task :setup_user => :environment do
+      raise "You should provide a valid e-mail"    if ENV['EMAIL'].blank?
+      raise "You should provide a valid password"  if ENV['PASSWORD'].blank?
+      raise "You should provide a valid subdomain" if ENV['SUBDOMAIN'].blank?
+
+      # Reload User model schema to avoid errors
+      # when running this task along with db:migrate
+      ::User.set_dataset :users
+
+      u = ::User.new
+      u.email = ENV['EMAIL']
+      u.password = ENV['PASSWORD']
+      u.password_confirmation = ENV['PASSWORD']
+      u.username = ENV['SUBDOMAIN']
+      u.database_host = ENV['DATABASE_HOST'] || ::SequelRails.configuration.environment_for(Rails.env)['host']
+
+      if ENV['BUILDER_ENABLED'] == "true"
+        u.builder_enabled = true
+      end
+
+      u.save
+
+      raise u.errors.inspect if u.new?
+      puts "User #{u.username} created successfully"
+    end
+
+    task :create_dev_user =>
+      ["cartodb:db:create_publicuser"] do
+      raise "You should provide a valid e-mail"    if ENV['EMAIL'].blank?
+      raise "You should provide a valid password"  if ENV['PASSWORD'].blank?
+      raise "You should provide a valid subdomain" if ENV['SUBDOMAIN'].blank?
+
+      # Reload User model schema to avoid errors
+      # when running this task along with db:migrate
+      ::User.set_dataset :users
+
+      user = ::User.new
+      user.email = ENV['EMAIL']
+      user.password = ENV['PASSWORD']
+      user.password_confirmation = ENV['PASSWORD']
+      user.username = ENV['SUBDOMAIN']
+      user.database_host = ENV['DATABASE_HOST'] || ::SequelRails.configuration.environment_for(Rails.env)['host']
+
+      if ENV['BUILDER_ENABLED'] == "true"
+        user.builder_enabled = true
+      end
+
+      unless user.valid?
+        puts
+        puts 'There are some problems with the info that you provided to create the new user:'
+        user.errors.full_messages.each do |error|
+          puts "\t> #{error}"
+        end
+        puts
+
+        exit
+      end
+
+      user.save
+      puts "User #{user.username} created successfully"
+
+      # 10 Gb of quota
+      quota = 1073741824
+      user.update(:quota_in_bytes => quota)
+
+      user.db_service.rebuild_quota_trigger
+      puts "User: #{user.username} quota updated to: 10 GB. #{user.tables.count} tables updated."
+
+      user.update(:table_quota => nil)
+      puts "User: #{user.username} table quota updated to: unlimited"
+
+      user.update(:private_tables_enabled => true)
+      puts "User: #{user.username} private tables enabled: true"
+
+      user.update(:account_type => '[DEDICATED]')
+      puts "User: #{user.username} table account type updated to: [DEDICATED]"
+
+      user.update(:sync_tables_enabled => true)
+      puts "User: #{user.username} sync tables enabled"
+
+      file_size_quota = 1500*1024*1024
+      row_count_quota = 5000000
+
+      user.update(:max_import_file_size => file_size_quota) if file_size_quota > user.max_import_file_size
+      puts "Twitter import user: #{user.username} max_import_file_size is now #{user.max_import_file_size}"
+
+      user.update(:max_import_table_row_count => row_count_quota) if row_count_quota > user.max_import_table_row_count
+      puts "Twitter import user: #{user.username} max_import_table_row_count is now #{user.max_import_table_row_count}"
+    end
+
+    desc "make public and tile users"
+    task :create_publicuser => :environment do
+      [CartoDB::PUBLIC_DB_USER, CartoDB::TILE_DB_USER].each do |u|
+        puts "Creating user #{u}"
+        ::SequelRails.connection.run("DO $$
+        BEGIN
+          IF NOT EXISTS ( SELECT * FROM pg_user WHERE usename = '#{u}') THEN
+            CREATE USER #{u}; -- TODO: with password '...'
+          ELSE
+            RAISE NOTICE 'User #{u} already exists';
+          END IF;
+          RETURN;
+        END;
+        $$ LANGUAGE 'plpgsql';")
+      end
+    end
+
+    # TODO: remove text bit and just use env
+    desc "Create a plain user account"
+    task :create_user => :environment do
+      begin
+        raise "You should provide a valid e-mail"    if ENV['EMAIL'].blank?
+        raise "You should provide a valid password"  if ENV['PASSWORD'].blank?
+        raise "You should provide a valid subdomain" if ENV['SUBDOMAIN'].blank?
+
+        u = ::User.new
+        u.email = ENV['EMAIL']
+        u.password = ENV['PASSWORD']
+        u.password_confirmation = ENV['PASSWORD']
+        u.username = ENV['SUBDOMAIN']
+        if ENV['DATABASE_HOST'].blank?
+          u.database_host = ::SequelRails.configuration.environment_for(Rails.env)['host']
+        else
+          u.database_host = ENV['DATABASE_HOST']
+        end
+
+        if ENV['BUILDER_ENABLED'] == "true"
+          u.builder_enabled = true
+        end
+
+        u.save
+        if u.new?
+          raise u.errors.inspect
+        end
+        puts "USER_ID #{u.id}"
+      rescue => e
+        puts e.inspect
+      end
+    end
+
+    desc "Set the password of the user in the USER_NAME environment variable to the value of the USER_PASSWORD environment variable"
+    task :change_user_password => :environment do
+      raise "Set USER_NAME environment variable" if ENV['USER_NAME'].blank?
+      raise "Set USER_PASSWORD environment variable" if ENV['USER_PASSWORD'].blank?
+      password = ENV['USER_PASSWORD']
+
+      users = ::User.filter(:username => ENV['USER_NAME']).all
+      if users.empty?
+        raise "User doesn't exist"
+      else
+        u = users.first
+        u.password = password
+        u.password_confirmation = password
+        if !u.save
+          rais u.errors.inspect
+        else
+          puts "Password changed"
+        end
+      end
+    end
+  end
+end

lib/tasks/sync_tables.rake

@@ -0,0 +1,68 @@
+namespace :cartodb do
+  # This rake retrieves all sync tables that should get synchronized, and puts the synchronization tasks at Resque
+  # NOTE: This version does not mark the tables as "enqueued", should be done if planning to run multiple instances
+  desc 'Runs the sync tables process'
+  task :sync_tables, [:force_all_arg] => [:environment] do |task, args|
+    puts '> Sync tables started' if ENV['VERBOSE']
+
+    require_relative '../../services/synchronizer/lib/synchronizer/collection'
+    collection = CartoDB::Synchronizer::Collection.new
+
+    # This fetches and enqueues
+    collection.fetch_and_enqueue(args[:force_all_arg].present? ? args[:force_all_arg] : false)
+
+    puts '> Sync tables finished' if ENV['VERBOSE']
+  end
+
+
+  desc 'Adds visualization_id to every Synchronization'
+  task :populate_synchronization_visualization_ids => [:environment] do |task, args|
+    require_relative '../../services/synchronizer/lib/synchronizer/collection'
+    collection = CartoDB::Synchronizer::Collection.new
+
+    collection.fetch_all.each { |record|
+      begin
+        synchronization = CartoDB::Synchronization::Member.new(id: record[:id]).fetch
+      rescue KeyError
+        synchronization = nil
+      end
+      if synchronization
+        begin
+          table = UserTable.where({
+              name: synchronization.name,
+              user_id: synchronization.user_id
+            }).first
+          if table.nil?
+            puts "\nSync id '#{record[:id]}' related table not found"
+          else
+            table = table.service
+          end
+        rescue => exception
+          table = nil
+          puts "\nSync id '#{record[:id]}' errored: #{exception.inspect}"
+        end
+        unless table.nil?
+          if synchronization.visualization_id.nil?
+            begin
+              synchronization.visualization_id = table.table_visualization.id
+            rescue => exception
+              puts "\nSync id '#{record[:id]}' errored, canonical visualization not found"
+            end
+            begin
+              synchronization.store
+              printf '.'
+            rescue => exception
+              puts "\nSync id '#{record[:id]}' errored: #{exception.inspect}"
+            end
+          else
+            printf 'S'
+          end
+        end
+      else
+        puts "\nSync id '#{record[:id]}' errored: missing synchronization entry"
+      end
+    }
+    puts "\nFINISHED"
+
+  end
+end

lib/tasks/table_maintenance.rake

@@ -0,0 +1,61 @@
+namespace :cartodb do
+  namespace :tables do
+    namespace :maintenance do
+      def ask_for_confirmation(key = 'continue')
+        puts "\nType '#{key}' to proceed or anything else to abort:"
+
+        raise 'user aborted' unless STDIN.gets.strip == key
+      end
+
+      desc "Sync user's UserTable(s)' table_id(s) with their corresponding physical table's oid"
+      task :sync_user_table_oids, [:username] => :environment do |_, args|
+        raise 'A username must be provided' unless args[:username].present?
+
+        user = ::User.where(username: args[:username]).first
+
+        raise "No user with username '#{args[:username]}' found" if user.nil?
+
+        user_tables = user.tables.all
+
+        unsynced_user_tables = user_tables.reject do |user_table|
+          user_table.table_id != user_table.service.fetch_table_id
+        end
+
+        user_tables_count = unsynced_user_tables.count
+        puts "#{user_tables_count} #{'table'.pluralize(user_tables_count)} will be processed, see a list below:"
+
+        unsynced_user_tables.each do |user_table|
+          puts "\t'#{user_table.name}'"
+        end
+
+        ask_for_confirmation('c')
+
+        synced_tables = 0
+        errored_tables = 0
+
+        unsynced_user_tables.each do |user_table|
+          printf "\tSynching '#{user_table.name}'... \r"
+
+          user_table.sync_table_id
+
+          if user_table.save
+            printf "\tSynching '#{user_table.name}'... ok\n"
+            synced_tables += 1
+          else
+            printf "\tSynching '#{user_table.name}'... ok\n"
+            errored_tables += 1
+
+            puts "ATENTION: errored save for table '#{user_table.name}':"
+            puts user_table.errors.full_messages
+
+            ask_for_confirmation
+          end
+        end
+
+        puts "\n#{user_tables_count} #{'table'.pluralize(user_tables_count)} processed"
+        puts "\t#{synced_tables} #{'table'.pluralize(synced_tables)} synced"
+        puts "\t#{errored_tables} #{'table'.pluralize(errored_tables)} errored"
+      end
+    end
+  end
+end

lib/tasks/tables.rake

@@ -0,0 +1,110 @@
+namespace :cartodb do
+  namespace :db do
+
+    desc "Add timezone to timestamp users columns"
+    task :add_timezones_to_timestamps => :environment do
+      users_length = ::User.all.length
+      cu = 1
+      task_errors = []
+      ::User.all.each do |u|
+        puts "#{u.username} (#{cu}/#{users_length})"
+        user_tables = u.real_tables.collect {|rt| rt[:relname]}
+        user_tables.each do |ut|
+          table_columns = u.in_database.schema(ut)
+          pending_columns = table_columns.select do |t|
+            t.first == :updated_at || t.first == :created_at
+          end
+          pending_columns = pending_columns.map do |pc|
+            pc.first.to_s if pc.last[:db_type] == "timestamp without time zone"
+          end.compact
+          if !pending_columns.empty?
+            alter_columns_sql = pending_columns.map {|c| "ALTER COLUMN #{c} TYPE timestamptz"}.join(', ')
+            alter_sql = "ALTER TABLE #{ut} #{alter_columns_sql};"
+            #puts alter_sql
+            begin
+              u.in_database.run(alter_sql)
+            rescue => e
+              puts "FAIL in #{alter_sql}"
+              puts "REASON: #{e.message}"
+              task_errors << {:user => u.username, :alter => alter_sql, :error => e.message}
+            end
+          end
+        end
+        cu = cu + 1
+      end
+      if task_errors.empty?
+        puts "All changes succeded"
+      else
+        puts "Some fails during the process"
+        task_errors.each do |value|
+          puts "User:  #{value[:user]}"
+          puts "Task:  #{value[:alter]}"
+          puts "Error: #{value[:error]}"
+          puts ""
+        end
+      end
+    end
+
+    desc "Unshare all entities for a given user"
+    task :unshare_all_entities, [:user] => :environment do |t, args|
+      # First unshare everything owned by the user itself
+      user = ::User.find(username: args[:user])
+      CartoDB::Permission.where(owner_id: user.id).each do |permission|
+        unless permission.acl.empty?
+          puts "Deleting permission: #{permission.acl}"
+          permission.acl = []
+          permission.save
+        end
+      end
+
+      # Then, remove permissions of everything he has access to
+      CartoDB::SharedEntity.where(recipient_id: user.id).each do |shared_entity|
+        permissions = CartoDB::Permission.where(entity_id: shared_entity.entity_id)
+        permissions.each do |permission|
+          acl = permission.acl
+          puts "Dropping permission from: #{permission.acl}"
+          acl.reject!{|acl_entry| acl_entry[:type] == 'user' && acl_entry[:id] == user.id}
+          # Hack to make ACLs be set correctly (input format for permission.acl= is different than permission.acl output)
+          acl.map!{|acl_entry| acl_entry[:entity] = {id: acl_entry[:id]}; acl_entry.delete(:id); acl_entry }
+          permission.acl = acl
+          permission.save
+        end
+      end
+    end
+
+    desc "Remove overview tables"
+    task :remove_overview_tables => :environment do
+      users_length = ::User.count
+      cu = 1
+      task_errors = []
+      ::User.paged_each do |u|
+        puts "#{u.username} (#{cu}/#{users_length})"
+        user_tables = u.real_tables.map { |rt| rt[:relname] }
+        user_tables.each do |ut|
+          next if ut.start_with?("_vovw_")
+          drop_overviews_sql = "select CDB_DropOverviews('#{ut}'::regclass)"
+          # puts drop_overviews_sql
+          begin
+            u.in_database.run(drop_overviews_sql)
+          rescue PG::Error => e
+            puts "FAIL in #{drop_overviews_sql}"
+            puts "REASON: #{e.message}"
+            task_errors << { :user => u.username, :drop => drop_overviews_sql, :error => e.message }
+          end
+        end
+        cu = cu + 1
+      end
+      if task_errors.empty?
+        puts "All changes succeded"
+      else
+        puts "Some fails during the process"
+        task_errors.each do |value|
+          puts "User:  #{value[:user]}"
+          puts "Task:  #{value[:drop]}"
+          puts "Error: #{value[:error]}"
+          puts ""
+        end
+      end
+    end
+  end
+end

lib/tasks/thread_pool.rb

@@ -0,0 +1,56 @@
+require 'monitor'
+
+# Basic thread pool with (unlimited) queue
+# Based on http://burgestrand.se/code/ruby-thread-pool/
+class ThreadPool
+
+  def initialize(size, sleep_time=0.1)
+    @size = size
+    @sleep_time = sleep_time
+    @workers_queue = []
+    @workers_queue.extend(MonitorMixin)
+    @mutex = @workers_queue.new_cond
+
+    @pool = Array.new(@size) do |i|
+      Thread.new do
+        Thread.current[:id] = i
+        catch(:exit) do
+          loop do
+            unless @workers_queue.empty?
+              @workers_queue.synchronize do
+                @mutex.wait_while { @workers_queue.empty? }
+              end
+              job, args = @workers_queue.pop
+              job.call(*args)
+              job = nil
+              args = nil
+            end
+          end
+        end
+      end
+    end
+  end
+
+  def schedule(*args, &block)
+    enqueued = false
+    begin
+      if @workers_queue.length < @size
+        @workers_queue.synchronize do
+          @workers_queue << [block, args]
+          enqueued = true
+          @mutex.broadcast
+        end
+      else
+        sleep(@sleep_time)
+      end
+    end while (!enqueued)
+  end
+
+  def shutdown
+    @size.times do
+      schedule { throw :exit }
+    end
+    @pool.map(&:join)
+  end
+
+end

lib/tasks/user.rake

@@ -0,0 +1,154 @@
+namespace :user do
+  namespace :deletion do
+    desc 'Delete a user given a username'
+    task :by_username, [:username] => [:environment] do |task, args|
+      raise 'Please specify the username of the user to be deleted' if args[:username].blank?
+
+      user = ::User.find(username: args[:username])
+      raise "The username '#{args[:username]}' does not correspond to any user" if user.nil?
+
+      raise 'Deletion aborted due to bad confirmation' if !deletion_confirmed?(user)
+
+      user.destroy
+    end
+
+    desc 'Delete a user given an email'
+    task :by_email, [:email] => [:environment] do |task, args|
+      raise 'Please specify the email of the user to be deleted' if args[:email].blank?
+
+      user = ::User.find(email: args[:email])
+      raise "The email '#{args[:email]}' does not correspond to any user" if user.nil?
+
+      raise 'Deletion aborted due to bad confirmation' if !deletion_confirmed?(user)
+
+      user.destroy
+    end
+
+    def deletion_confirmed?(user)
+      puts ""
+      puts "You are about to delete the following user:"
+      puts "\t> username: #{user.username}"
+      puts "\t> email: #{user.email}"
+      puts ""
+      puts "Alongside '#{user.username}', the following will be deleted:"
+      puts "\t> #{user.maps.length} maps"
+      puts "\t> #{user.tables.count} datasets"
+      puts ""
+      puts "Type in the user's username (#{user.username}) to confirm:"
+
+      confirm = STDIN.gets.strip
+
+      confirm == user.username
+    end
+  end
+
+  namespace :change do
+    desc 'Change the number of maximum layers allowed on a map'
+    task :max_layers, [:username, :max_layers] => [:environment] do |task, args|
+      max_layers = args[:max_layers].to_i
+
+      raise 'Please specify the username of the user to be modified' if args[:username].blank?
+      raise 'Please specify a number of layers that is a positive integer' if max_layers < 1
+
+      user = ::User.find(username: args[:username])
+      raise "The username '#{args[:username]}' does not correspond to any user" if user.nil?
+
+      old_max_layers = user.max_layers
+
+      user.max_layers = max_layers
+      user.save
+
+      puts "Changed the number of max layers for '#{user.username}' from #{old_max_layers} to #{max_layers}."
+    end
+  end
+
+  namespace :likes do
+    desc 'Clean likes for all the users of the organization'
+    task :clean_org_likes, [:organization] => [:environment] do |_, args|
+      raise 'Please specify the organization to be cleaned' if args[:organization].blank?
+      org = Carto::Organization.find_by_name(args[:organization])
+      raise "The organization '#{args[:organization]}' does not correspond to any organization" if org.nil?
+      org.users.find_each do |u|
+        Carto::Like.where(actor: u.id).delete_all
+      end
+    end
+    desc 'Clean likes for a user'
+    task :clean_user_likes, [:username] => [:environment] do |_, args|
+      raise 'Please specify the username to be cleaned' if args[:username].blank?
+      u = Carto::User.find_by_username(args[:username])
+      raise "The username '#{args[:username]}' does not correspond to any user" if u.nil?
+      Carto::Like.where(actor: u.id).delete_all
+    end
+  end
+
+  namespace :ghost_tables do
+    desc 'Install ghost tables trigger to all the organizations owners and regular users'
+    task :install_ghost_tables_trigger => :environment do
+      Carto::User.where(organization_id: nil).find_each do |user|
+        install_ghost_tables_trigger(user.id)
+      end
+
+      Organization.each { |org| install_ghost_tables_trigger(org.owner_id) }
+    end
+
+    def install_ghost_tables_trigger(user_id)
+      user = ::User.find(id: user_id)
+      if user.nil?
+        puts "ERROR: User #{user_id} does not exist"
+      elsif user.organization_user? && !user.organization_owner?
+        puts "ERROR: User #{user_id} must be an org owner or not to be an organization user"
+      elsif user.has_feature_flag?('ghost_tables_trigger_disabled')
+        puts "WARN: Skipping user #{user_id} (it has 'ghost_tables_trigger_disabled' feature flag)"
+      else
+        begin
+          user.db_service.create_ghost_tables_event_trigger
+        rescue StandardError => error
+          puts "ERROR creating ghost tables trigger for user #{user_id}: #{error.message}"
+        end
+      end
+    end
+
+    desc 'Drop ghost_tables_trigger to all the organizations owners and regular users'
+    task :drop_ghost_tables_trigger => :environment do
+      Carto::User.where(organization_id: nil).find_each do |user|
+        drop_ghost_tables_trigger(user.id)
+      end
+
+      Organization.each { |org| drop_ghost_tables_trigger(org.owner_id) }
+    end
+
+    def drop_ghost_tables_trigger(user_id)
+      user = ::User.find(id: user_id)
+      if user.nil?
+        puts "ERROR: User #{username} does not exist"
+      elsif user.organization_user? && !user.organization_owner?
+        puts "ERROR: User #{username} must be an org owner or not to be an organization user"
+      else
+        user.db_service.drop_ghost_tables_event_trigger
+      end
+    end
+  end
+
+  namespace :notifications do
+    desc 'Add a text in the notification field for users filtered by field'
+    task :add_by_field, [:filter_field, :filter_value, :notification] => [:environment] do |task, args|
+      allowed_fields = ['database_host']
+      raise "Filter field and value are needed" if args[:filter_field].nil? || args[:filter_value].nil?
+      raise "Unknown field #{args[:filter_field]} for filtering. Allowed fields are #{allowed_fields.join(',')}" unless allowed_fields.include?(args[:filter_field])
+      raise "Notification not provided. Please include it" if args[:notification].nil?
+      sql = "UPDATE users SET notification = '%s' WHERE %s = '%s'"
+      query = ActiveRecord::Base.send(:sanitize_sql_array, [sql, args[:notification], args[:filter_field], args[:filter_value]])
+      ActiveRecord::Base.connection.execute(query)
+    end
+
+    desc 'Clean notification for users filtered by field'
+    task :clean_by_field, [:filter_field, :filter_value] => [:environment] do |task, args|
+      allowed_fields = ['database_host']
+      raise "Filter field and value are needed" if args[:filter_field].nil? || args[:filter_value].nil?
+      raise "Unknown field #{args[:filter_field]} for filtering. Allowed fields are #{allowed_fields.join(',')}" unless allowed_fields.include?(args[:filter_field])
+      sql = "UPDATE users SET notification = NULL WHERE %s = '%s'"
+      query = ActiveRecord::Base.send(:sanitize_sql_array, [sql, args[:filter_field], args[:filter_value]])
+      ActiveRecord::Base.connection.execute(query)
+    end
+  end
+end

lib/tasks/user_database_host_update.rake

@@ -0,0 +1,35 @@
+namespace :cartodb do
+  namespace :database_host do
+    desc 'Change database host for users and update metadata'
+    task :update_dbm_and_redis, [:origin_ip, :dest_ip] => [:environment] do |_, args|
+      raise 'Origin IP parameter is mandatory' unless args[:origin_ip].present?
+      raise 'Destination IP parameter is mandatory' unless args[:dest_ip].present?
+      affected_users = ::User.where(database_host: args[:origin_ip]).count
+
+      # think about message
+      reflection_seconds = ENV['ENABLE_FF_REFLECTION_SECONDS'] || 10
+
+      puts "############################"
+      puts "#"
+      puts "# The database_host of #{affected_users} users will be updated"
+      puts "#"
+      puts "# Origin IP: #{args[:origin_ip]}"
+      puts "# Destination IP: #{args[:dest_ip]}"
+      puts "#"
+      puts "# You have #{reflection_seconds} seconds to cancel "
+      puts "# the task before it starts"
+      puts "#"
+      puts "############################"
+
+      sleep reflection_seconds.to_i
+
+      # update dbm without ORM
+      dbm_query = "UPDATE users SET database_host='%s' WHERE database_host='%s'"
+      query = ActiveRecord::Base.send(:sanitize_sql_array, [dbm_query, args[:dest_ip], args[:origin_ip]])
+      ActiveRecord::Base.connection.execute(query)
+
+      # update Redis
+      ::User.where(database_host: args[:dest_ip]).order(:id).paged_each(&:save_metadata)
+    end
+  end
+end

lib/tasks/user_migrator.rake

@@ -0,0 +1,197 @@
+require_relative '../../services/user-mover/export_user.rb'
+require_relative '../../services/user-mover/import_user.rb'
+require_relative '../../app/services/carto/organization_metadata_export_service'
+require_relative '../../app/services/carto/redis_export_service'
+
+namespace :cartodb do
+  namespace :user_migrator do
+    namespace :export do
+      def run_user_migrator_export_job(ume)
+        if ume.organization
+          puts "Exporting organization #{ume.organization.name}"
+        else
+          puts "Exporting user #{ume.user.username}"
+        end
+        if ume.export_metadata
+          puts "Including metadata (moving between clouds)"
+        else
+          puts "Without metadata (moving inside a single cloud)"
+        end
+        puts 'Control + C to cancel (will continue in 5 seconds)'
+        sleep 5
+
+        ume.save!
+        puts "Running export with id = #{ume.id}"
+        ume.run_export
+        puts "Export finished with status #{ume.state}"
+        if ume.state == Carto::UserMigrationExport::STATE_COMPLETE
+          import_params = {
+            org_import: ume.organization.present?,
+            import_metadata: ume.export_metadata,
+            import_data: ume.export_data?,
+            exported_file: ume.exported_file,
+            json_file: ume.json_file,
+            dry: false,
+            database_host: '<destination db host>'
+          }
+          import_params[:user_id] = ume.user_id if !ume.export_metadata && ume.user_id
+          import_params[:organization_id] = ume.organization_id if !ume.export_metadata && ume.organization_id
+          config_filename = "/tmp/import_for_#{ume.id}.json"
+          puts "Pass the following parameters to the import task"
+          puts JSON.pretty_generate(import_params)
+          File.write(config_filename, JSON.pretty_generate(import_params))
+          puts "Written to #{config_filename}"
+          puts "Don't forget to update the database host"
+        else
+          puts ume.log.entries
+          raise 'Failed Export!'
+        end
+      end
+
+      desc 'Export an organization'
+      task :organization, [:orgname, :metadata, :metadata_only] => :environment do |_task, args|
+        organization = Carto::Organization.find_by_name(args[:orgname])
+        raise 'Organization not found' unless organization
+        export_metadata = args[:metadata] == 'true'
+        metadata_only = args[:metadata_only] == 'true'
+
+        ume = Carto::UserMigrationExport.new(
+          organization: organization,
+          export_metadata: export_metadata,
+          export_data: !metadata_only
+        )
+        run_user_migrator_export_job(ume)
+      end
+
+      desc 'Export a user'
+      task :user, [:username, :metadata, :metadata_only] => :environment do |_task, args|
+        user = Carto::User.find_by_username(args[:username])
+        raise 'User not found' unless user
+        export_metadata = args[:metadata] == 'true'
+        metadata_only = args[:metadata_only] == 'true'
+
+        ume = Carto::UserMigrationExport.new(
+          user: user,
+          export_metadata: export_metadata,
+          export_data: !metadata_only
+        )
+        run_user_migrator_export_job(ume)
+      end
+    end
+
+    desc 'Import a user migrator dump'
+    task :import, [:config_file] => :environment do |_task, args|
+      config = JSON.parse(File.read(args[:config_file])).symbolize_keys
+      raise 'Set a database host' if config[:database_host].blank? || config[:database_host].include?('<')
+
+      umi = Carto::UserMigrationImport.new(config)
+      umi.save!
+      puts "Running import with id = #{umi.id}"
+      umi.run_import
+      puts "Import finished with status #{umi.state}"
+      puts umi.log.entries unless umi.state == Carto::UserMigrationImport::STATE_COMPLETE
+    end
+
+    namespace :cleanup do
+      module OrganizationMigrationCleanup
+        def clean_user_metadata(user)
+          return unless user.present? && user.persisted?
+          carto_user = Carto::User.find(user.id)
+          carto_user.assets.each(&:delete)
+          user.destroy
+        end
+
+        def clean_organization_metadata(organization)
+          return unless organization.persisted?
+          organization.assets.each(&:delete)
+          organization.users.each { |u| u.assets.each(&:delete) }
+          organization.destroy_cascade
+        end
+
+        def clean_redis_user(user)
+          remove_keys(
+            $users_metadata,
+            $users_metadata.keys("user:#{user.username}*")
+          )
+          remove_keys(
+            $tables_metadata,
+            ["map_tpl|#{user.username}"]
+          )
+        end
+
+        def clean_redis_organization(organization)
+          remove_keys(
+            $users_metadata,
+            $users_metadata.keys("org:#{organization.name}*")
+          )
+          organization.users.each { |u| clean_redis_user(u) }
+        end
+
+        def clean_organization(organization)
+          puts "Cleaning organization #{organization.name}"
+          clean_redis_organization(organization)
+          clean_organization_metadata(organization)
+        end
+
+        def clean_user(user)
+          puts "Cleaning user #{user.username}"
+          clean_redis_user(user)
+          clean_user_metadata(user)
+        end
+      end
+
+      desc 'Cleans all organizations data and metadata matching filter in config file'
+      task :organizations_from_file, [:config_file] => :environment do |_, args|
+        include CartoDB::DataMover::Utils
+        include OrganizationMigrationCleanup
+        include ::Carto::RedisExportServiceImporter
+        Organization.where(File.read(args[:config_file]).to_s).each { |org| clean_organization(org) }
+      end
+
+      desc 'Cleans all user data and metadata matching filter in config file'
+      task :users_from_file, [:config_file] => :environment do |_, args|
+        include CartoDB::DataMover::Utils
+        include OrganizationMigrationCleanup
+        include ::Carto::RedisExportServiceImporter
+        User.where(File.read(args[:config_file]).to_s).each { |u| clean_user(u) }
+      end
+
+      desc 'Cleans all data for organization with given name'
+      task :organization, [:name] => :environment do |_, args|
+        include CartoDB::DataMover::Utils
+        include OrganizationMigrationCleanup
+        include ::Carto::RedisExportServiceImporter
+        organization = Organization.where("name = '#{args[:name]}'").first || Organization.new(name: args[:orgname])
+        clean_organization(organization)
+      end
+
+      desc 'Cleans all data for user with given username'
+      task :user, [:username] => :environment do |_, args|
+        include CartoDB::DataMover::Utils
+        include OrganizationMigrationCleanup
+        include ::Carto::RedisExportServiceImporter
+        user = User.where("username = '#{args[:username]}'").first || User.new(username: args[:username])
+        clean_user(user)
+      end
+
+
+      desc 'Cleans redis keys for org with given name'
+      task :redis_for_orgname, [:orgname] => :environment do |_, args|
+        include CartoDB::DataMover::Utils
+        include OrganizationMigrationCleanup
+        include ::Carto::RedisExportServiceImporter
+        organization = Organization.where("name = '#{args[:orgname]}'").first || Organization.new(name: args[:orgname])
+        clean_redis_organization(organization)
+      end
+
+      desc 'Cleans all redis keys for given username'
+      task :redis_for_username, [:username] => :environment do |_, args|
+        include CartoDB::DataMover::Utils
+        include OrganizationMigrationCleanup
+        include ::Carto::RedisExportServiceImporter
+        user = User.where("username = '#{args[:username]}'").first || User.new(username: args[:username])
+        clean_redis_user(user)
+      end
+    end
+  end
+end

lib/tasks/user_mover.rake

@@ -0,0 +1,60 @@
+require_relative '../../services/user-mover/export_user'
+require_relative '../../services/user-mover/import_user'
+
+namespace :cartodb do
+  namespace :user_mover do
+    namespace :export do
+      desc 'Export an user'
+      task :export_user, [:id, :path, :job_uuid, :schema_mode, :database_only, :metadata_only] => :environment do |_task, args|
+        args.with_defaults(job_uuid: nil, schema_mode: false, database_only: false, metadata_only: false)
+        job_args = args.to_hash
+        if job_args[:database_only] == 'true'
+          job_args[:data] = true
+          job_args[:metadata] = false
+        elsif job_args[:metadata_only] == 'true'
+          job_args[:data] = false
+          job_args[:metadata] = true
+        end
+        CartoDB::DataMover::ExportJob.new(job_args)
+      end
+      desc 'Export an organization'
+      task :export_org, [:organization_name, :path, :job_uuid, :database_only, :metadata_only, :split_user_schemas] => :environment do |_task, args|
+        args.with_defaults(job_uuid: nil, database_only: false, metadata_only: false)
+        job_args = args.to_hash
+        if job_args[:database_only] == 'true'
+          job_args[:data] = true
+          job_args[:metadata] = false
+        elsif job_args[:metadata_only] == 'true'
+          job_args[:data] = false
+          job_args[:metadata] = true
+        end
+        job_args[:split_user_schemas] = job_args[:split_user_schemas] == 'true' ? true : false
+        CartoDB::DataMover::ExportJob.new(job_args)
+      end
+    end
+
+    namespace :import do
+      desc 'Import a JSON export (either username or organization)'
+      task :import, [:file, :data_only, :into_org_name, :host, :job_uuid] => :environment do |_task, args|
+        args.with_defaults(job_uuid: nil, data_only: false, host: nil, rollback: false, into_org_name: nil, mode: :import)
+        job_args = args.to_hash
+        if job_args[:data_only] == 'true'
+          job_args[:data] = true
+          job_args[:metadata] = false
+        end
+        job_args[:into_org_name] = (job_args[:into_org] != '' ? args[:into_org] : nil)
+        CartoDB::DataMover::ImportJob.new(job_args).run!
+      end
+      desc 'Rollback from a JSON export (either username or organization)'
+      task :rollback, [:file, :data_only, :host] => :environment do |_task, args|
+        args.with_defaults(data_only: false, host: nil, rollback: false, mode: :rollback)
+        job_args = args.to_hash
+        if job_args[:data_only] == 'true'
+          job_args[:data] = true
+          job_args[:metadata] = false
+        end
+        CartoDB::DataMover::ImportJob.new(job_args).run!
+      end
+    end
+  end
+end

lib/tasks/user_overquota.rake

@@ -0,0 +1,14 @@
+namespace :cartodb do
+  namespace :user_overquota do
+    # e.g. bundle exec rake cartodb:user_overquota:calculate
+    #      bundle exec rake cartodb:user_overquota:calculate[0.20]
+    desc 'Calculate and store the daily users with overquota greater than some delta'
+    task :calculate, [:delta] => :environment do |_task, args|
+      args.with_defaults(delta: 0.20)
+      delta = args[:delta].to_f
+      puts 'Executing overquota calculation'
+      Carto::OverquotaUsersService.new.store_overquota_users(delta)
+      puts "Ended getting the overquota users for delta #{delta}"
+    end
+  end
+end

lib/tasks/user_tables_bounding_box.rake

@@ -0,0 +1,23 @@
+namespace :cartodb do
+  namespace :tables do
+    desc 'Set the bounding box for the canonical tables of the user'
+    task :set_bounding_box, [:user_id] => :environment do |t, args|
+      user_id = args[:user_id]
+      if user_id.nil?
+        raise Exception.new("User id is a mandatory parameter")
+      end
+      puts "Setting bounding boxes for canonical of the user #{user_id}..."
+      user = Carto::User.find(user_id)
+      table_names = SequelRails.connection.fetch(
+        %Q{SELECT name FROM user_tables WHERE user_tables.user_id = ?}, user_id
+      )
+      updated = 0
+      table_names.each do |table_name|
+        table = Helpers::TableLocator.new.get_by_id_or_name(table_name[:name], user)
+        table.update_bounding_box
+        updated += 1
+      end
+      puts "Updated #{updated} tables"
+    end
+  end
+end # namespace :cartodb

lib/tasks/viz_maintenance.rake

@@ -0,0 +1,209 @@
+require_relative "../../app/model_factories/layer_factory"
+require_relative "../../app/model_factories/map_factory"
+require 'carto/mapcapped_visualization_updater'
+
+namespace :cartodb do
+  namespace :vizs do
+
+    desc "Purges broken visualizations due to bug during deletion."
+    task :delete_inconsistent, [:username] => :environment do |t, args|
+      username = args[:username]
+      raise "You should pass a username param" unless username
+      user = ::User[username: username]
+      collection = Carto::Visualization.where(user_id: user.id)
+
+      collection.each do |viz|
+        if inconsistent?(viz)
+          delete_with_confirmation(viz)
+        end
+      end
+    end
+
+    desc "Purges broken canonical visualizations without related tables"
+    task :delete_inconsistent_canonical_viz_without_tables => :environment do |_|
+      Carto::Visualization.joins("left join user_tables ut on visualizations.map_id = ut.map_id").where("visualizations.type = 'table' and ut.id is null").find_each do |viz|
+        begin
+          puts "Checking for deletion --> User: #{viz.user.username} | Viz id: #{viz.id}"
+          if inconsistent_table?(viz)
+            puts "Deleting viz --> User: #{viz.user.username} | Viz id: #{viz.id}"
+            viz.destroy!
+          end
+        rescue => e
+          puts "Error deleting viz #{viz.id}: #{e}"
+        end
+      end
+    end
+
+    desc "Create named maps for all eligible existing visualizations"
+    task :create_named_maps, [:order] => :environment do |t, args|
+      sort_order = args[:order] == ':desc' ? :desc : :asc
+      puts "Retrieving by :created_at #{sort_order}"
+
+      puts "> #{Time.now}"
+
+      vqb = Carto::VisualizationQueryBuilder.new
+                                            .with_types([
+                                                Carto::Visualization::TYPE_CANONICAL,
+                                                Carto::Visualization::TYPE_DERIVED
+                                              ])
+                                            .with_order(:created_at, sort_order)
+                                            .build
+
+      count = vqb.count
+      current = 0
+
+      puts "Fetched ##{count} items"
+      puts "> #{Time.now}"
+
+      vqb.each do |vis|
+        begin
+          current += 1
+
+          Carto::NamedMaps::Api.new(vis).upsert
+          if current % 50 == 0
+            print '.'
+          end
+          if current % 500 == 0
+            puts "\n> #{Time.now} #{current}/#{count}"
+          end
+          vis = nil
+        rescue => ex
+          printf "E"
+        end
+      end
+
+      puts "\n> #{Time.now}\nFinished ##{count} items"
+    end
+
+    desc "Exports a .carto file including visualization metadata and the tables"
+    task :export_full_visualization, [:vis_id] => :environment do |_, args|
+      visualization_id = args[:vis_id]
+      raise "Missing visualization id argument" unless visualization_id
+
+      visualization = Carto::Visualization.where(id: visualization_id).first
+      raise "Visualization not found" unless visualization
+
+      file = Carto::VisualizationExport.new.export(visualization, visualization.user)
+      puts "Visualization exported: #{file}"
+    end
+
+    desc "Updates visualizations auth tokens from named maps"
+    task update_auth_tokens: :environment do |_|
+      Carto::Visualization.find_each(conditions: "privacy = 'password'") do |visualization|
+        puts "Updating #{visualization.id}"
+        begin
+          tokens = visualization.get_auth_tokens
+          puts "  from #{visualization.auth_token} to #{tokens.first}"
+          visualization.update_column(:auth_token, tokens.first)
+        rescue => e
+          puts "ERROR #{e.inspect}"
+        end
+      end
+    end
+
+    desc "Have all Builder visualizations mapcapped. Dry mode: `bundle exec rake cartodb:vizs:mapcap_builder_visualizations['--dry']`"
+    task :mapcap_builder_visualizations, [:dry] => :environment do |_, args|
+      dry = args[:dry] == '--dry'
+
+      puts "Mapcapping v3 visualizations. Dry mode #{dry ? 'on' : 'off'}"
+
+      Carto::Visualization.find_each(conditions: "version = 3 and type = 'derived' and privacy != 'private'") do |visualization|
+        begin
+          if !visualization.mapcapped?
+            puts "Mapcapping #{visualization.id}"
+            Carto::Mapcap.create!(visualization_id: visualization.id) unless dry
+          end
+        rescue => e
+          puts "ERROR mapcapping #{visualization}: #{e.inspect}"
+        end
+      end
+    end
+
+    desc "Create analyses for all v3 visualizations"
+    task :create_analyses_for_v3_visualizations, [:dry] => :environment do |_, args|
+      include Carto::MapcappedVisualizationUpdater
+      dry = args[:dry] == '--dry'
+
+      puts "Adding analyses to v3 visualizations. Dry mode #{dry ? 'on' : 'off'}"
+
+      puts "=== STEP 1/2: Visualizations ==="
+      v3_no_analyses = Carto::Visualization.joins('LEFT JOIN analyses ON visualizations.id = analyses.visualization_id')
+                                           .where(version: 3, type: 'derived', analyses: { id: nil })
+
+      v3_no_analyses.find_each do |visualization|
+        begin
+          puts "Adding analyses to visualization #{visualization.id}"
+          visualization.add_source_analyses unless dry
+        rescue => e
+          puts "ERROR adding analyses to visualization #{visualization.id}: #{e.inspect}"
+        end
+      end
+
+      puts "=== STEP 2/2: Mapcaps ==="
+      mapcap_no_analyses = Carto::Mapcap.where("json_array_length(export_json->'visualization'->'analyses') = 0")
+
+      mapcap_no_analyses.find_each do |mapcap|
+        puts "Adding analyses to mapcap #{mapcap.id}"
+        unless dry
+          begin
+            rv = mapcap.regenerate_visualization
+
+            rv.data_layers.each_with_index do |layer, index|
+              analysis = Carto::Analysis.source_analysis_for_layer(layer, index)
+              rv.analyses << analysis
+              layer.options[:source] = analysis.natural_id
+              layer.options[:letter] = analysis.natural_id.first
+            end
+
+            mapcap.export_json = export_in_memory_visualization(rv, rv.user)
+            mapcap.save
+          rescue => e
+            puts "ERROR adding analyses to mapcap: #{mapcap.id}: #{e.inspect}"
+          end
+        end
+      end
+    end
+
+    desc "Restore visualization from backup"
+    task :restore_visualization, [:backup_id] => :environment do |_, args|
+      include Carto::VisualizationBackupService
+
+      backup_id = args[:backup_id]
+      visualization = restore_visualization_backup(backup_id)
+      puts "Error restoring visualization" if visualization.nil?
+      puts "Visualization #{visualization.id} restored" unless visualization.nil?
+    end
+
+    private
+
+    def inconsistent?(viz)
+      (viz.table? && viz.related_tables.empty?) || (viz.derived? && viz.map.nil?)
+    end
+
+    def inconsistent_table?(viz)
+      (viz.user_table.nil? && viz.related_tables.empty?)
+    end
+
+    def delete_with_confirmation(viz)
+      display_info(viz)
+      if ok_to_delete?
+        viz.delete
+        STDOUT.puts "deleted!"
+      end
+    end
+
+    def display_info(viz)
+      STDOUT.puts "\nviz.name = #{viz.name}"
+      STDOUT.puts "viz.type = #{viz.type}"
+      STDOUT.puts "viz.related_tables = #{viz.related_tables.map {|t| t.name}}"
+      STDOUT.puts "viz.map_id = #{viz.map_id}"
+    end
+
+    def ok_to_delete?
+      STDOUT.puts "About to delete. Are you sure? (y/n)"
+      input = STDIN.gets.strip
+      return input == 'y'
+    end
+
+  end
+end

lib/tasks/widgets.rake

@@ -0,0 +1,28 @@
+namespace :cartodb do
+  namespace :widgets do
+    desc 'fills source_id in old widgets'
+    task fill_source_id: :environment do |_t, _args|
+      Carto::Widget.where(source_id: nil).find_each do |widget|
+        layer = widget.layer
+        layer.reload
+        source = layer.options[:source]
+        if source
+          # Set to same source as layer
+          puts "Updating Widget #{widget.id} from #{widget.source_id} to #{source}"
+          widget.update_attribute(:source_id, source)
+        elsif layer.visualization
+          # Create source analyses and set as widget source
+          puts "Creating source analyses for visualization #{widget.visualization.id}"
+          widget.visualization.add_source_analyses
+          layer.reload
+          puts "Updating Widget #{widget.id} from #{widget.source_id} to #{source}"
+          widget.update_attribute(:source_id, layer.options[:source])
+        else
+          # Orphan layer
+          puts "Destroying Widget #{widget.attributes}"
+          widget.destroy
+        end
+      end
+    end
+  end
+end