Initial commit

lib/tasks/import.rake

@@ -0,0 +1,93 @@
+require 'fileutils'
+require_relative '../../app/helpers/file_upload'
+
+namespace :cartodb do
+  desc 'Import a file to CartoDB. "times" parameter is there for load test purposes. Defaults to 1.'
+  task :import, [:username, :filepath, :times] => [:environment] do |_task, args|
+    times = (args[:times] || 1).to_i
+    user        = ::User.where(username: args[:username]).first
+    filepath    = File.expand_path(args[:filepath])
+
+    data_import = DataImport.create(
+      :user_id       => user.id,
+      :data_source   => filepath,
+      :updated_at    => Time.now,
+      :append        => false
+    )
+    data_import.values[:data_source] = filepath
+
+    (1..times).each do
+      data_import.run_import!
+      puts data_import.log
+    end
+  end
+
+
+  desc 'Uploads a single data import enqueued file to S3, triggering the normal flow afterwards'
+  task upload_to_s3: [:environment] do
+    data_import_item = DataImport.where(state: DataImport::STATE_ENQUEUED)
+                                 .where(upload_host: Socket.gethostname)
+                                 .order(:created_at)
+                                 .first
+
+    file_upload_helper = CartoDB::FileUpload.new(Cartodb.config[:importer].fetch("uploads_path", nil))
+
+    unless data_import_item.nil?
+      # be 100% safe in rescue blocks when trying to log the failed id
+      data_import_item_id = data_import_item.id rescue nil
+
+      begin
+        puts "Retrieved #{data_import_item.id}"
+
+        data_import_item.state = DataImport::STATE_PENDING
+        data_import_item.save
+
+        filepath = File.realpath data_import_item.data_source
+        filename = File.basename data_import_item.data_source
+
+        uploads_path = File.realpath file_upload_helper.get_uploads_path
+
+        # Files are temp stored in "/%{uploads_path}/token/basename.ext"
+        token = File.basename File.dirname filepath
+
+        file_uri = file_upload_helper.upload_file_to_s3(filepath, filename, token, Cartodb.config[:importer]['s3'])
+        begin
+          File.delete(filepath)
+          folder = filepath.slice(0, filepath.rindex('/')).gsub('..', '')
+          FileUtils.rm_rf(folder) unless folder.length < uploads_path.to_s.length
+        rescue => exception
+          puts "Errored #{data_import_item_id} : #{exception}"
+          CartoDB::notify_error(
+            exception,
+            task: 'cartodb:upload_to_s3 Rake',
+            import_id: data_import_item_id,
+            item: data_import_item.inspect
+          )
+        end
+
+        data_import_item.data_source = file_uri
+        data_import_item.save
+
+        Resque.enqueue(Resque::ImporterJobs, job_id: data_import_item.id)
+
+        puts "Uploaded #{data_import_item.id}"
+      rescue => exception
+        puts "Errored #{data_import_item_id} : #{exception}"
+
+        CartoDB::notify_error(
+          exception,
+          task: 'cartodb:upload_to_s3 Rake',
+          import_id: data_import_item_id,
+          item: data_import_item.inspect
+        )
+
+        data_import_item.state = DataImport::STATE_FAILURE
+        data_import_item.save
+
+        File.delete(filepath)
+        folder = filepath.slice(0, filepath.rindex('/')).gsub('..', '')
+        FileUtils.rm_rf(folder) unless folder.length < file_upload_helper.get_uploads_path.to_s.length
+      end
+    end
+  end
+end