Refactor and add importer to Installer module #443 (#444)

* Refactor and add importer to Installer module #443

* Refactor for finances to use in import

* Import groups into roles

* Formatting

* Formatting

* Add interface in installer for import

* Notes about importing

* Check for installer folder

* Formatting

* Fix pirep->user mapping

* Unused import

* Formatting

modules/.gitignore

@@ -4,6 +4,7 @@
 /*/
 !.gitignore
 !/Awards
+!/Importer
 !/Installer
 !/Sample
 !/Vacentral

modules/Awards/Providers/AwardServiceProvider.php

@@ -0,0 +1,9 @@
+<?php
+
+namespace Modules\Awards\Providers;
+
+use Illuminate\Support\ServiceProvider;
+
+class AwardServiceProvider extends ServiceProvider
+{
+}

modules/Awards/module.json

@@ -5,7 +5,9 @@
     "keywords": [],
     "active": 1,
     "order": 0,
-    "providers": [],
+    "providers": [
+        "Modules\\Awards\\Providers\\AwardServiceProvider"
+    ],
     "aliases": {},
     "files": [],
     "requires": []

modules/Installer/Config/config.php

@@ -1,5 +1,12 @@
 <?php
 
+use Modules\Installer\Services\Importer\Stages\Stage1;
+use Modules\Installer\Services\Importer\Stages\Stage2;
+use Modules\Installer\Services\Importer\Stages\Stage3;
+use Modules\Installer\Services\Importer\Stages\Stage4;
+use Modules\Installer\Services\Importer\Stages\Stage5;
+use Modules\Installer\Services\Importer\Stages\Stage6;
+
 return [
     'php' => [
         'version' => '7.2',
@@ -37,4 +44,16 @@ return [
         'storage/framework/views',
         'storage/logs',
     ],
+
+    'importer' => [
+        'batch_size' => 150,
+        'stages'     => [
+            'stage1' => Stage1::class,
+            'stage2' => Stage2::class,
+            'stage3' => Stage3::class,
+            'stage4' => Stage4::class,
+            'stage5' => Stage5::class,
+            'stage6' => Stage6::class,
+        ],
+    ],
 ];

modules/Installer/Console/Commands/ImportFromClassicCommand.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Modules\Installer\Console\Commands;
+
+use App\Contracts\Command;
+use Illuminate\Support\Facades\Log;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\StageCompleted;
+use Modules\Installer\Services\Importer\ImporterService;
+
+class ImportFromClassicCommand extends Command
+{
+    protected $signature = 'phpvms:importer {db_host} {db_name} {db_user} {db_pass?} {table_prefix=phpvms_}';
+    protected $description = 'Import from an older version of phpVMS';
+
+    /**
+     * Run dev related commands
+     */
+    public function handle()
+    {
+        $creds = [
+            'host'         => $this->argument('db_host'),
+            'name'         => $this->argument('db_name'),
+            'user'         => $this->argument('db_user'),
+            'pass'         => $this->argument('db_pass'),
+            'table_prefix' => $this->argument('table_prefix'),
+        ];
+
+        $importerSvc = new ImporterService();
+
+        $importerSvc->saveCredentials($creds);
+
+        $stage = 'stage1';
+        $start = 0;
+
+        while (true) {
+            try {
+                $importerSvc->run($stage, $start);
+            } catch (ImporterNextRecordSet $e) {
+                Log::info('More records, starting from '.$e->nextOffset);
+                $start = $e->nextOffset;
+            } catch (StageCompleted $e) {
+                $stage = $e->nextStage;
+                $start = 0;
+
+                Log::info('Stage '.$stage.' completed, moving to '.$e->nextStage);
+                if ($e->nextStage === 'complete') {
+                    break;
+                }
+            }
+        }
+    }
+}

modules/Installer/Exceptions/ImporterNextRecordSet.php

@@ -0,0 +1,22 @@
+<?php
+
+namespace Modules\Installer\Exceptions;
+
+/**
+ * Go to the next page of the importer
+ */
+class ImporterNextRecordSet extends \Exception
+{
+    public $nextOffset;
+
+    /**
+     * ImporterNextRecordSet constructor.
+     *
+     * @param int $nextOffset Where to start the next set of reads from
+     */
+    public function __construct($nextOffset)
+    {
+        parent::__construct('', 0, null);
+        $this->nextOffset = $nextOffset;
+    }
+}

modules/Installer/Exceptions/ImporterNoMoreRecords.php

@@ -0,0 +1,11 @@
+<?php
+
+namespace Modules\Installer\Exceptions;
+
+class ImporterNoMoreRecords extends \Exception
+{
+    public function __construct()
+    {
+        parent::__construct('', 0, null);
+    }
+}

modules/Installer/Exceptions/StageCompleted.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Modules\Installer\Exceptions;
+
+/**
+ * Signals that the stage is completed and we should go to the next one
+ */
+class StageCompleted extends \Exception
+{
+    public $nextStage;
+
+    public function __construct($nextStage)
+    {
+        $this->nextStage = $nextStage;
+        parent::__construct('', 0, null);
+    }
+}

modules/Installer/Http/Controllers/ImporterController.php

@@ -0,0 +1,87 @@
+<?php
+
+namespace Modules\Installer\Http\Controllers;
+
+use App\Contracts\Controller;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Log;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\StageCompleted;
+use Modules\Installer\Services\Importer\ImporterService;
+
+class ImporterController extends Controller
+{
+    private $importerSvc;
+
+    public function __construct(ImporterService $importerSvc)
+    {
+        $this->importerSvc = $importerSvc;
+    }
+
+    /**
+     * Show the main page for the importer; show form for the admin email
+     * and the credentials for the other database
+     *
+     * @param \Illuminate\Http\Request $request
+     *
+     * @return \Illuminate\Contracts\View\Factory|\Illuminate\View\View
+     */
+    public function index(Request $request)
+    {
+        return view('installer::importer/step1-configure');
+    }
+
+    /**
+     * The post from the above
+     *
+     * @param \Illuminate\Http\Request $request
+     *
+     * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
+     */
+    public function config(Request $request)
+    {
+        // Save the credentials to use later
+        $this->importerSvc->saveCredentialsFromRequest($request);
+
+        // Send it to run, step1
+        return redirect(route('importer.run').'?stage=stage1&start=0');
+    }
+
+    /**
+     * Run the importer. Pass in query string with a few different parameters:
+     *
+     * stage=STAGE NAME
+     * start=record_start
+     *
+     * @param \Illuminate\Http\Request $request
+     *
+     * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector
+     */
+    public function run(Request $request)
+    {
+        $stage = $request->get('stage');
+        $start = $request->get('start');
+
+        Log::info('Starting stage '.$stage.' from offset '.$start);
+
+        try {
+            $this->importerSvc->run($stage, $start);
+        }
+
+        // The importer wants to move onto the next set of records, so refresh this page and continue
+        catch (ImporterNextRecordSet $e) {
+            Log::info('Getting more records for stage '.$stage.', starting at '.$e->nextOffset);
+            return redirect(route('importer.run').'?stage='.$stage.'&start='.$e->nextOffset);
+        }
+
+        // This stage is completed, so move onto the next one
+        catch (StageCompleted $e) {
+            if ($e->nextStage === 'complete') {
+                return view('installer::importer/complete');
+            }
+
+            Log::info('Completed stage '.$stage.', redirect to '.$e->nextStage);
+            return redirect(route('importer.run').'?stage='.$e->nextStage.'&start=0');
+        }
+    }
+}

modules/Installer/Http/Controllers/InstallerController.php

@@ -21,9 +21,6 @@ use Modules\Installer\Services\DatabaseService;
 use Modules\Installer\Services\RequirementsService;
 use Symfony\Component\HttpFoundation\File\Exception\FileException;
 
-/**
- * Class InstallerController
- */
 class InstallerController extends Controller
 {
     private $airlineRepo;

modules/Installer/Http/Controllers/UpdaterController.php

@@ -7,7 +7,7 @@ use App\Services\Installer\MigrationService;
 use App\Services\Installer\SeederService;
 use function count;
 use Illuminate\Http\Request;
-use Log;
+use Illuminate\Support\Facades\Log;
 
 /**
  * Class UpdaterController

modules/Installer/Http/Routes/importer.php

@@ -0,0 +1,9 @@
+<?php
+
+use Illuminate\Support\Facades\Route;
+
+Route::get('/', 'ImporterController@index')->name('index');
+Route::post('/config', 'ImporterController@config')->name('config');
+Route::get('/run', 'ImporterController@run')->name('run');
+
+Route::get('/complete', 'ImporterController@complete')->name('complete');

modules/Installer/Providers/InstallerServiceProvider.php

@@ -3,8 +3,9 @@
 namespace Modules\Installer\Providers;
 
 use App\Services\ModuleService;
+use Illuminate\Support\Facades\Route;
 use Illuminate\Support\ServiceProvider;
-use Route;
+use Modules\Installer\Console\Commands\ImportFromClassicCommand;
 
 class InstallerServiceProvider extends ServiceProvider
 {
@@ -17,6 +18,7 @@ class InstallerServiceProvider extends ServiceProvider
     {
         $this->moduleSvc = app(ModuleService::class);
 
+        $this->registerCommands();
         $this->registerRoutes();
         $this->registerTranslations();
         $this->registerConfig();
@@ -25,6 +27,13 @@ class InstallerServiceProvider extends ServiceProvider
         $this->loadMigrationsFrom(__DIR__.'/../Database/migrations');
     }
 
+    protected function registerCommands()
+    {
+        $this->commands([
+            ImportFromClassicCommand::class,
+        ]);
+    }
+
     /**
      * Register the routes
      */
@@ -47,6 +56,15 @@ class InstallerServiceProvider extends ServiceProvider
          ], function () {
              $this->loadRoutesFrom(__DIR__.'/../Http/Routes/update.php');
          });
+
+        Route::group([
+            'as'         => 'importer.',
+            'prefix'     => 'importer',
+            'middleware' => ['web'],
+            'namespace'  => 'Modules\Installer\Http\Controllers',
+        ], function () {
+            $this->loadRoutesFrom(__DIR__.'/../Http/Routes/importer.php');
+        });
     }
 
     /**

modules/Installer/Resources/views/importer/complete.blade.php

@@ -0,0 +1,20 @@
+@extends('installer::app')
+@section('title', 'Import Completed!')
+
+@section('content')
+  <div style="align-content: center;">
+    {{ Form::open(['route' => 'installer.complete', 'method' => 'GET']) }}
+
+    <h4>Installer Completed!</h4>
+
+    <p>Edit the <span class="code">config.php</span> to fill in some additional settings. </p>
+    <p>Click the button to proceed to the login screen!</p>
+
+    <p style="text-align: right">
+      {{ Form::submit('Install Complete! Continue to Log-In >>',
+                       ['class' => 'btn btn-success'])
+      }}
+    </p>
+    {{ Form::close() }}
+  </div>
+@endsection

modules/Installer/Resources/views/importer/step1-configure.blade.php

@@ -0,0 +1,134 @@
+@extends('installer::app')
+@section('title', 'Import Configuration')
+
+@section('content')
+  <div style="align-content: center;">
+    {{ Form::open(['route' => 'importer.config', 'method' => 'POST']) }}
+    <table class="table" width="25%">
+
+      <tr>
+        <td colspan="2">
+          <h4>IMPORTANT NOTES</h4>
+          <ul>
+            <li>If you have more than 1000 PIREPs or flights, it's best to use the command-line importer!
+              <a href="http://docs.phpvms.net/setup/importing-from-v2-v5" target="_blank">Click here</a> to
+              see the documentation of how to use it.</li>
+            <li><strong>THIS WILL WIPE OUT YOUR EXISTING DATA</strong> - this is required to make sure that things like
+            pilot IDs match up</li>
+          </ul>
+        </td>
+      </tr>
+
+      <tr>
+        <td colspan="2"><h4>Site Config</h4></td>
+      </tr>
+
+      <tr>
+        <td>Admin Email</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'email', '', ['class' => 'form-control']) }}
+            <p>The admin's email address, the password for this will be reset</p>
+          </div>
+        </td>
+      </tr>
+
+      <tr>
+        <td colspan="2"><h4>Database Config</h4></td>
+      </tr>
+
+      <tbody id="mysql_settings">
+      <tr>
+        <td>Database Host</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'db_host', '127.0.0.1', ['class' => 'form-control']) }}
+          </div>
+        </td>
+      </tr>
+
+      <tr>
+        <td>Database Port</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'db_port', '3306', ['class' => 'form-control']) }}
+          </div>
+        </td>
+      </tr>
+
+      <tr>
+        <td>Database Name</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'db_name', 'phpvms', ['class' => 'form-control']) }}
+          </div>
+        </td>
+      </tr>
+
+      <tr>
+        <td>Database User</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'db_user', null, ['class' => 'form-control']) }}
+          </div>
+        </td>
+      </tr>
+
+      <tr>
+        <td>Database Password</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'db_pass', null, ['class' => 'form-control']) }}
+          </div>
+        </td>
+      </tr>
+
+      <tr>
+        <td colspan="2" style="text-align: right;">
+          {{ Form::submit('Test Database Credentials', ['class' => 'btn btn-info', 'id' => 'dbtest_button']) }}
+        </td>
+      </tr>
+      </tbody>
+
+      <tr>
+        <td>Database Prefix</td>
+        <td style="text-align:center;">
+          <div class="form-group">
+            {{ Form::input('text', 'db_prefix', 'phpvms_', ['class' => 'form-control']) }}
+            <p>Prefix of the tables, if you're using one</p>
+          </div>
+        </td>
+      </tr>
+
+    </table>
+    <div id="dbtest"></div>
+    <p style="text-align: right">
+      {{ Form::submit('Start Importer >>', ['class' => 'btn btn-success']) }}
+    </p>
+    {{ Form::close() }}
+  </div>
+@endsection
+
+@section('scripts')
+  <script>
+    $(document).ready(() => {
+
+      $("#dbtest_button").click((e) => {
+        e.preventDefault();
+        const opts = {
+          _token: "{{ csrf_token() }}",
+          db_conn: $("#db_conn option:selected").text(),
+          db_host: $("input[name=db_host]").val(),
+          db_port: $("input[name=db_port]").val(),
+          db_name: $("input[name=db_name]").val(),
+          db_user: $("input[name=db_user]").val(),
+          db_pass: $("input[name=db_pass]").val(),
+        };
+
+        $.post("{{ route('installer.dbtest') }}", opts, (data) => {
+          $("#dbtest").html(data);
+        })
+      })
+    });
+  </script>
+@endsection

modules/Installer/Resources/views/install/steps/step3-user.blade.php

@@ -7,6 +7,12 @@
     {{ Form::open(['route' => 'installer.usersetup', 'method' => 'POST']) }}
     <table class="table" width="25%">
 
+      <tr>
+        <td colspan="2" style="text-align: right">
+          <a href="{{ route('importer.index') }}">Importing from a legacy install?</a>
+        </td>
+      </tr>
+
         <tr>
             <td colspan="2"><h4>Airline Information</h4></td>
         </tr>

modules/Installer/Services/Importer/BaseImporter.php

@@ -0,0 +1,78 @@
+<?php
+
+namespace Modules\Installer\Services\Importer;
+
+use Carbon\Carbon;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\ImporterNoMoreRecords;
+use Modules\Installer\Utils\IdMapper;
+use Modules\Installer\Utils\ImporterDB;
+use Modules\Installer\Utils\LoggerTrait;
+
+abstract class BaseImporter implements ShouldQueue
+{
+    use LoggerTrait, Dispatchable, InteractsWithQueue, Queueable;
+
+    protected $db;
+    protected $idMapper;
+
+    public function __construct()
+    {
+        $importerService = app(ImporterService::class);
+        $this->db = new ImporterDB($importerService->getCredentials());
+        $this->idMapper = app(IdMapper::class);
+    }
+
+    /**
+     * The start method. Takes the offset to start from
+     *
+     * @param int $start
+     *
+     * @throws ImporterNoMoreRecords
+     * @throws ImporterNextRecordSet
+     *
+     * @return mixed
+     */
+    abstract public function run($start = 0);
+
+    /**
+     * @param $date
+     *
+     * @return Carbon
+     */
+    protected function parseDate($date)
+    {
+        $carbon = Carbon::parse($date);
+
+        return $carbon;
+    }
+
+    /**
+     * Take a decimal duration and convert it to minutes
+     *
+     * @param $duration
+     *
+     * @return float|int
+     */
+    protected function convertDuration($duration)
+    {
+        if (strpos($duration, '.') !== false) {
+            $delim = '.';
+        } elseif (strpos($duration, ':')) {
+            $delim = ':';
+        } else {
+            // no delimiter, assume it's just a straight hour
+            return (int) $duration * 60;
+        }
+
+        $hm = explode($delim, $duration);
+        $hours = (int) $hm[0] * 60;
+        $mins = (int) $hm[1];
+
+        return $hours + $mins;
+    }
+}

modules/Installer/Services/Importer/BaseStage.php

@@ -0,0 +1,68 @@
+<?php
+
+namespace Modules\Installer\Services\Importer;
+
+use App\Repositories\KvpRepository;
+use Illuminate\Support\Facades\Log;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\ImporterNoMoreRecords;
+use Modules\Installer\Exceptions\StageCompleted;
+use Modules\Installer\Utils\IdMapper;
+use Modules\Installer\Utils\ImporterDB;
+use Modules\Installer\Utils\LoggerTrait;
+
+class BaseStage
+{
+    use LoggerTrait;
+
+    public $importers = [];
+    public $nextStage = '';
+
+    protected $db;
+    protected $idMapper;
+
+    /**
+     * @var KvpRepository
+     */
+    protected $kvpRepo;
+
+    public function __construct(ImporterDB $db, IdMapper $mapper)
+    {
+        $this->db = $db;
+        $this->idMapper = $mapper;
+
+        $this->kvpRepo = app(KvpRepository::class);
+    }
+
+    /**
+     * Run all of the given importers
+     *
+     * @param $start
+     *
+     * @throws ImporterNextRecordSet
+     * @throws StageCompleted
+     */
+    public function run($start)
+    {
+        $importersRun = $this->kvpRepo->get('importers.run', []);
+
+        foreach ($this->importers as $klass) {
+            /** @var $importer \Modules\Installer\Services\Importer\BaseImporter */
+            $importer = new $klass($this->db, $this->idMapper);
+
+            try {
+                $importer->run($start);
+            } catch (ImporterNextRecordSet $e) {
+                Log::info('Requesting next set of records');
+
+                throw $e;
+            } catch (ImporterNoMoreRecords $e) {
+                $importersRun[] = $importer;
+            }
+        }
+
+        $this->kvpRepo->save('importers.run', $importersRun);
+
+        throw new StageCompleted($this->nextStage);
+    }
+}

modules/Installer/Services/Importer/ImporterService.php

@@ -0,0 +1,111 @@
+<?php
+
+namespace Modules\Installer\Services\Importer;
+
+use App\Contracts\Service;
+use App\Repositories\KvpRepository;
+use Illuminate\Http\Request;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\StageCompleted;
+use Modules\Installer\Utils\IdMapper;
+use Modules\Installer\Utils\ImporterDB;
+
+class ImporterService extends Service
+{
+    private $CREDENTIALS_KEY = 'legacy.importer.db';
+
+    /**
+     * @var KvpRepository
+     */
+    private $kvpRepo;
+
+    /**
+     * Hold some of our data on disk for the migration
+     *
+     * @var IdMapper
+     */
+    private $idMapper;
+
+    /**
+     * Hold the PDO connection to the old database
+     *
+     * @var ImporterDB
+     */
+    private $db;
+
+    public function __construct()
+    {
+        $this->idMapper = app(IdMapper::class);
+        $this->kvpRepo = app(KvpRepository::class);
+    }
+
+    /**
+     * Save the credentials from a request
+     *
+     * @param \Illuminate\Http\Request $request
+     */
+    public function saveCredentialsFromRequest(Request $request)
+    {
+        $creds = [
+            'admin_email'  => $request->post('email'),
+            'host'         => $request->post('db_host'),
+            'port'         => $request->post('db_port'),
+            'name'         => $request->post('db_name'),
+            'user'         => $request->post('db_user'),
+            'pass'         => $request->post('db_pass'),
+            'table_prefix' => $request->post('db_prefix'),
+        ];
+
+        $this->saveCredentials($creds);
+    }
+
+    /**
+     * Save the given credentials
+     *
+     * @param array $creds
+     */
+    public function saveCredentials(array $creds)
+    {
+        $creds = array_merge([
+            'admin_email'  => '',
+            'host'         => '',
+            'port'         => '',
+            'name'         => '',
+            'user'         => '',
+            'pass'         => 3306,
+            'table_prefix' => 'phpvms_',
+        ], $creds);
+
+        $this->kvpRepo->save($this->CREDENTIALS_KEY, $creds);
+    }
+
+    /**
+     * Get the saved credentials
+     */
+    public function getCredentials()
+    {
+        return $this->kvpRepo->get($this->CREDENTIALS_KEY);
+    }
+
+    /**
+     * Run a given stage
+     *
+     * @param     $stage
+     * @param int $start
+     *
+     * @throws ImporterNextRecordSet
+     * @throws StageCompleted
+     *
+     * @return int|void
+     */
+    public function run($stage, $start = 0)
+    {
+        $db = new ImporterDB($this->kvpRepo->get($this->CREDENTIALS_KEY));
+
+        $stageKlass = config('installer.importer.stages.'.$stage);
+
+        /** @var $stage \Modules\Installer\Services\Importer\BaseStage */
+        $stage = new $stageKlass($db, $this->idMapper);
+        $stage->run($start);
+    }
+}

modules/Installer/Services/Importer/Importers/AircraftImporter.php

@@ -0,0 +1,71 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Aircraft;
+use App\Models\Airline;
+use App\Models\Subfleet;
+use Modules\Installer\Exceptions\ImporterNoMoreRecords;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class AircraftImporter extends BaseImporter
+{
+    /**
+     * CONSTANTS
+     */
+    public const SUBFLEET_NAME = 'Imported Aircraft';
+
+    /**
+     * @param int $start
+     *
+     * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
+     */
+    public function run($start = 0)
+    {
+        $this->comment('--- AIRCRAFT IMPORT ---');
+
+        $subfleet = $this->getSubfleet();
+
+        $this->info('Subfleet ID is '.$subfleet->id);
+
+        $count = 0;
+        foreach ($this->db->readRows('aircraft') as $row) {
+            $where = [
+                'name'         => $row->fullname,
+                'registration' => $row->registration,
+            ];
+
+            $aircraft = Aircraft::firstOrCreate($where, [
+                'icao'        => $row->icao,
+                'subfleet_id' => $subfleet->id,
+                'active'      => $row->enabled,
+            ]);
+
+            $this->idMapper->addMapping('aircraft', $row->id, $aircraft->id);
+
+            if ($aircraft->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' aircraft');
+
+        throw new ImporterNoMoreRecords();
+    }
+
+    /**
+     * Return the subfleet
+     *
+     * @return mixed
+     */
+    protected function getSubfleet()
+    {
+        $airline = Airline::first();
+        $subfleet = Subfleet::firstOrCreate([
+            'airline_id' => $airline->id,
+            'name'       => self::SUBFLEET_NAME,
+        ], ['type' => 'PHPVMS']);
+
+        return $subfleet;
+    }
+}

modules/Installer/Services/Importer/Importers/AirlineImporter.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Airline;
+use Illuminate\Support\Facades\Log;
+use Modules\Installer\Exceptions\ImporterNoMoreRecords;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class AirlineImporter extends BaseImporter
+{
+    /**
+     * @param int $start
+     *
+     * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
+     */
+    public function run($start = 0)
+    {
+        $this->comment('--- AIRLINE IMPORT ---');
+
+        $count = 0;
+        foreach ($this->db->readRows('airlines', $start) as $row) {
+            $airline = Airline::firstOrCreate(['icao' => $row->code], [
+                    'iata'   => $row->code,
+                    'name'   => $row->name,
+                    'active' => $row->enabled,
+                ]);
+
+            $this->idMapper->addMapping('airlines', $row->id, $airline->id);
+            $this->idMapper->addMapping('airlines', $row->code, $airline->id);
+
+            Log::debug('Mapping '.$row->id.'/'.$row->code.' to ID '.$airline->id);
+
+            if ($airline->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' airlines');
+
+        throw new ImporterNoMoreRecords();
+    }
+}

modules/Installer/Services/Importer/Importers/AirportImporter.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Airport;
+use Modules\Installer\Exceptions\ImporterNoMoreRecords;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class AirportImporter extends BaseImporter
+{
+    /**
+     * @param int $start
+     *
+     * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
+     */
+    public function run($start = 0)
+    {
+        $this->comment('--- AIRPORT IMPORT ---');
+
+        $count = 0;
+        foreach ($this->db->readRows('airports', $start) as $row) {
+            $attrs = [
+                'id'      => trim($row->icao),
+                'icao'    => trim($row->icao),
+                'name'    => $row->name,
+                'country' => $row->country,
+                'lat'     => $row->lat,
+                'lon'     => $row->lng,
+                'hub'     => $row->hub,
+            ];
+
+            $airport = Airport::updateOrCreate(['id' => $attrs['id']], $attrs);
+
+            if ($airport->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' airports');
+
+        throw new ImporterNoMoreRecords();
+    }
+}

modules/Installer/Services/Importer/Importers/FlightImporter.php

@@ -0,0 +1,51 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Flight;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class FlightImporter extends BaseImporter
+{
+    public function run($start = 0)
+    {
+        $this->comment('--- FLIGHT SCHEDULE IMPORT ---');
+
+        $count = 0;
+        foreach ($this->db->readRows('schedules', $start) as $row) {
+            $airline_id = $this->idMapper->getMapping('airlines', $row->code);
+
+            $flight_num = trim($row->flightnum);
+
+            $attrs = [
+                'dpt_airport_id' => $row->depicao,
+                'arr_airport_id' => $row->arricao,
+                'route'          => $row->route ?: '',
+                'distance'       => round($row->distance ?: 0, 2),
+                'level'          => $row->flightlevel ?: 0,
+                'dpt_time'       => $row->deptime ?: '',
+                'arr_time'       => $row->arrtime ?: '',
+                'flight_time'    => $this->convertDuration($row->flighttime) ?: '',
+                'notes'          => $row->notes ?: '',
+                'active'         => $row->enabled ?: true,
+            ];
+
+            try {
+                $w = ['airline_id' => $airline_id, 'flight_number' => $flight_num];
+                $flight = Flight::updateOrCreate($w, $attrs);
+            } catch (\Exception $e) {
+                //$this->error($e);
+            }
+
+            $this->idMapper->addMapping('flights', $row->id, $flight->id);
+
+            // TODO: deserialize route_details into ACARS table
+
+            if ($flight->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' flights');
+    }
+}

modules/Installer/Services/Importer/Importers/GroupImporter.php

@@ -0,0 +1,100 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Role;
+use App\Services\RoleService;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+/**
+ * Imports the groups into the permissions feature(s)
+ */
+class GroupImporter extends BaseImporter
+{
+    /**
+     * Permissions in the legacy system, mapping them to the current system
+     */
+    protected $legacy_permission_set = [
+        'EDIT_NEWS'              => 0x1,
+        'EDIT_PAGES'             => 0x2,
+        'EDIT_DOWNLOADS'         => 0x4,
+        'EMAIL_PILOTS'           => 0x8,
+        'EDIT_AIRLINES'          => 0x10, //
+        'EDIT_FLEET'             => 0x20, //
+        'EDIT_SCHEDULES'         => 0x80, //
+        'IMPORT_SCHEDULES'       => 0x100, //
+        'MODERATE_REGISTRATIONS' => 0x200,
+        'EDIT_PILOTS'            => 0x400, //
+        'EDIT_GROUPS'            => 0x800,
+        'EDIT_RANKS'             => 0x1000, //
+        'EDIT_AWARDS'            => 0x2000, //
+        'MODERATE_PIREPS'        => 0x4000, //
+        'VIEW_FINANCES'          => 0x8000, //
+        'EDIT_EXPENSES'          => 0x10000, //
+        'EDIT_SETTINGS'          => 0x20000, //
+        'EDIT_PIREPS_FIELDS'     => 0x40000, //
+        'EDIT_PROFILE_FIELDS'    => 0x80000, //
+        'EDIT_VACENTRAL'         => 0x100000,
+        'ACCESS_ADMIN'           => 0x2000000,
+        'FULL_ADMIN'             => 35651519, //
+    ];
+
+    /**
+     * Map a legacy value over to one of the current permission values
+     */
+    protected $legacy_to_permission = [
+        'FULL_ADMIN'             => 'admin',
+        'EDIT_AIRLINES'          => 'airlines',
+        'EDIT_AWARDS'            => 'awards',
+        'EDIT_FLEET'             => 'fleet',
+        'EDIT_EXPENCES'          => 'finances',
+        'VIEW_FINANCES'          => 'finances',
+        'EDIT_SCHEDULES'         => 'flights',
+        'EDIT_PILOTS'            => 'users',
+        'EDIT_PROFILE_FIELDS'    => 'users',
+        'EDIT_SETTINGS'          => 'settings',
+        'MODERATE_PIREPS'        => 'pireps',
+        'EDIT_PIREPS_FIELDS'     => 'pireps',
+        'EDIT_RANKS'             => 'ranks',
+        'MODERATE_REGISTRATIONS' => 'users',
+    ];
+
+    public function run($start = 0)
+    {
+        $this->comment('--- ROLES/GROUPS IMPORT ---');
+
+        $roleSvc = app(RoleService::class);
+
+        $count = 0;
+        foreach ($this->db->readRows('groups') as $row) {
+            $name = str_slug($row->name);
+            $role = Role::firstOrCreate(
+                ['name' => $name],
+                ['display_name' => $row->name]
+            );
+
+            $this->idMapper->addMapping('group', $row->groupid, $role->id);
+
+            // See if the permission set mask contains one of the mappings above
+            // Add all of the ones which apply, and then set them on the new role
+            $permissions = [];
+            foreach ($this->legacy_permission_set as $legacy_name => $mask) {
+                if (($row->permissions & $mask) === true) {
+                    if (!array_key_exists($legacy_name, $this->legacy_to_permission)) {
+                        continue;
+                    }
+
+                    $permissions[] = $this->legacy_to_permission[$legacy_name];
+                }
+            }
+
+            $roleSvc->setPermissionsForRole($role, $permissions);
+
+            if ($role->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' ranks');
+    }
+}

modules/Installer/Services/Importer/Importers/PirepImporter.php

@@ -0,0 +1,122 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Enums\FlightType;
+use App\Models\Enums\PirepSource;
+use App\Models\Enums\PirepState;
+use App\Models\Pirep;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class PirepImporter extends BaseImporter
+{
+    public function run($start = 0)
+    {
+        $this->comment('--- PIREP IMPORT ---');
+
+        $count = 0;
+        foreach ($this->db->readRows('pireps', $start) as $row) {
+            $pirep_id = $row->pirepid;
+            $user_id = $this->idMapper->getMapping('users', $row->pilotid);
+            $airline_id = $this->idMapper->getMapping('airlines', $row->code);
+            $aircraft_id = $this->idMapper->getMapping('aircraft', $row->aircraft);
+
+            $attrs = [
+                'user_id'        => $user_id,
+                'airline_id'     => $airline_id,
+                'aircraft_id'    => $aircraft_id,
+                'flight_number'  => $row->flightnum ?: '',
+                'dpt_airport_id' => $row->depicao,
+                'arr_airport_id' => $row->arricao,
+                'block_fuel'     => $row->fuelused,
+                'route'          => $row->route ?: '',
+                'source_name'    => $row->source,
+                'state'          => $this->mapState($row->accepted),
+                'created_at'     => $this->parseDate($row->submitdate),
+                'updated_at'     => $this->parseDate($row->submitdate),
+            ];
+
+            // Set the distance
+            $distance = round($row->distance ?: 0, 2);
+            $attrs['distance'] = $distance;
+            $attrs['planned_distance'] = $distance;
+
+            // Set the flight time properly
+            $duration = $this->convertDuration($row->flighttime_stamp);
+            $attrs['flight_time'] = $duration;
+            $attrs['planned_flight_time'] = $duration;
+
+            // Set how it was filed
+            if (strtoupper($row->source) === 'MANUAL') {
+                $attrs['source'] = PirepSource::MANUAL;
+            } else {
+                $attrs['source'] = PirepSource::ACARS;
+            }
+
+            // Set the flight type
+            $row->flighttype = strtoupper($row->flighttype);
+            if ($row->flighttype === 'P') {
+                $attrs['flight_type'] = FlightType::SCHED_PAX;
+            } elseif ($row->flighttype === 'C') {
+                $attrs['flight_type'] = FlightType::SCHED_CARGO;
+            } else {
+                $attrs['flight_type'] = FlightType::CHARTER_PAX_ONLY;
+            }
+
+            // Set the flight level of the PIREP is set
+            if (property_exists($row, 'flightlevel')) {
+                $attrs['level'] = $row->flightlevel;
+            } else {
+                $attrs['level'] = 0;
+            }
+
+            $pirep = Pirep::updateOrCreate(['id' => $pirep_id], $attrs);
+
+            //Log::debug('pirep oldid='.$pirep_id.', olduserid='.$row->pilotid
+            //    .'; new id='.$pirep->id.', user id='.$user_id);
+
+            $source = strtoupper($row->source);
+            if ($source === 'SMARTCARS') {
+                // TODO: Parse smartcars log into the acars table
+            } elseif ($source === 'KACARS') {
+                // TODO: Parse kACARS log into acars table
+            } elseif ($source === 'XACARS') {
+                // TODO: Parse XACARS log into acars table
+            }
+
+            // TODO: Add extra fields in as PIREP fields
+            $this->idMapper->addMapping('pireps', $row->pirepid, $pirep->id);
+
+            if ($pirep->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' pireps');
+    }
+
+    /**
+     * Map the old status to the current
+     * https://github.com/nabeelio/phpvms_v2/blob/master/core/app.config.php#L450
+     *
+     * @param int $old_state
+     *
+     * @return int
+     */
+    private function mapState($old_state)
+    {
+        $map = [
+            0 => PirepState::PENDING,
+            1 => PirepState::ACCEPTED,
+            2 => PirepState::REJECTED,
+            3 => PirepState::IN_PROGRESS,
+        ];
+
+        $old_state = (int) $old_state;
+        if (!in_array($old_state, $map)) {
+            return PirepState::PENDING;
+        }
+
+        return $map[$old_state];
+    }
+}

modules/Installer/Services/Importer/Importers/RankImport.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Models\Rank;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class RankImport extends BaseImporter
+{
+    public function run($start = 0)
+    {
+        $this->comment('--- RANK IMPORT ---');
+
+        $count = 0;
+        foreach ($this->db->readRows('ranks') as $row) {
+            $rank = Rank::firstOrCreate(['name' => $row->rank], [
+                    'image_url' => $row->rankimage,
+                    'hours'     => $row->minhours,
+                ]);
+
+            $this->idMapper->addMapping('ranks', $row->rankid, $rank->id);
+            $this->idMapper->addMapping('ranks', $row->rank, $rank->id);
+
+            if ($rank->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' ranks');
+    }
+}

modules/Installer/Services/Importer/Importers/UserImport.php

@@ -0,0 +1,139 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Importers;
+
+use App\Facades\Utils;
+use App\Models\Enums\UserState;
+use App\Models\User;
+use App\Services\UserService;
+use Illuminate\Support\Facades\Hash;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Str;
+use Modules\Installer\Services\Importer\BaseImporter;
+
+class UserImport extends BaseImporter
+{
+    /**
+     * @var UserService
+     */
+    private $userSvc;
+
+    public function run($start = 0)
+    {
+        $this->comment('--- USER IMPORT ---');
+
+        $this->userSvc = app(UserService::class);
+
+        $count = 0;
+        $first_row = true;
+        foreach ($this->db->readRows('pilots', $start) as $row) {
+            $pilot_id = $row->pilotid; // This isn't their actual ID
+            $name = $row->firstname.' '.$row->lastname;
+
+            // Figure out which airline, etc, they belong to
+            $airline_id = $this->idMapper->getMapping('airlines', $row->code);
+            // Log::info('User airline from '.$row->code.' to ID '.$airline_id);
+
+            $rank_id = $this->idMapper->getMapping('ranks', $row->rank);
+            $state = $this->getUserState($row->retired);
+
+            if ($first_row) {
+                $new_password = 'admin';
+                $first_row = false;
+            } else {
+                $new_password = Str::random(60);
+            }
+
+            // Look for a user with that pilot ID already. If someone has it
+            if ($this->userSvc->isPilotIdAlreadyUsed($pilot_id)) {
+                Log::info('User with pilot id '.$pilot_id.' exists, reassigning');
+                $pilot_id = $this->userSvc->getNextAvailablePilotId();
+            }
+
+            $attrs = [
+                'pilot_id'        => $pilot_id,
+                'name'            => $name,
+                'password'        => Hash::make($new_password),
+                'api_key'         => Utils::generateApiKey(),
+                'airline_id'      => $airline_id,
+                'rank_id'         => $rank_id,
+                'home_airport_id' => $row->hub,
+                'curr_airport_id' => $row->hub,
+                'flights'         => (int) $row->totalflights,
+                'flight_time'     => Utils::hoursToMinutes($row->totalhours),
+                'state'           => $state,
+                'created_at'      => $this->parseDate($row->joindate),
+            ];
+
+            $user = User::updateOrCreate(['email' => $row->email], $attrs);
+            $this->idMapper->addMapping('users', $row->pilotid, $user->id);
+
+            $this->updateUserRoles($user, $row->pilotid);
+
+            if ($user->wasRecentlyCreated) {
+                $count++;
+            }
+        }
+
+        $this->info('Imported '.$count.' users');
+    }
+
+    /**
+     * Update a user's roles and add them to the proper ones
+     *
+     * @param User   $user
+     * @param string $old_pilot_id
+     */
+    protected function updateUserRoles(User $user, $old_pilot_id)
+    {
+        // Be default add them to the user role, and then determine if they
+        // belong to any other groups, and add them to that
+        $this->userSvc->addUserToRole($user, 'user');
+
+        // Figure out what other groups they belong to
+    }
+
+    /**
+     * Get the user's new state from their original state
+     *
+     * @param $state
+     *
+     * @return int
+     */
+    protected function getUserState($state)
+    {
+        // Return active for now, let the stats/cron determine the status later
+        return UserState::ACTIVE;
+        /*$state = (int) $state;
+
+        // Declare array of classic states
+        $phpvms_classic_states = [
+            'ACTIVE'   => 0,
+            'INACTIVE' => 1,
+            'BANNED'   => 2,
+            'ON_LEAVE' => 3,
+        ];
+
+        // Decide which state they will be in accordance with v7
+        if ($state === $phpvms_classic_states['ACTIVE']) {
+            return UserState::ACTIVE;
+        }
+
+        if ($state === $phpvms_classic_states['INACTIVE']) {
+            // TODO: Make an inactive state?
+            return UserState::REJECTED;
+        }
+
+        if ($state === $phpvms_classic_states['BANNED']) {
+            return UserState::SUSPENDED;
+        }
+
+        if ($state === $phpvms_classic_states['ON_LEAVE']) {
+            return UserState::ON_LEAVE;
+        }
+
+        $this->error('Unknown status: '.$state);
+
+        return UserState::ACTIVE;*/
+    }
+}

modules/Installer/Services/Importer/Stages/Stage1.php

@@ -0,0 +1,97 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Stages;
+
+use App\Models\Acars;
+use App\Models\Airline;
+use App\Models\Airport;
+use App\Models\Bid;
+use App\Models\Expense;
+use App\Models\File;
+use App\Models\Flight;
+use App\Models\FlightField;
+use App\Models\FlightFieldValue;
+use App\Models\Journal;
+use App\Models\JournalTransaction;
+use App\Models\News;
+use App\Models\Pirep;
+use App\Models\Subfleet;
+use App\Models\User;
+use App\Models\UserAward;
+use Illuminate\Support\Facades\DB;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\StageCompleted;
+use Modules\Installer\Services\Importer\BaseStage;
+use Modules\Installer\Services\Importer\Importers\AircraftImporter;
+use Modules\Installer\Services\Importer\Importers\AirlineImporter;
+use Modules\Installer\Services\Importer\Importers\GroupImporter;
+use Modules\Installer\Services\Importer\Importers\RankImport;
+
+class Stage1 extends BaseStage
+{
+    public $importers = [
+        RankImport::class,
+        AirlineImporter::class,
+        AircraftImporter::class,
+        GroupImporter::class,
+    ];
+
+    public $nextStage = 'stage2';
+
+    /**
+     * @param int $start Record number to start from
+     *
+     * @throws ImporterNextRecordSet
+     * @throws StageCompleted
+     */
+    public function run($start = 0)
+    {
+        $this->cleanupDb();
+
+        // Run the first set of importers
+        parent::run($start);
+    }
+
+    /**
+     * Cleanup the local database of any users and other data that might conflict
+     * before running the importer
+     */
+    protected function cleanupDb()
+    {
+        $this->info('Running database cleanup/empty before starting');
+
+        Bid::truncate();
+        File::truncate();
+        News::truncate();
+
+        Expense::truncate();
+        JournalTransaction::truncate();
+        Journal::truncate();
+
+        // Clear flights
+        DB::table('flight_fare')->truncate();
+        DB::table('flight_subfleet')->truncate();
+        FlightField::truncate();
+        FlightFieldValue::truncate();
+        Flight::truncate();
+        Subfleet::truncate();
+
+        // Clear permissions
+//        DB::table('permission_role')->truncate();
+//        DB::table('permission_user')->truncate();
+//        DB::table('role_user')->truncate();
+//        Role::truncate();
+
+        Airline::truncate();
+        Airport::truncate();
+        Acars::truncate();
+        Pirep::truncate();
+
+        UserAward::truncate();
+        User::truncate();
+
+        // Re-run the base seeds
+        //$seederSvc = app(SeederService::class);
+        //$seederSvc->syncAllSeeds();
+    }
+}

modules/Installer/Services/Importer/Stages/Stage2.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Stages;
+
+use Modules\Installer\Services\Importer\BaseStage;
+use Modules\Installer\Services\Importer\Importers\AirportImporter;
+
+class Stage2 extends BaseStage
+{
+    public $importers = [
+        AirportImporter::class,
+    ];
+
+    public $nextStage = 'stage3';
+}

modules/Installer/Services/Importer/Stages/Stage3.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Stages;
+
+use Modules\Installer\Services\Importer\BaseStage;
+use Modules\Installer\Services\Importer\Importers\UserImport;
+
+class Stage3 extends BaseStage
+{
+    public $importers = [
+        UserImport::class,
+    ];
+
+    public $nextStage = 'stage4';
+}

modules/Installer/Services/Importer/Stages/Stage4.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Stages;
+
+use Modules\Installer\Services\Importer\BaseStage;
+use Modules\Installer\Services\Importer\Importers\FlightImporter;
+
+class Stage4 extends BaseStage
+{
+    public $importers = [
+        FlightImporter::class,
+    ];
+
+    public $nextStage = 'stage5';
+}

modules/Installer/Services/Importer/Stages/Stage5.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Stages;
+
+use Modules\Installer\Services\Importer\BaseStage;
+use Modules\Installer\Services\Importer\Importers\PirepImporter;
+
+class Stage5 extends BaseStage
+{
+    public $importers = [
+        PirepImporter::class,
+    ];
+
+    public $nextStage = 'stage6';
+}

modules/Installer/Services/Importer/Stages/Stage6.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace Modules\Installer\Services\Importer\Stages;
+
+use App\Models\User;
+use App\Services\UserService;
+use Modules\Installer\Exceptions\StageCompleted;
+use Modules\Installer\Services\Importer\BaseStage;
+
+class Stage6 extends BaseStage
+{
+    public $nextStage = 'complete';
+
+    public function run($start = 0)
+    {
+        $this->findLastPireps();
+        $this->recalculateUserStats();
+
+        throw new StageCompleted($this->nextStage);
+    }
+
+    /**
+     * Go through and set the last PIREP ID for the users
+     */
+    protected function findLastPireps()
+    {
+    }
+
+    /**
+     * Recalculate all of the user stats
+     */
+    protected function recalculateUserStats()
+    {
+        $this->comment('--- RECALCULATING USER STATS ---');
+        $userSvc = app(UserService::class);
+
+        User::all()->each(function ($user) use ($userSvc) {
+            $userSvc->recalculateStats($user);
+        });
+    }
+}

modules/Installer/Utils/IdMapper.php

@@ -0,0 +1,49 @@
+<?php
+
+namespace Modules\Installer\Utils;
+
+use App\Contracts\Service;
+use Spatie\Valuestore\Valuestore;
+
+class IdMapper extends Service
+{
+    private $valueStore;
+
+    public function __construct()
+    {
+        $this->valueStore = Valuestore::make(storage_path('app/legacy_migration.json'));
+    }
+
+    /**
+     * Create a new mapping between an old ID and the new one
+     *
+     * @param string $entity Name of the entity (e,g table)
+     * @param string $old_id
+     * @param string $new_id
+     */
+    public function addMapping($entity, $old_id, $new_id)
+    {
+        $key_name = $entity.'_'.$old_id;
+        if (!$this->valueStore->has($key_name)) {
+            $this->valueStore->put($key_name, $new_id);
+        }
+    }
+
+    /**
+     * Return the ID for a mapping
+     *
+     * @param $entity
+     * @param $old_id
+     *
+     * @return bool
+     */
+    public function getMapping($entity, $old_id)
+    {
+        $key_name = $entity.'_'.$old_id;
+        if (!$this->valueStore->has($key_name)) {
+            return 0;
+        }
+
+        return $this->valueStore->get($key_name);
+    }
+}

modules/Installer/Utils/ImporterDB.php

@@ -0,0 +1,159 @@
+<?php
+
+namespace Modules\Installer\Utils;
+
+use Illuminate\Support\Facades\Log;
+use Modules\Installer\Exceptions\ImporterNextRecordSet;
+use Modules\Installer\Exceptions\ImporterNoMoreRecords;
+use PDO;
+use PDOException;
+
+/**
+ * Real basic to interface with an importer
+ */
+class ImporterDB
+{
+    /**
+     * @var PDO
+     */
+    private $conn;
+
+    private $dsn;
+    private $creds;
+    private $batchSize;
+
+    public function __construct($creds)
+    {
+        $this->creds = $creds;
+        $this->dsn = 'mysql:'.implode(';', [
+            'host='.$this->creds['host'],
+            'port='.$this->creds['port'],
+            'dbname='.$this->creds['name'],
+            ]);
+
+        Log::info('Using DSN: '.$this->dsn);
+
+        $this->batchSize = config('installer.importer.batch_size');
+    }
+
+    public function connect()
+    {
+        try {
+            $this->conn = new PDO($this->dsn, $this->creds['user'], $this->creds['pass']);
+            $this->conn->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
+        } catch (PDOException $e) {
+            Log::error($e);
+            exit();
+        }
+    }
+
+    /**
+     * Return the table name with the prefix
+     *
+     * @param $table
+     *
+     * @return string
+     */
+    public function tableName($table)
+    {
+        if ($this->creds['table_prefix'] !== false) {
+            return $this->creds['table_prefix'].$table;
+        }
+
+        return $table;
+    }
+
+    /**
+     * @param $table
+     *
+     * @return mixed
+     */
+    public function getTotalRows($table)
+    {
+        $this->connect();
+
+        $sql = 'SELECT COUNT(*) FROM '.$this->tableName($table);
+        $rows = $this->conn->query($sql)->fetchColumn();
+
+        Log::info('Found '.$rows.' rows in '.$table);
+
+        return (int) $rows;
+    }
+
+    /**
+     * Read all the rows in a table, but read them in a batched manner
+     *
+     * @param string $table The name of the table
+     * @param int    [$start_offset]
+     *
+     * @throws \Modules\Installer\Exceptions\ImporterNextRecordSet
+     * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
+     *
+     * @return \Generator
+     */
+    public function readRows($table, $start_offset = 0)
+    {
+        $this->connect();
+
+        $offset = $start_offset;
+        $total_rows = $this->getTotalRows($table);
+
+        while ($offset < $total_rows) {
+            $rows_to_read = $offset + $this->batchSize;
+            if ($rows_to_read > $total_rows) {
+                $rows_to_read = $total_rows;
+            }
+
+            Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows);
+            yield from $this->readRowsOffset($table, $this->batchSize, $offset);
+
+            $offset += $this->batchSize;
+        }
+    }
+
+    /**
+     * @param string $table
+     * @param int    $limit  Number of rows to read
+     * @param int    $offset Where to start from
+     *
+     * @throws ImporterNextRecordSet
+     * @throws ImporterNoMoreRecords
+     *
+     * @return \Generator
+     */
+    public function readRowsOffset($table, $limit, $offset)
+    {
+        $sql = 'SELECT * FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset;
+
+        try {
+            $result = $this->conn->query($sql);
+            if (!$result) {
+                throw new ImporterNoMoreRecords();
+            }
+
+            $rowCount = $result->rowCount();
+
+            if (!$rowCount === 0) {
+                throw new ImporterNoMoreRecords();
+            }
+
+            foreach ($result as $row) {
+                yield $row;
+            }
+
+            // No more records left since we got the number below the limit
+            if ($rowCount < $limit) {
+                throw new ImporterNoMoreRecords();
+            }
+
+            throw new ImporterNextRecordSet($offset + $limit);
+        } catch (PDOException $e) {
+            // Without incrementing the offset, it should re-run the same query
+            Log::error('Error readRowsOffset: '.$e->getMessage());
+
+            if (strpos($e->getMessage(), 'server has gone away') !== false) {
+                $this->connect();
+            }
+        }
+    }
+}

modules/Installer/Utils/LoggerTrait.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace Modules\Installer\Utils;
+
+use Illuminate\Support\Facades\Log;
+
+trait LoggerTrait
+{
+    protected function comment($text)
+    {
+        Log::info($text);
+    }
+
+    protected function info($text)
+    {
+        Log::info($text);
+    }
+
+    protected function error($text)
+    {
+        Log::error($text);
+    }
+}