FGFS-4.1/phpvms
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace importer with AJAX powered; better error handling #443 (#447)

Browse Source 
* Replace importer with AJAX powered; better error handling #443

* Formatting

* Fix command line importer
This commit is contained in:
Nabeel S
2019-12-02 09:57:35 -05:00
committed by GitHub
parent 50dc79bc8d
commit 68eff40753
31 changed files with 544 additions and 354 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
modules/Installer/Services/DatabaseService.php
View File

@@ -3,7 +3,8 @@
namespace Modules\Installer\Services;
use App\Contracts\Service;
use Log;
use Illuminate\Support\Facades\Artisan;
use Illuminate\Support\Facades\Log;
use PDO;
class DatabaseService extends Service
@@ -24,19 +25,8 @@ class DatabaseService extends Service
{
Log::info('Testing Connection: '.$driver.'::'.$user.':<hidden>@'.$host.':'.$port.';'.$name);
if ($driver === 'mysql') {
$dsn = "mysql:host=$host;port=$port;dbname=$name";
Log::info('Connection string: '.$dsn);
try {
$conn = new PDO($dsn, $user, $pass);
} catch (\PDOException $e) {
throw $e;
}
}
// TODO: Needs testing
elseif ($driver === 'postgres') {
if ($driver === 'postgres') {
$dsn = "pgsql:host=$host;port=$port;dbname=$name";
try {
@@ -44,8 +34,19 @@ class DatabaseService extends Service
} catch (\PDOException $e) {
throw $e;
}
return true;
}
// Default MySQL
$dsn = "mysql:host=$host;port=$port;dbname=$name";
Log::info('Connection string: '.$dsn);
try {
$conn = new PDO($dsn, $user, $pass);
} catch (\PDOException $e) {
throw $e;
}
return true;
}
@@ -59,8 +60,8 @@ class DatabaseService extends Service
$output = '';
if (config('database.default') === 'sqlite') {
\Artisan::call('database:create');
$output .= \Artisan::output();
Artisan::call('database:create');
$output .= Artisan::output();
}
return trim($output);

48
modules/Installer/Services/Importer/BaseImporter.php
View File

@@ -7,8 +7,6 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Utils\IdMapper;
use Modules\Installer\Utils\ImporterDB;
use Modules\Installer\Utils\LoggerTrait;
@@ -19,6 +17,7 @@ abstract class BaseImporter implements ShouldQueue
protected $db;
protected $idMapper;
protected $table;
public function __construct()
{
@@ -32,13 +31,52 @@ abstract class BaseImporter implements ShouldQueue
*
* @param int $start
*
* @throws ImporterNoMoreRecords
* @throws ImporterNextRecordSet
*
* @return mixed
*/
abstract public function run($start = 0);
/**
* Return a manifest of the import tasks to run. Returns an array of objects,
* which contain a start and end row
*
* @return array
*/
public function getManifest(): array
{
$manifest = [];
$start = 0;
$total_rows = $this->db->getTotalRows($this->table);
do {
$end = $start + $this->db->batchSize;
if ($end > $total_rows) {
$end = $total_rows;
}
$idx = $start + 1;
$manifest[] = [
'importer' => get_class($this),
'start' => $start,
'end' => $end,
'message' => 'Importing '.$this->table.' ('.$idx.' - '.$end.' of '.$total_rows.')',
];
$start += $this->db->batchSize;
} while ($start < $total_rows);
return $manifest;
}
/**
* Determine what columns exist, can be used for feature testing between v2/v5
*
* @return array
*/
public function getColumns(): array
{
}
/**
* @param $date
*

68
modules/Installer/Services/Importer/BaseStage.php
View File

@@ -1,68 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer;
use App\Repositories\KvpRepository;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Utils\IdMapper;
use Modules\Installer\Utils\ImporterDB;
use Modules\Installer\Utils\LoggerTrait;
class BaseStage
{
use LoggerTrait;
public $importers = [];
public $nextStage = '';
protected $db;
protected $idMapper;
/**
* @var KvpRepository
*/
protected $kvpRepo;
public function __construct(ImporterDB $db, IdMapper $mapper)
{
$this->db = $db;
$this->idMapper = $mapper;
$this->kvpRepo = app(KvpRepository::class);
}
/**
* Run all of the given importers
*
* @param $start
*
* @throws ImporterNextRecordSet
* @throws StageCompleted
*/
public function run($start)
{
$importersRun = $this->kvpRepo->get('importers.run', []);
foreach ($this->importers as $klass) {
/** @var $importer \Modules\Installer\Services\Importer\BaseImporter */
$importer = new $klass($this->db, $this->idMapper);
try {
$importer->run($start);
} catch (ImporterNextRecordSet $e) {
Log::info('Requesting next set of records');
throw $e;
} catch (ImporterNoMoreRecords $e) {
$importersRun[] = $importer;
}
}
$this->kvpRepo->save('importers.run', $importersRun);
throw new StageCompleted($this->nextStage);
}
}

76
modules/Installer/Services/Importer/ImporterService.php
View File

@@ -4,11 +4,18 @@ namespace Modules\Installer\Services\Importer;
use App\Contracts\Service;
use App\Repositories\KvpRepository;
use Exception;
use Illuminate\Http\Request;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Utils\IdMapper;
use Modules\Installer\Utils\ImporterDB;
use Modules\Installer\Services\Importer\Importers\AircraftImporter;
use Modules\Installer\Services\Importer\Importers\AirlineImporter;
use Modules\Installer\Services\Importer\Importers\AirportImporter;
use Modules\Installer\Services\Importer\Importers\ClearDatabase;
use Modules\Installer\Services\Importer\Importers\FinalizeImporter;
use Modules\Installer\Services\Importer\Importers\FlightImporter;
use Modules\Installer\Services\Importer\Importers\GroupImporter;
use Modules\Installer\Services\Importer\Importers\PirepImporter;
use Modules\Installer\Services\Importer\Importers\RankImport;
use Modules\Installer\Services\Importer\Importers\UserImport;
class ImporterService extends Service
{
@@ -20,22 +27,23 @@ class ImporterService extends Service
private $kvpRepo;
/**
* Hold some of our data on disk for the migration
*
* @var IdMapper
* The list of importers, in proper order
*/
private $idMapper;
/**
* Hold the PDO connection to the old database
*
* @var ImporterDB
*/
private $db;
private $importList = [
ClearDatabase::class,
RankImport::class,
GroupImporter::class,
AirlineImporter::class,
AircraftImporter::class,
AirportImporter::class,
FlightImporter::class,
UserImport::class,
PirepImporter::class,
FinalizeImporter::class,
];
public function __construct()
{
$this->idMapper = app(IdMapper::class);
$this->kvpRepo = app(KvpRepository::class);
}
@@ -87,25 +95,41 @@ class ImporterService extends Service
return $this->kvpRepo->get($this->CREDENTIALS_KEY);
}
/**
* Create a manifest of the import. Creates an array with the importer name,
* which then has a subarray of all of the different steps/stages it needs to run
*/
public function generateImportManifest()
{
$manifest = [];
foreach ($this->importList as $importerKlass) {
/** @var \Modules\Installer\Services\Importer\BaseImporter $importer */
$importer = new $importerKlass();
$manifest = array_merge($manifest, $importer->getManifest());
}
return $manifest;
}
/**
* Run a given stage
*
* @param $stage
* @param $importer
* @param int $start
*
* @throws ImporterNextRecordSet
* @throws StageCompleted
* @throws \Exception
*
* @return int|void
*/
public function run($stage, $start = 0)
public function run($importer, $start = 0)
{
$db = new ImporterDB($this->kvpRepo->get($this->CREDENTIALS_KEY));
if (!in_array($importer, $this->importList)) {
throw new Exception('Unknown importer "'.$importer.'"');
}
$stageKlass = config('installer.importer.stages.'.$stage);
/** @var $stage \Modules\Installer\Services\Importer\BaseStage */
$stage = new $stageKlass($db, $this->idMapper);
$stage->run($start);
/** @var $importerInst \Modules\Installer\Services\Importer\BaseImporter */
$importerInst = new $importer();
$importerInst->run($start);
}
}

12
modules/Installer/Services/Importer/Importers/AircraftImporter.php
View File

@@ -5,21 +5,17 @@ namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Aircraft;
use App\Models\Airline;
use App\Models\Subfleet;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Services\Importer\BaseImporter;
class AircraftImporter extends BaseImporter
{
public $table = 'aircraft';
/**
* CONSTANTS
*/
public const SUBFLEET_NAME = 'Imported Aircraft';
/**
* @param int $start
*
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
*/
public function run($start = 0)
{
$this->comment('--- AIRCRAFT IMPORT ---');
@@ -29,7 +25,7 @@ class AircraftImporter extends BaseImporter
$this->info('Subfleet ID is '.$subfleet->id);
$count = 0;
foreach ($this->db->readRows('aircraft') as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$where = [
'name' => $row->fullname,
'registration' => $row->registration,
@@ -49,8 +45,6 @@ class AircraftImporter extends BaseImporter
}
$this->info('Imported '.$count.' aircraft');
throw new ImporterNoMoreRecords();
}
/**

24
modules/Installer/Services/Importer/Importers/AirlineImporter.php
View File

@@ -4,27 +4,31 @@ namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Airline;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Modules\Installer\Services\Importer\BaseImporter;
class AirlineImporter extends BaseImporter
{
public $table = 'airlines';
/**
* @param int $start
*
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
*/
public function run($start = 0)
{
$this->comment('--- AIRLINE IMPORT ---');
$count = 0;
foreach ($this->db->readRows('airlines', $start) as $row) {
$airline = Airline::firstOrCreate(['icao' => $row->code], [
'iata' => $row->code,
'name' => $row->name,
'active' => $row->enabled,
]);
foreach ($this->db->readRows($this->table, $start) as $row) {
$attrs = [
'iata' => $row->code,
'name' => $row->name,
'active' => $row->enabled,
];
$w = ['icao' => $row->code];
//$airline = Airline::firstOrCreate($w, $attrs);
$airline = Airline::create(array_merge($w, $attrs));
$this->idMapper->addMapping('airlines', $row->id, $airline->id);
$this->idMapper->addMapping('airlines', $row->code, $airline->id);
@@ -37,7 +41,5 @@ class AirlineImporter extends BaseImporter
}
$this->info('Imported '.$count.' airlines');
throw new ImporterNoMoreRecords();
}
}

39
modules/Installer/Services/Importer/Importers/AirportImporter.php
View File

@@ -3,22 +3,29 @@
namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Airport;
use Modules\Installer\Exceptions\ImporterNoMoreRecords;
use Illuminate\Database\QueryException;
use Illuminate\Support\Facades\Log;
use Modules\Installer\Services\Importer\BaseImporter;
class AirportImporter extends BaseImporter
{
/**
* @param int $start
*
* @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords
*/
protected $table = 'airports';
public function run($start = 0)
{
$this->comment('--- AIRPORT IMPORT ---');
$fields = [
'icao',
'name',
'country',
'lat',
'lng',
'hub',
];
$count = 0;
foreach ($this->db->readRows('airports', $start) as $row) {
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$attrs = [
'id' => trim($row->icao),
'icao' => trim($row->icao),
@@ -29,7 +36,21 @@ class AirportImporter extends BaseImporter
'hub' => $row->hub,
];
$airport = Airport::updateOrCreate(['id' => $attrs['id']], $attrs);
$w = ['id' => $attrs['id']];
//$airport = Airport::updateOrCreate($w, $attrs);
try {
$airport = Airport::create(array_merge($w, $attrs));
} catch (QueryException $e) {
$sqlState = $e->errorInfo[0];
$errorCode = $e->errorInfo[1];
if ($sqlState === '23000' && $errorCode === 1062) {
Log::info('Found duplicate for '.$row->icao.', ignoring');
return true;
}
return false;
}
if ($airport->wasRecentlyCreated) {
$count++;
@@ -37,7 +58,5 @@ class AirportImporter extends BaseImporter
}
$this->info('Imported '.$count.' airports');
throw new ImporterNoMoreRecords();
}
}

55
modules/Installer/Services/Importer/Stages/Stage1.php → modules/Installer/Services/Importer/Importers/ClearDatabase.php
View File

@@ -1,6 +1,6 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
namespace Modules\Installer\Services\Importer\Importers;
use App\Models\Acars;
use App\Models\Airline;
@@ -19,37 +19,28 @@ use App\Models\Subfleet;
use App\Models\User;
use App\Models\UserAward;
use Illuminate\Support\Facades\DB;
use Modules\Installer\Exceptions\ImporterNextRecordSet;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\AircraftImporter;
use Modules\Installer\Services\Importer\Importers\AirlineImporter;
use Modules\Installer\Services\Importer\Importers\GroupImporter;
use Modules\Installer\Services\Importer\Importers\RankImport;
use Modules\Installer\Services\Importer\BaseImporter;
class Stage1 extends BaseStage
class ClearDatabase extends BaseImporter
{
public $importers = [
RankImport::class,
AirlineImporter::class,
AircraftImporter::class,
GroupImporter::class,
];
public $nextStage = 'stage2';
/**
* @param int $start Record number to start from
*
* @throws ImporterNextRecordSet
* @throws StageCompleted
* Returns a default manifest just so this step gets run
*/
public function getManifest(): array
{
return [
[
'importer' => get_class($this),
'start' => 0,
'end' => 1,
'message' => 'Clearing database',
],
];
}
public function run($start = 0)
{
$this->cleanupDb();
// Run the first set of importers
parent::run($start);
}
/**
@@ -60,6 +51,8 @@ class Stage1 extends BaseStage
{
$this->info('Running database cleanup/empty before starting');
DB::statement('SET FOREIGN_KEY_CHECKS=0');
Bid::truncate();
File::truncate();
News::truncate();
@@ -77,10 +70,10 @@ class Stage1 extends BaseStage
Subfleet::truncate();
// Clear permissions
// DB::table('permission_role')->truncate();
// DB::table('permission_user')->truncate();
// DB::table('role_user')->truncate();
// Role::truncate();
// DB::table('permission_role')->truncate();
// DB::table('permission_user')->truncate();
// DB::table('role_user')->truncate();
// Role::truncate();
Airline::truncate();
Airport::truncate();
@@ -90,8 +83,6 @@ class Stage1 extends BaseStage
UserAward::truncate();
User::truncate();
// Re-run the base seeds
//$seederSvc = app(SeederService::class);
//$seederSvc->syncAllSeeds();
DB::statement('SET FOREIGN_KEY_CHECKS=1');
}
}

31
modules/Installer/Services/Importer/Stages/Stage6.php → modules/Installer/Services/Importer/Importers/FinalizeImporter.php
View File

@@ -1,22 +1,39 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
namespace Modules\Installer\Services\Importer\Importers;
use App\Models\User;
use App\Services\UserService;
use Modules\Installer\Exceptions\StageCompleted;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\BaseImporter;
class Stage6 extends BaseStage
class FinalizeImporter extends BaseImporter
{
public $nextStage = 'complete';
/**
* Returns a default manifest just so this step gets run
*/
public function getManifest(): array
{
return [
[
'importer' => get_class($this),
'start' => 0,
'end' => 1,
'message' => 'Finalizing import',
],
];
}
/**
* The start method. Takes the offset to start from
*
* @param int $start
*
* @return mixed
*/
public function run($start = 0)
{
$this->findLastPireps();
$this->recalculateUserStats();
throw new StageCompleted($this->nextStage);
}
/**

23
modules/Installer/Services/Importer/Importers/FlightImporter.php
View File

@@ -7,12 +7,30 @@ use Modules\Installer\Services\Importer\BaseImporter;
class FlightImporter extends BaseImporter
{
protected $table = 'schedules';
public function run($start = 0)
{
$this->comment('--- FLIGHT SCHEDULE IMPORT ---');
$fields = [
'id',
'code',
'flightnum',
'depicao',
'arricao',
'route',
'distance',
'flightlevel',
'deptime',
'arrtime',
'flightttime',
'notes',
'enabled',
];
$count = 0;
foreach ($this->db->readRows('schedules', $start) as $row) {
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$airline_id = $this->idMapper->getMapping('airlines', $row->code);
$flight_num = trim($row->flightnum);
@@ -32,7 +50,8 @@ class FlightImporter extends BaseImporter
try {
$w = ['airline_id' => $airline_id, 'flight_number' => $flight_num];
$flight = Flight::updateOrCreate($w, $attrs);
// $flight = Flight::updateOrCreate($w, $attrs);
$flight = Flight::create(array_merge($w, $attrs));
} catch (\Exception $e) {
//$this->error($e);
}

4
modules/Installer/Services/Importer/Importers/GroupImporter.php
View File

@@ -11,6 +11,8 @@ use Modules\Installer\Services\Importer\BaseImporter;
*/
class GroupImporter extends BaseImporter
{
protected $table = 'groups';
/**
* Permissions in the legacy system, mapping them to the current system
*/
@@ -66,7 +68,7 @@ class GroupImporter extends BaseImporter
$roleSvc = app(RoleService::class);
$count = 0;
foreach ($this->db->readRows('groups') as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$name = str_slug($row->name);
$role = Role::firstOrCreate(
['name' => $name],

28
modules/Installer/Services/Importer/Importers/PirepImporter.php
View File

@@ -10,12 +10,33 @@ use Modules\Installer\Services\Importer\BaseImporter;
class PirepImporter extends BaseImporter
{
protected $table = 'pireps';
public function run($start = 0)
{
$this->comment('--- PIREP IMPORT ---');
$fields = [
'pirepid',
'pilotid',
'code',
'aircraft',
'flightnum',
'depicao',
'arricao',
'fuelused',
'route',
'source',
'accepted',
'submitdate',
'distance',
'flighttime_stamp',
'flighttype',
'flightlevel',
];
$count = 0;
foreach ($this->db->readRows('pireps', $start) as $row) {
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$pirep_id = $row->pirepid;
$user_id = $this->idMapper->getMapping('users', $row->pilotid);
$airline_id = $this->idMapper->getMapping('airlines', $row->code);
@@ -70,7 +91,10 @@ class PirepImporter extends BaseImporter
$attrs['level'] = 0;
}
$pirep = Pirep::updateOrCreate(['id' => $pirep_id], $attrs);
$w = ['id' => $pirep_id];
$pirep = Pirep::updateOrCreate($w, $attrs);
//$pirep = Pirep::create(array_merge($w, $attrs));
//Log::debug('pirep oldid='.$pirep_id.', olduserid='.$row->pilotid
// .'; new id='.$pirep->id.', user id='.$user_id);

4
modules/Installer/Services/Importer/Importers/RankImport.php
View File

@@ -7,12 +7,14 @@ use Modules\Installer\Services\Importer\BaseImporter;
class RankImport extends BaseImporter
{
protected $table = 'ranks';
public function run($start = 0)
{
$this->comment('--- RANK IMPORT ---');
$count = 0;
foreach ($this->db->readRows('ranks') as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$rank = Rank::firstOrCreate(['name' => $row->rank], [
'image_url' => $row->rankimage,
'hours' => $row->minhours,

4
modules/Installer/Services/Importer/Importers/UserImport.php
View File

@@ -13,6 +13,8 @@ use Modules\Installer\Services\Importer\BaseImporter;
class UserImport extends BaseImporter
{
protected $table = 'pilots';
/**
* @var UserService
*/
@@ -26,7 +28,7 @@ class UserImport extends BaseImporter
$count = 0;
$first_row = true;
foreach ($this->db->readRows('pilots', $start) as $row) {
foreach ($this->db->readRows($this->table, $start) as $row) {
$pilot_id = $row->pilotid; // This isn't their actual ID
$name = $row->firstname.' '.$row->lastname;

15
modules/Installer/Services/Importer/Stages/Stage2.php
View File

@@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\AirportImporter;
class Stage2 extends BaseStage
{
public $importers = [
AirportImporter::class,
];
public $nextStage = 'stage3';
}

15
modules/Installer/Services/Importer/Stages/Stage3.php
View File

@@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\UserImport;
class Stage3 extends BaseStage
{
public $importers = [
UserImport::class,
];
public $nextStage = 'stage4';
}

15
modules/Installer/Services/Importer/Stages/Stage4.php
View File

@@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\FlightImporter;
class Stage4 extends BaseStage
{
public $importers = [
FlightImporter::class,
];
public $nextStage = 'stage5';
}

15
modules/Installer/Services/Importer/Stages/Stage5.php
View File

@@ -1,15 +0,0 @@
<?php
namespace Modules\Installer\Services\Importer\Stages;
use Modules\Installer\Services\Importer\BaseStage;
use Modules\Installer\Services\Importer\Importers\PirepImporter;
class Stage5 extends BaseStage
{
public $importers = [
PirepImporter::class,
];
public $nextStage = 'stage6';
}