diff --git a/app/Contracts/Model.php b/app/Contracts/Model.php index 916bce44..69844dba 100644 --- a/app/Contracts/Model.php +++ b/app/Contracts/Model.php @@ -6,8 +6,10 @@ namespace App\Contracts; * @property mixed $id * @property bool $skip_mutator * - * @method static Model find(int $airline_id) + * @method static create(array $attrs) + * @method static Model find(int $id) * @method static Model where(array $array) + * @method static Model firstOrCreate(array $where, array $array) * @method static Model updateOrCreate(array $array, array $attrs) * @method static truncate() */ diff --git a/app/Models/Aircraft.php b/app/Models/Aircraft.php index d8bfd8cf..d73bd05f 100644 --- a/app/Models/Aircraft.php +++ b/app/Models/Aircraft.php @@ -6,6 +6,7 @@ use App\Contracts\Model; use App\Models\Enums\AircraftStatus; use App\Models\Traits\ExpensableTrait; use App\Models\Traits\FilesTrait; +use Carbon\Carbon; /** * @property int id diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 9b3acf51..cd1d087b 100755 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -25,6 +25,7 @@ use App\Models\Subfleet; use App\Models\User; use App\Repositories\SettingRepository; use App\Services\ModuleService; +use Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider; use Illuminate\Support\Facades\Schema; use Illuminate\Support\Facades\View; use Illuminate\Support\ServiceProvider; @@ -66,13 +67,11 @@ class AppServiceProvider extends ServiceProvider */ public function register(): void { - // Only dev environment stuff - if ($this->app->environment() === 'dev') { - // Only load the IDE helper if it's included. This lets use distribute the - // package without any dev dependencies + // Only load the IDE helper if it's included and enabled + if (config('app.debug_toolbar') === true) { /* @noinspection NestedPositiveIfStatementsInspection */ - if (class_exists(\Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider::class)) { - $this->app->register(\Barryvdh\LaravelIdeHelper\IdeHelperServiceProvider::class); + if (class_exists(IdeHelperServiceProvider::class)) { + $this->app->register(IdeHelperServiceProvider::class); } } } diff --git a/config/app.php b/config/app.php index 69d570c8..2fafc44c 100755 --- a/config/app.php +++ b/config/app.php @@ -8,11 +8,12 @@ */ return [ - 'name' => env('APP_NAME', 'phpvms'), - 'env' => env('APP_ENV', 'dev'), - 'debug' => env('APP_DEBUG', true), - 'url' => env('APP_URL', 'http://localhost'), - 'version' => '7.0.0', + 'name' => env('APP_NAME', 'phpvms'), + 'env' => env('APP_ENV', 'dev'), + 'debug' => env('APP_DEBUG', true), + 'url' => env('APP_URL', 'http://localhost'), + 'version' => '7.0.0', + 'debug_toolbar' => false, 'locale' => env('APP_LOCALE', 'en'), 'fallback_locale' => 'en', diff --git a/modules/Installer/Config/config.php b/modules/Installer/Config/config.php index c423a8af..cdd6e2fe 100644 --- a/modules/Installer/Config/config.php +++ b/modules/Installer/Config/config.php @@ -1,12 +1,5 @@ [ 'version' => '7.2', @@ -46,14 +39,6 @@ return [ ], 'importer' => [ - 'batch_size' => 150, - 'stages' => [ - 'stage1' => Stage1::class, - 'stage2' => Stage2::class, - 'stage3' => Stage3::class, - 'stage4' => Stage4::class, - 'stage5' => Stage5::class, - 'stage6' => Stage6::class, - ], + 'batch_size' => 20, ], ]; diff --git a/modules/Installer/Console/Commands/ImportFromClassicCommand.php b/modules/Installer/Console/Commands/ImportFromClassicCommand.php index c6b7b24e..89d3b72b 100644 --- a/modules/Installer/Console/Commands/ImportFromClassicCommand.php +++ b/modules/Installer/Console/Commands/ImportFromClassicCommand.php @@ -4,8 +4,6 @@ namespace Modules\Installer\Console\Commands; use App\Contracts\Command; use Illuminate\Support\Facades\Log; -use Modules\Installer\Exceptions\ImporterNextRecordSet; -use Modules\Installer\Exceptions\StageCompleted; use Modules\Installer\Services\Importer\ImporterService; class ImportFromClassicCommand extends Command @@ -29,24 +27,13 @@ class ImportFromClassicCommand extends Command $importerSvc = new ImporterService(); $importerSvc->saveCredentials($creds); + $manifest = $importerSvc->generateImportManifest(); - $stage = 'stage1'; - $start = 0; - - while (true) { + foreach ($manifest as $record) { try { - $importerSvc->run($stage, $start); - } catch (ImporterNextRecordSet $e) { - Log::info('More records, starting from '.$e->nextOffset); - $start = $e->nextOffset; - } catch (StageCompleted $e) { - $stage = $e->nextStage; - $start = 0; - - Log::info('Stage '.$stage.' completed, moving to '.$e->nextStage); - if ($e->nextStage === 'complete') { - break; - } + $importerSvc->run($record['importer'], $record['start']); + } catch (\Exception $e) { + Log::error($e->getMessage()); } } } diff --git a/modules/Installer/Http/Controllers/ImporterController.php b/modules/Installer/Http/Controllers/ImporterController.php index 8e950de9..ddc93023 100644 --- a/modules/Installer/Http/Controllers/ImporterController.php +++ b/modules/Installer/Http/Controllers/ImporterController.php @@ -5,8 +5,6 @@ namespace Modules\Installer\Http\Controllers; use App\Contracts\Controller; use Illuminate\Http\Request; use Illuminate\Support\Facades\Log; -use Modules\Installer\Exceptions\ImporterNextRecordSet; -use Modules\Installer\Exceptions\StageCompleted; use Modules\Installer\Services\Importer\ImporterService; class ImporterController extends Controller @@ -28,6 +26,8 @@ class ImporterController extends Controller */ public function index(Request $request) { + app('debugbar')->disable(); // saves the query logging + return view('installer::importer/step1-configure'); } @@ -40,11 +40,26 @@ class ImporterController extends Controller */ public function config(Request $request) { - // Save the credentials to use later - $this->importerSvc->saveCredentialsFromRequest($request); + app('debugbar')->disable(); // saves the query logging + + try { + // Save the credentials to use later + $this->importerSvc->saveCredentialsFromRequest($request); + + // Generate the import manifest + $manifest = $this->importerSvc->generateImportManifest(); + } catch (\Exception $e) { + Log::error($e->getMessage()); + // Send it to run, step1 + return view('installer::importer/error', [ + 'error' => $e->getMessage(), + ]); + } // Send it to run, step1 - return redirect(route('importer.run').'?stage=stage1&start=0'); + return view('installer::importer/step2-processing', [ + 'manifest' => $manifest, + ]); } /** @@ -55,33 +70,31 @@ class ImporterController extends Controller * * @param \Illuminate\Http\Request $request * + * @throws \Exception + * * @return \Illuminate\Http\RedirectResponse|\Illuminate\Routing\Redirector */ public function run(Request $request) { - $stage = $request->get('stage'); - $start = $request->get('start'); + app('debugbar')->disable(); // saves the query logging - Log::info('Starting stage '.$stage.' from offset '.$start); + $importer = $request->input('importer'); + $start = $request->input('start'); - try { - $this->importerSvc->run($stage, $start); - } + Log::info('Starting stage '.$importer.' from offset '.$start); - // The importer wants to move onto the next set of records, so refresh this page and continue - catch (ImporterNextRecordSet $e) { - Log::info('Getting more records for stage '.$stage.', starting at '.$e->nextOffset); - return redirect(route('importer.run').'?stage='.$stage.'&start='.$e->nextOffset); - } + $this->importerSvc->run($importer, $start); - // This stage is completed, so move onto the next one - catch (StageCompleted $e) { - if ($e->nextStage === 'complete') { - return view('installer::importer/complete'); - } + return response()->json([ + 'message' => 'completed', + ]); + } - Log::info('Completed stage '.$stage.', redirect to '.$e->nextStage); - return redirect(route('importer.run').'?stage='.$e->nextStage.'&start=0'); - } + /** + * Complete the import + */ + public function complete() + { + return redirect('/'); } } diff --git a/modules/Installer/Http/Routes/importer.php b/modules/Installer/Http/Routes/importer.php index 2f9c0f26..92821138 100644 --- a/modules/Installer/Http/Routes/importer.php +++ b/modules/Installer/Http/Routes/importer.php @@ -4,6 +4,6 @@ use Illuminate\Support\Facades\Route; Route::get('/', 'ImporterController@index')->name('index'); Route::post('/config', 'ImporterController@config')->name('config'); -Route::get('/run', 'ImporterController@run')->name('run'); +Route::post('/run', 'ImporterController@run')->middleware('api')->name('run'); -Route::get('/complete', 'ImporterController@complete')->name('complete'); +Route::post('/complete', 'ImporterController@complete')->name('complete'); diff --git a/modules/Installer/Resources/views/app.blade.php b/modules/Installer/Resources/views/app.blade.php index 41ff67e8..e646f37f 100644 --- a/modules/Installer/Resources/views/app.blade.php +++ b/modules/Installer/Resources/views/app.blade.php @@ -12,6 +12,9 @@ + + + @@ -63,6 +66,9 @@ {{----}} + + + diff --git a/modules/Installer/Resources/views/importer/error.blade.php b/modules/Installer/Resources/views/importer/error.blade.php new file mode 100644 index 00000000..41610bde --- /dev/null +++ b/modules/Installer/Resources/views/importer/error.blade.php @@ -0,0 +1,9 @@ +@extends('installer::app') +@section('title', 'Import Error!') + +@section('content') +
+

Error!

+

{{ $error }}

+
+@endsection diff --git a/modules/Installer/Resources/views/importer/step1-configure.blade.php b/modules/Installer/Resources/views/importer/step1-configure.blade.php index 91eaabcc..3a45e6f2 100644 --- a/modules/Installer/Resources/views/importer/step1-configure.blade.php +++ b/modules/Installer/Resources/views/importer/step1-configure.blade.php @@ -117,7 +117,7 @@ e.preventDefault(); const opts = { _token: "{{ csrf_token() }}", - db_conn: $("#db_conn option:selected").text(), + db_conn: 'mysql', db_host: $("input[name=db_host]").val(), db_port: $("input[name=db_port]").val(), db_name: $("input[name=db_name]").val(), diff --git a/modules/Installer/Resources/views/importer/step2-processing.blade.php b/modules/Installer/Resources/views/importer/step2-processing.blade.php new file mode 100644 index 00000000..e482b457 --- /dev/null +++ b/modules/Installer/Resources/views/importer/step2-processing.blade.php @@ -0,0 +1,171 @@ +@extends('installer::app') +@section('title', 'Import Configuration') + +@section('content') +
+ {{ Form::open(['route' => 'importer.complete', 'method' => 'POST']) }} + + + + + + + + + + +

Running Importer

+
+
+
+
+

+

+
+
+

+ {{ Form::submit('Complete Import', [ + 'id' => 'completebutton', + 'class' => 'btn btn-success' + ]) }} +

+ {{ Form::close() }} +
+@endsection + +@section('scripts') + +@endsection diff --git a/modules/Installer/Services/DatabaseService.php b/modules/Installer/Services/DatabaseService.php index 64cd4915..9a1618b6 100644 --- a/modules/Installer/Services/DatabaseService.php +++ b/modules/Installer/Services/DatabaseService.php @@ -3,7 +3,8 @@ namespace Modules\Installer\Services; use App\Contracts\Service; -use Log; +use Illuminate\Support\Facades\Artisan; +use Illuminate\Support\Facades\Log; use PDO; class DatabaseService extends Service @@ -24,19 +25,8 @@ class DatabaseService extends Service { Log::info('Testing Connection: '.$driver.'::'.$user.':@'.$host.':'.$port.';'.$name); - if ($driver === 'mysql') { - $dsn = "mysql:host=$host;port=$port;dbname=$name"; - Log::info('Connection string: '.$dsn); - - try { - $conn = new PDO($dsn, $user, $pass); - } catch (\PDOException $e) { - throw $e; - } - } - // TODO: Needs testing - elseif ($driver === 'postgres') { + if ($driver === 'postgres') { $dsn = "pgsql:host=$host;port=$port;dbname=$name"; try { @@ -44,8 +34,19 @@ class DatabaseService extends Service } catch (\PDOException $e) { throw $e; } + + return true; } + // Default MySQL + $dsn = "mysql:host=$host;port=$port;dbname=$name"; + Log::info('Connection string: '.$dsn); + + try { + $conn = new PDO($dsn, $user, $pass); + } catch (\PDOException $e) { + throw $e; + } return true; } @@ -59,8 +60,8 @@ class DatabaseService extends Service $output = ''; if (config('database.default') === 'sqlite') { - \Artisan::call('database:create'); - $output .= \Artisan::output(); + Artisan::call('database:create'); + $output .= Artisan::output(); } return trim($output); diff --git a/modules/Installer/Services/Importer/BaseImporter.php b/modules/Installer/Services/Importer/BaseImporter.php index 091c68e9..d410626d 100644 --- a/modules/Installer/Services/Importer/BaseImporter.php +++ b/modules/Installer/Services/Importer/BaseImporter.php @@ -7,8 +7,6 @@ use Illuminate\Bus\Queueable; use Illuminate\Contracts\Queue\ShouldQueue; use Illuminate\Foundation\Bus\Dispatchable; use Illuminate\Queue\InteractsWithQueue; -use Modules\Installer\Exceptions\ImporterNextRecordSet; -use Modules\Installer\Exceptions\ImporterNoMoreRecords; use Modules\Installer\Utils\IdMapper; use Modules\Installer\Utils\ImporterDB; use Modules\Installer\Utils\LoggerTrait; @@ -19,6 +17,7 @@ abstract class BaseImporter implements ShouldQueue protected $db; protected $idMapper; + protected $table; public function __construct() { @@ -32,13 +31,52 @@ abstract class BaseImporter implements ShouldQueue * * @param int $start * - * @throws ImporterNoMoreRecords - * @throws ImporterNextRecordSet - * * @return mixed */ abstract public function run($start = 0); + /** + * Return a manifest of the import tasks to run. Returns an array of objects, + * which contain a start and end row + * + * @return array + */ + public function getManifest(): array + { + $manifest = []; + + $start = 0; + $total_rows = $this->db->getTotalRows($this->table); + do { + $end = $start + $this->db->batchSize; + if ($end > $total_rows) { + $end = $total_rows; + } + + $idx = $start + 1; + + $manifest[] = [ + 'importer' => get_class($this), + 'start' => $start, + 'end' => $end, + 'message' => 'Importing '.$this->table.' ('.$idx.' - '.$end.' of '.$total_rows.')', + ]; + + $start += $this->db->batchSize; + } while ($start < $total_rows); + + return $manifest; + } + + /** + * Determine what columns exist, can be used for feature testing between v2/v5 + * + * @return array + */ + public function getColumns(): array + { + } + /** * @param $date * diff --git a/modules/Installer/Services/Importer/BaseStage.php b/modules/Installer/Services/Importer/BaseStage.php deleted file mode 100644 index 613c5ed5..00000000 --- a/modules/Installer/Services/Importer/BaseStage.php +++ /dev/null @@ -1,68 +0,0 @@ -db = $db; - $this->idMapper = $mapper; - - $this->kvpRepo = app(KvpRepository::class); - } - - /** - * Run all of the given importers - * - * @param $start - * - * @throws ImporterNextRecordSet - * @throws StageCompleted - */ - public function run($start) - { - $importersRun = $this->kvpRepo->get('importers.run', []); - - foreach ($this->importers as $klass) { - /** @var $importer \Modules\Installer\Services\Importer\BaseImporter */ - $importer = new $klass($this->db, $this->idMapper); - - try { - $importer->run($start); - } catch (ImporterNextRecordSet $e) { - Log::info('Requesting next set of records'); - - throw $e; - } catch (ImporterNoMoreRecords $e) { - $importersRun[] = $importer; - } - } - - $this->kvpRepo->save('importers.run', $importersRun); - - throw new StageCompleted($this->nextStage); - } -} diff --git a/modules/Installer/Services/Importer/ImporterService.php b/modules/Installer/Services/Importer/ImporterService.php index dca5e0f0..dcd9fe4d 100644 --- a/modules/Installer/Services/Importer/ImporterService.php +++ b/modules/Installer/Services/Importer/ImporterService.php @@ -4,11 +4,18 @@ namespace Modules\Installer\Services\Importer; use App\Contracts\Service; use App\Repositories\KvpRepository; +use Exception; use Illuminate\Http\Request; -use Modules\Installer\Exceptions\ImporterNextRecordSet; -use Modules\Installer\Exceptions\StageCompleted; -use Modules\Installer\Utils\IdMapper; -use Modules\Installer\Utils\ImporterDB; +use Modules\Installer\Services\Importer\Importers\AircraftImporter; +use Modules\Installer\Services\Importer\Importers\AirlineImporter; +use Modules\Installer\Services\Importer\Importers\AirportImporter; +use Modules\Installer\Services\Importer\Importers\ClearDatabase; +use Modules\Installer\Services\Importer\Importers\FinalizeImporter; +use Modules\Installer\Services\Importer\Importers\FlightImporter; +use Modules\Installer\Services\Importer\Importers\GroupImporter; +use Modules\Installer\Services\Importer\Importers\PirepImporter; +use Modules\Installer\Services\Importer\Importers\RankImport; +use Modules\Installer\Services\Importer\Importers\UserImport; class ImporterService extends Service { @@ -20,22 +27,23 @@ class ImporterService extends Service private $kvpRepo; /** - * Hold some of our data on disk for the migration - * - * @var IdMapper + * The list of importers, in proper order */ - private $idMapper; - - /** - * Hold the PDO connection to the old database - * - * @var ImporterDB - */ - private $db; + private $importList = [ + ClearDatabase::class, + RankImport::class, + GroupImporter::class, + AirlineImporter::class, + AircraftImporter::class, + AirportImporter::class, + FlightImporter::class, + UserImport::class, + PirepImporter::class, + FinalizeImporter::class, + ]; public function __construct() { - $this->idMapper = app(IdMapper::class); $this->kvpRepo = app(KvpRepository::class); } @@ -87,25 +95,41 @@ class ImporterService extends Service return $this->kvpRepo->get($this->CREDENTIALS_KEY); } + /** + * Create a manifest of the import. Creates an array with the importer name, + * which then has a subarray of all of the different steps/stages it needs to run + */ + public function generateImportManifest() + { + $manifest = []; + + foreach ($this->importList as $importerKlass) { + /** @var \Modules\Installer\Services\Importer\BaseImporter $importer */ + $importer = new $importerKlass(); + $manifest = array_merge($manifest, $importer->getManifest()); + } + + return $manifest; + } + /** * Run a given stage * - * @param $stage + * @param $importer * @param int $start * - * @throws ImporterNextRecordSet - * @throws StageCompleted + * @throws \Exception * * @return int|void */ - public function run($stage, $start = 0) + public function run($importer, $start = 0) { - $db = new ImporterDB($this->kvpRepo->get($this->CREDENTIALS_KEY)); + if (!in_array($importer, $this->importList)) { + throw new Exception('Unknown importer "'.$importer.'"'); + } - $stageKlass = config('installer.importer.stages.'.$stage); - - /** @var $stage \Modules\Installer\Services\Importer\BaseStage */ - $stage = new $stageKlass($db, $this->idMapper); - $stage->run($start); + /** @var $importerInst \Modules\Installer\Services\Importer\BaseImporter */ + $importerInst = new $importer(); + $importerInst->run($start); } } diff --git a/modules/Installer/Services/Importer/Importers/AircraftImporter.php b/modules/Installer/Services/Importer/Importers/AircraftImporter.php index f90f3f8c..b2f6389a 100644 --- a/modules/Installer/Services/Importer/Importers/AircraftImporter.php +++ b/modules/Installer/Services/Importer/Importers/AircraftImporter.php @@ -5,21 +5,17 @@ namespace Modules\Installer\Services\Importer\Importers; use App\Models\Aircraft; use App\Models\Airline; use App\Models\Subfleet; -use Modules\Installer\Exceptions\ImporterNoMoreRecords; use Modules\Installer\Services\Importer\BaseImporter; class AircraftImporter extends BaseImporter { + public $table = 'aircraft'; + /** * CONSTANTS */ public const SUBFLEET_NAME = 'Imported Aircraft'; - /** - * @param int $start - * - * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords - */ public function run($start = 0) { $this->comment('--- AIRCRAFT IMPORT ---'); @@ -29,7 +25,7 @@ class AircraftImporter extends BaseImporter $this->info('Subfleet ID is '.$subfleet->id); $count = 0; - foreach ($this->db->readRows('aircraft') as $row) { + foreach ($this->db->readRows($this->table, $start) as $row) { $where = [ 'name' => $row->fullname, 'registration' => $row->registration, @@ -49,8 +45,6 @@ class AircraftImporter extends BaseImporter } $this->info('Imported '.$count.' aircraft'); - - throw new ImporterNoMoreRecords(); } /** diff --git a/modules/Installer/Services/Importer/Importers/AirlineImporter.php b/modules/Installer/Services/Importer/Importers/AirlineImporter.php index b297ceeb..f2b1e298 100644 --- a/modules/Installer/Services/Importer/Importers/AirlineImporter.php +++ b/modules/Installer/Services/Importer/Importers/AirlineImporter.php @@ -4,27 +4,31 @@ namespace Modules\Installer\Services\Importer\Importers; use App\Models\Airline; use Illuminate\Support\Facades\Log; -use Modules\Installer\Exceptions\ImporterNoMoreRecords; use Modules\Installer\Services\Importer\BaseImporter; class AirlineImporter extends BaseImporter { + public $table = 'airlines'; + /** * @param int $start - * - * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords */ public function run($start = 0) { $this->comment('--- AIRLINE IMPORT ---'); $count = 0; - foreach ($this->db->readRows('airlines', $start) as $row) { - $airline = Airline::firstOrCreate(['icao' => $row->code], [ - 'iata' => $row->code, - 'name' => $row->name, - 'active' => $row->enabled, - ]); + foreach ($this->db->readRows($this->table, $start) as $row) { + $attrs = [ + 'iata' => $row->code, + 'name' => $row->name, + 'active' => $row->enabled, + ]; + + $w = ['icao' => $row->code]; + + //$airline = Airline::firstOrCreate($w, $attrs); + $airline = Airline::create(array_merge($w, $attrs)); $this->idMapper->addMapping('airlines', $row->id, $airline->id); $this->idMapper->addMapping('airlines', $row->code, $airline->id); @@ -37,7 +41,5 @@ class AirlineImporter extends BaseImporter } $this->info('Imported '.$count.' airlines'); - - throw new ImporterNoMoreRecords(); } } diff --git a/modules/Installer/Services/Importer/Importers/AirportImporter.php b/modules/Installer/Services/Importer/Importers/AirportImporter.php index bf50efc5..7cf17bde 100644 --- a/modules/Installer/Services/Importer/Importers/AirportImporter.php +++ b/modules/Installer/Services/Importer/Importers/AirportImporter.php @@ -3,22 +3,29 @@ namespace Modules\Installer\Services\Importer\Importers; use App\Models\Airport; -use Modules\Installer\Exceptions\ImporterNoMoreRecords; +use Illuminate\Database\QueryException; +use Illuminate\Support\Facades\Log; use Modules\Installer\Services\Importer\BaseImporter; class AirportImporter extends BaseImporter { - /** - * @param int $start - * - * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords - */ + protected $table = 'airports'; + public function run($start = 0) { $this->comment('--- AIRPORT IMPORT ---'); + $fields = [ + 'icao', + 'name', + 'country', + 'lat', + 'lng', + 'hub', + ]; + $count = 0; - foreach ($this->db->readRows('airports', $start) as $row) { + foreach ($this->db->readRows($this->table, $start, $fields) as $row) { $attrs = [ 'id' => trim($row->icao), 'icao' => trim($row->icao), @@ -29,7 +36,21 @@ class AirportImporter extends BaseImporter 'hub' => $row->hub, ]; - $airport = Airport::updateOrCreate(['id' => $attrs['id']], $attrs); + $w = ['id' => $attrs['id']]; + //$airport = Airport::updateOrCreate($w, $attrs); + + try { + $airport = Airport::create(array_merge($w, $attrs)); + } catch (QueryException $e) { + $sqlState = $e->errorInfo[0]; + $errorCode = $e->errorInfo[1]; + if ($sqlState === '23000' && $errorCode === 1062) { + Log::info('Found duplicate for '.$row->icao.', ignoring'); + return true; + } + + return false; + } if ($airport->wasRecentlyCreated) { $count++; @@ -37,7 +58,5 @@ class AirportImporter extends BaseImporter } $this->info('Imported '.$count.' airports'); - - throw new ImporterNoMoreRecords(); } } diff --git a/modules/Installer/Services/Importer/Stages/Stage1.php b/modules/Installer/Services/Importer/Importers/ClearDatabase.php similarity index 54% rename from modules/Installer/Services/Importer/Stages/Stage1.php rename to modules/Installer/Services/Importer/Importers/ClearDatabase.php index ad3d7376..6e98d72a 100644 --- a/modules/Installer/Services/Importer/Stages/Stage1.php +++ b/modules/Installer/Services/Importer/Importers/ClearDatabase.php @@ -1,6 +1,6 @@ get_class($this), + 'start' => 0, + 'end' => 1, + 'message' => 'Clearing database', + ], + ]; + } + public function run($start = 0) { $this->cleanupDb(); - - // Run the first set of importers - parent::run($start); } /** @@ -60,6 +51,8 @@ class Stage1 extends BaseStage { $this->info('Running database cleanup/empty before starting'); + DB::statement('SET FOREIGN_KEY_CHECKS=0'); + Bid::truncate(); File::truncate(); News::truncate(); @@ -77,10 +70,10 @@ class Stage1 extends BaseStage Subfleet::truncate(); // Clear permissions -// DB::table('permission_role')->truncate(); -// DB::table('permission_user')->truncate(); -// DB::table('role_user')->truncate(); -// Role::truncate(); + // DB::table('permission_role')->truncate(); + // DB::table('permission_user')->truncate(); + // DB::table('role_user')->truncate(); + // Role::truncate(); Airline::truncate(); Airport::truncate(); @@ -90,8 +83,6 @@ class Stage1 extends BaseStage UserAward::truncate(); User::truncate(); - // Re-run the base seeds - //$seederSvc = app(SeederService::class); - //$seederSvc->syncAllSeeds(); + DB::statement('SET FOREIGN_KEY_CHECKS=1'); } } diff --git a/modules/Installer/Services/Importer/Stages/Stage6.php b/modules/Installer/Services/Importer/Importers/FinalizeImporter.php similarity index 50% rename from modules/Installer/Services/Importer/Stages/Stage6.php rename to modules/Installer/Services/Importer/Importers/FinalizeImporter.php index 07e605c7..87ed7249 100644 --- a/modules/Installer/Services/Importer/Stages/Stage6.php +++ b/modules/Installer/Services/Importer/Importers/FinalizeImporter.php @@ -1,22 +1,39 @@ get_class($this), + 'start' => 0, + 'end' => 1, + 'message' => 'Finalizing import', + ], + ]; + } + /** + * The start method. Takes the offset to start from + * + * @param int $start + * + * @return mixed + */ public function run($start = 0) { $this->findLastPireps(); $this->recalculateUserStats(); - - throw new StageCompleted($this->nextStage); } /** diff --git a/modules/Installer/Services/Importer/Importers/FlightImporter.php b/modules/Installer/Services/Importer/Importers/FlightImporter.php index c70427be..f4fd6261 100644 --- a/modules/Installer/Services/Importer/Importers/FlightImporter.php +++ b/modules/Installer/Services/Importer/Importers/FlightImporter.php @@ -7,12 +7,30 @@ use Modules\Installer\Services\Importer\BaseImporter; class FlightImporter extends BaseImporter { + protected $table = 'schedules'; + public function run($start = 0) { $this->comment('--- FLIGHT SCHEDULE IMPORT ---'); + $fields = [ + 'id', + 'code', + 'flightnum', + 'depicao', + 'arricao', + 'route', + 'distance', + 'flightlevel', + 'deptime', + 'arrtime', + 'flightttime', + 'notes', + 'enabled', + ]; + $count = 0; - foreach ($this->db->readRows('schedules', $start) as $row) { + foreach ($this->db->readRows($this->table, $start, $fields) as $row) { $airline_id = $this->idMapper->getMapping('airlines', $row->code); $flight_num = trim($row->flightnum); @@ -32,7 +50,8 @@ class FlightImporter extends BaseImporter try { $w = ['airline_id' => $airline_id, 'flight_number' => $flight_num]; - $flight = Flight::updateOrCreate($w, $attrs); + // $flight = Flight::updateOrCreate($w, $attrs); + $flight = Flight::create(array_merge($w, $attrs)); } catch (\Exception $e) { //$this->error($e); } diff --git a/modules/Installer/Services/Importer/Importers/GroupImporter.php b/modules/Installer/Services/Importer/Importers/GroupImporter.php index ae1b9cf2..00650833 100644 --- a/modules/Installer/Services/Importer/Importers/GroupImporter.php +++ b/modules/Installer/Services/Importer/Importers/GroupImporter.php @@ -11,6 +11,8 @@ use Modules\Installer\Services\Importer\BaseImporter; */ class GroupImporter extends BaseImporter { + protected $table = 'groups'; + /** * Permissions in the legacy system, mapping them to the current system */ @@ -66,7 +68,7 @@ class GroupImporter extends BaseImporter $roleSvc = app(RoleService::class); $count = 0; - foreach ($this->db->readRows('groups') as $row) { + foreach ($this->db->readRows($this->table, $start) as $row) { $name = str_slug($row->name); $role = Role::firstOrCreate( ['name' => $name], diff --git a/modules/Installer/Services/Importer/Importers/PirepImporter.php b/modules/Installer/Services/Importer/Importers/PirepImporter.php index 502178b3..e9265355 100644 --- a/modules/Installer/Services/Importer/Importers/PirepImporter.php +++ b/modules/Installer/Services/Importer/Importers/PirepImporter.php @@ -10,12 +10,33 @@ use Modules\Installer\Services\Importer\BaseImporter; class PirepImporter extends BaseImporter { + protected $table = 'pireps'; + public function run($start = 0) { $this->comment('--- PIREP IMPORT ---'); + $fields = [ + 'pirepid', + 'pilotid', + 'code', + 'aircraft', + 'flightnum', + 'depicao', + 'arricao', + 'fuelused', + 'route', + 'source', + 'accepted', + 'submitdate', + 'distance', + 'flighttime_stamp', + 'flighttype', + 'flightlevel', + ]; + $count = 0; - foreach ($this->db->readRows('pireps', $start) as $row) { + foreach ($this->db->readRows($this->table, $start, $fields) as $row) { $pirep_id = $row->pirepid; $user_id = $this->idMapper->getMapping('users', $row->pilotid); $airline_id = $this->idMapper->getMapping('airlines', $row->code); @@ -70,7 +91,10 @@ class PirepImporter extends BaseImporter $attrs['level'] = 0; } - $pirep = Pirep::updateOrCreate(['id' => $pirep_id], $attrs); + $w = ['id' => $pirep_id]; + + $pirep = Pirep::updateOrCreate($w, $attrs); + //$pirep = Pirep::create(array_merge($w, $attrs)); //Log::debug('pirep oldid='.$pirep_id.', olduserid='.$row->pilotid // .'; new id='.$pirep->id.', user id='.$user_id); diff --git a/modules/Installer/Services/Importer/Importers/RankImport.php b/modules/Installer/Services/Importer/Importers/RankImport.php index 42ea72f3..aba00cb7 100644 --- a/modules/Installer/Services/Importer/Importers/RankImport.php +++ b/modules/Installer/Services/Importer/Importers/RankImport.php @@ -7,12 +7,14 @@ use Modules\Installer\Services\Importer\BaseImporter; class RankImport extends BaseImporter { + protected $table = 'ranks'; + public function run($start = 0) { $this->comment('--- RANK IMPORT ---'); $count = 0; - foreach ($this->db->readRows('ranks') as $row) { + foreach ($this->db->readRows($this->table, $start) as $row) { $rank = Rank::firstOrCreate(['name' => $row->rank], [ 'image_url' => $row->rankimage, 'hours' => $row->minhours, diff --git a/modules/Installer/Services/Importer/Importers/UserImport.php b/modules/Installer/Services/Importer/Importers/UserImport.php index 30077c55..968b3be4 100644 --- a/modules/Installer/Services/Importer/Importers/UserImport.php +++ b/modules/Installer/Services/Importer/Importers/UserImport.php @@ -13,6 +13,8 @@ use Modules\Installer\Services\Importer\BaseImporter; class UserImport extends BaseImporter { + protected $table = 'pilots'; + /** * @var UserService */ @@ -26,7 +28,7 @@ class UserImport extends BaseImporter $count = 0; $first_row = true; - foreach ($this->db->readRows('pilots', $start) as $row) { + foreach ($this->db->readRows($this->table, $start) as $row) { $pilot_id = $row->pilotid; // This isn't their actual ID $name = $row->firstname.' '.$row->lastname; diff --git a/modules/Installer/Services/Importer/Stages/Stage2.php b/modules/Installer/Services/Importer/Stages/Stage2.php deleted file mode 100644 index 198e9205..00000000 --- a/modules/Installer/Services/Importer/Stages/Stage2.php +++ /dev/null @@ -1,15 +0,0 @@ -dsn); - $this->batchSize = config('installer.importer.batch_size'); + $this->batchSize = config('installer.importer.batch_size', 20); + } + + public function __destruct() + { + $this->close(); } public function connect() @@ -43,7 +57,15 @@ class ImporterDB $this->conn->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ); } catch (PDOException $e) { Log::error($e); - exit(); + + throw $e; + } + } + + public function close() + { + if ($this->conn) { + $this->conn = null; } } @@ -85,13 +107,11 @@ class ImporterDB * * @param string $table The name of the table * @param int [$start_offset] - * - * @throws \Modules\Installer\Exceptions\ImporterNextRecordSet - * @throws \Modules\Installer\Exceptions\ImporterNoMoreRecords + * @param string [$fields] * * @return \Generator */ - public function readRows($table, $start_offset = 0) + public function readRows($table, $start_offset = 0, $fields = '*') { $this->connect(); @@ -104,8 +124,8 @@ class ImporterDB $rows_to_read = $total_rows; } - Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows); - yield from $this->readRowsOffset($table, $this->batchSize, $offset); + // Log::info('Reading '.$offset.' to '.$rows_to_read.' of '.$total_rows); + yield from $this->readRowsOffset($table, $this->batchSize, $offset, $fields); $offset += $this->batchSize; } @@ -115,38 +135,27 @@ class ImporterDB * @param string $table * @param int $limit Number of rows to read * @param int $offset Where to start from - * - * @throws ImporterNextRecordSet - * @throws ImporterNoMoreRecords + * @param string [$fields] * * @return \Generator */ - public function readRowsOffset($table, $limit, $offset) + public function readRowsOffset($table, $limit, $offset, $fields = '*') { - $sql = 'SELECT * FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset; + if (is_array($fields)) { + $fields = implode(',', $fields); + } + + $sql = 'SELECT '.$fields.' FROM '.$this->tableName($table).' LIMIT '.$limit.' OFFSET '.$offset; try { $result = $this->conn->query($sql); - if (!$result) { - throw new ImporterNoMoreRecords(); - } - - $rowCount = $result->rowCount(); - - if (!$rowCount === 0) { - throw new ImporterNoMoreRecords(); + if (!$result || $result->rowCount() === 0) { + return; } foreach ($result as $row) { yield $row; } - - // No more records left since we got the number below the limit - if ($rowCount < $limit) { - throw new ImporterNoMoreRecords(); - } - - throw new ImporterNextRecordSet($offset + $limit); } catch (PDOException $e) { // Without incrementing the offset, it should re-run the same query Log::error('Error readRowsOffset: '.$e->getMessage());