FGFS-4.1/phpvms
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix rowmapper generator, check for fields, map users #443 (#583)

Browse Source 
* Fix rowmapper generator, check for fields, map users #443

* Formatting

* Remove value store at the end of the import

* Update the notes for the importer about users

* Uncomment importers disabled during testing
This commit is contained in:
Nabeel S
2020-02-23 19:48:28 -05:00
committed by GitHub
parent a1d6fa17ad
commit dfbaa1afd3
17 changed files with 214 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
modules/Importer/Services/BaseImporter.php
View File

@@ -8,6 +8,7 @@ use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Support\Facades\Log;
use Modules\Importer\Utils\IdMapper;
use Modules\Importer\Utils\ImporterDB;
@@ -27,7 +28,7 @@ abstract class BaseImporter implements ShouldQueue
/**
* The mapper class used for old IDs to new IDs
*
* @var \Illuminate\Contracts\Foundation\Application|mixed
* @var IdMapper
*/
protected $idMapper;
@@ -38,6 +39,13 @@ abstract class BaseImporter implements ShouldQueue
*/
protected $table;
/**
* The column used for the ID, used for the ORDER BY
*
* @var string
*/
protected $idField = 'id';
public function __construct()
{
$importerService = app(ImporterService::class);
@@ -66,6 +74,8 @@ abstract class BaseImporter implements ShouldQueue
$start = 0;
$total_rows = $this->db->getTotalRows($this->table);
Log::info('Found '.$total_rows.' rows for '.$this->table);
do {
$end = $start + $this->db->batchSize;
if ($end > $total_rows) {

8
modules/Importer/Services/ImporterService.php
View File

@@ -6,6 +6,7 @@ use App\Contracts\Service;
use App\Repositories\KvpRepository;
use Exception;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Log;
use Modules\Importer\Services\Importers\AircraftImporter;
use Modules\Importer\Services\Importers\AirlineImporter;
use Modules\Importer\Services\Importers\AirportImporter;
@@ -130,6 +131,11 @@ class ImporterService extends Service
/** @var $importerInst \Modules\Importer\Services\BaseImporter */
$importerInst = new $importer();
$importerInst->run($start);
try {
$importerInst->run($start);
} catch (Exception $e) {
Log::error('Error running importer: '.$e->getMessage());
}
}
}

5
modules/Importer/Services/Importers/AircraftImporter.php
View File

@@ -9,7 +9,7 @@ use Modules\Importer\Services\BaseImporter;
class AircraftImporter extends BaseImporter
{
public $table = 'aircraft';
protected $table = 'aircraft';
/**
* CONSTANTS
@@ -25,7 +25,8 @@ class AircraftImporter extends BaseImporter
$this->info('Subfleet ID is '.$subfleet->id);
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$where = [
'name' => $row->fullname,
'registration' => $row->registration,

3
modules/Importer/Services/Importers/AirlineImporter.php
View File

@@ -18,7 +18,8 @@ class AirlineImporter extends BaseImporter
$this->comment('--- AIRLINE IMPORT ---');
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$attrs = [
'iata' => $row->code,
'name' => $row->name,

3
modules/Importer/Services/Importers/AirportImporter.php
View File

@@ -25,7 +25,8 @@ class AirportImporter extends BaseImporter
];
$count = 0;
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
foreach ($rows as $row) {
$attrs = [
'id' => trim($row->icao),
'icao' => trim($row->icao),

18
modules/Importer/Services/Importers/ClearDatabase.php
View File

@@ -3,6 +3,7 @@
namespace Modules\Importer\Services\Importers;
use App\Models\Acars;
use App\Models\Aircraft;
use App\Models\Airline;
use App\Models\Airport;
use App\Models\Bid;
@@ -15,6 +16,7 @@ use App\Models\Journal;
use App\Models\JournalTransaction;
use App\Models\News;
use App\Models\Pirep;
use App\Models\Role;
use App\Models\Subfleet;
use App\Models\User;
use App\Models\UserAward;
@@ -68,12 +70,7 @@ class ClearDatabase extends BaseImporter
FlightFieldValue::truncate();
Flight::truncate();
Subfleet::truncate();
// Clear permissions
// DB::table('permission_role')->truncate();
// DB::table('permission_user')->truncate();
// DB::table('role_user')->truncate();
// Role::truncate();
Aircraft::truncate();
Airline::truncate();
Airport::truncate();
@@ -83,6 +80,15 @@ class ClearDatabase extends BaseImporter
UserAward::truncate();
User::truncate();
// Clear permissions
DB::table('permission_role')->truncate();
DB::table('permission_user')->truncate();
DB::table('role_user')->truncate();
// Role::truncate();
DB::statement('SET FOREIGN_KEY_CHECKS=1');
$this->idMapper->clear();
}
}

10
modules/Importer/Services/Importers/FinalizeImporter.php
View File

@@ -34,6 +34,7 @@ class FinalizeImporter extends BaseImporter
{
$this->findLastPireps();
$this->recalculateUserStats();
$this->clearValueStore();
}
/**
@@ -41,6 +42,7 @@ class FinalizeImporter extends BaseImporter
*/
protected function findLastPireps()
{
// TODO
}
/**
@@ -55,4 +57,12 @@ class FinalizeImporter extends BaseImporter
$userSvc->recalculateStats($user);
});
}
/**
* Clear the value store of any old value mappings
*/
protected function clearValueStore()
{
$this->idMapper->clear();
}
}

5
modules/Importer/Services/Importers/FlightImporter.php
View File

@@ -24,13 +24,14 @@ class FlightImporter extends BaseImporter
'flightlevel',
'deptime',
'arrtime',
'flightttime',
'flighttime',
'notes',
'enabled',
];
$count = 0;
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
foreach ($rows as $row) {
$airline_id = $this->idMapper->getMapping('airlines', $row->code);
$flight_num = trim($row->flightnum);

11
modules/Importer/Services/Importers/GroupImporter.php
View File

@@ -12,6 +12,7 @@ use Modules\Importer\Services\BaseImporter;
class GroupImporter extends BaseImporter
{
protected $table = 'groups';
protected $idField = 'groupid';
/**
* Permissions in the legacy system, mapping them to the current system
@@ -68,7 +69,15 @@ class GroupImporter extends BaseImporter
$roleSvc = app(RoleService::class);
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
// Legacy "administrator" role is now "admin", just map that 1:1
if (strtolower($row->name) === 'administrators') {
$role = Role::where('name', 'admin')->first();
$this->idMapper->addMapping('group', $row->groupid, $role->id);
continue;
}
$name = str_slug($row->name);
$role = Role::firstOrCreate(
['name' => $name],

11
modules/Importer/Services/Importers/PirepImporter.php
View File

@@ -11,6 +11,7 @@ use Modules\Importer\Services\BaseImporter;
class PirepImporter extends BaseImporter
{
protected $table = 'pireps';
protected $idField = 'pirepid';
public function run($start = 0)
{
@@ -32,11 +33,17 @@ class PirepImporter extends BaseImporter
'distance',
'flighttime_stamp',
'flighttype',
'flightlevel',
];
// See if there's a flightlevel column, export that if there is
$columns = $this->db->getColumns($this->table);
if (in_array('flightlevel', $columns)) {
$fields[] = 'flightlevel';
}
$count = 0;
foreach ($this->db->readRows($this->table, $start, $fields) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start, $fields);
foreach ($rows as $row) {
$pirep_id = $row->pirepid;
$user_id = $this->idMapper->getMapping('users', $row->pilotid);
$airline_id = $this->idMapper->getMapping('airlines', $row->code);

4
modules/Importer/Services/Importers/RankImport.php
View File

@@ -8,13 +8,15 @@ use Modules\Importer\Services\BaseImporter;
class RankImport extends BaseImporter
{
protected $table = 'ranks';
protected $idField = 'rankid';
public function run($start = 0)
{
$this->comment('--- RANK IMPORT ---');
$count = 0;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$rank = Rank::firstOrCreate(['name' => $row->rank], [
'image_url' => $row->rankimage,
'hours' => $row->minhours,

35
modules/Importer/Services/Importers/UserImport.php
View File

@@ -3,6 +3,7 @@
namespace Modules\Importer\Services\Importers;
use App\Models\Enums\UserState;
use App\Models\Role;
use App\Models\User;
use App\Services\UserService;
use App\Support\Units\Time;
@@ -15,6 +16,7 @@ use Modules\Importer\Services\BaseImporter;
class UserImport extends BaseImporter
{
protected $table = 'pilots';
protected $idField = 'pilotid';
/**
* @var UserService
@@ -29,7 +31,8 @@ class UserImport extends BaseImporter
$count = 0;
$first_row = true;
foreach ($this->db->readRows($this->table, $start) as $row) {
$rows = $this->db->readRows($this->table, $this->idField, $start);
foreach ($rows as $row) {
$pilot_id = $row->pilotid; // This isn't their actual ID
$name = $row->firstname.' '.$row->lastname;
@@ -49,8 +52,13 @@ class UserImport extends BaseImporter
// Look for a user with that pilot ID already. If someone has it
if ($this->userSvc->isPilotIdAlreadyUsed($pilot_id)) {
Log::info('User with pilot id '.$pilot_id.' exists, reassigning');
$pilot_id = $this->userSvc->getNextAvailablePilotId();
Log::info('User with pilot id '.$pilot_id.' exists');
// Is this the same user? If not, get a new pilot ID
$user_exist = User::where('pilot_id', $pilot_id)->first();
if ($user_exist->email !== $row->email) {
$pilot_id = $this->userSvc->getNextAvailablePilotId();
}
}
$attrs = [
@@ -91,9 +99,26 @@ class UserImport extends BaseImporter
{
// Be default add them to the user role, and then determine if they
// belong to any other groups, and add them to that
$this->userSvc->addUserToRole($user, 'user');
$roleMappings = [];
$newRoles = [];
// Figure out what other groups they belong to
// Figure out what other groups they belong to... read from the old table, and map
// them to the new group(s)
$old_user_groups = $this->db->findBy('groupmembers', ['pilotid' => $old_pilot_id]);
foreach ($old_user_groups as $oldGroup) {
$newRoleId = $this->idMapper->getMapping('group', $oldGroup->groupid);
// Only lookup a new role ID if found
// if (!in_array($newRoleId, $roleMappings)) {
// $roleMappings[$newRoleId] = Role::where(['id' => $newRoleId])->first();
// }
// $newRoles[] = $roleMappings[$newRoleId];
$newRoles[] = $newRoleId;
}
// Assign the groups to the new user
$user->attachRoles($newRoles);
}
/**