DRCIT/snipe-it
1
0
Fork 0
mirror of https://github.com/snipe/snipe-it.git synced 2024-11-14 01:24:06 -08:00
Code Issues Actions 15 Packages Projects Releases Wiki Activity

Improved import performance

Browse source
This commit is contained in:
bryanlopezinc 2024-10-10 23:32:07 +01:00
parent 3ee5713740
commit 524a442724
2 changed files with 28 additions and 54 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
app/Console/Commands/ObjectImportCommand.php
View file

@ -6,6 +6,7 @@ use Illuminate\Console\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use Illuminate\Support\Facades\Log;
use Symfony\Component\Console\Helper\ProgressIndicator;
ini_set('max_execution_time', env('IMPORT_TIME_LIMIT', 600)); //600 seconds = 10 minutes
ini_set('memory_limit', env('IMPORT_MEMORY_LIMIT', '500M'));
@ -29,6 +30,11 @@ class ObjectImportCommand extends Command
*/
protected $description = 'Import Items from CSV';
/**
* The progress indicator instance.
*/
protected ProgressIndicator $progressIndicator;
/**
* Create a new command instance.
*
@ -39,8 +45,6 @@ class ObjectImportCommand extends Command
parent::__construct();
}
private $bar;
/**
* Execute the console command.
*
@ -48,6 +52,8 @@ class ObjectImportCommand extends Command
*/
public function handle()
{
$this->progressIndicator = new ProgressIndicator($this->output);
$filename = $this->argument('filename');
$class = title_case($this->option('item-type'));
$classString = "App\\Importer\\{$class}Importer";
@ -61,46 +67,25 @@ class ObjectImportCommand extends Command
// This $logFile/useFiles() bit is currently broken, so commenting it out for now
// $logFile = $this->option('logfile');
// Log::useFiles($logFile);
$this->comment('======= Importing Items from '.$filename.' =========');
$this->progressIndicator->start('======= Importing Items from '.$filename.' =========');
$importer->import();
$this->bar = null;
if (! empty($this->errors)) {
$this->comment('The following Errors were encountered.');
foreach ($this->errors as $asset => $error) {
$this->comment('Error: Item: '.$asset.' failed validation: '.json_encode($error));
}
} else {
$this->comment('All Items imported successfully!');
}
$this->comment('');
$this->progressIndicator->finish('Import finished.');
}
public function errorCallback($item, $field, $errorString)
public function errorCallback($item, $field, $error)
{
$this->errors[$item->name][$field] = $errorString;
$this->output->write("\x0D\x1B[2K");
$this->warn('Error: Item: '.$item->name.' failed validation: '.json_encode($error));
}
public function progress($count)
public function progress($importedItemsCount)
{
if (! $this->bar) {
$this->bar = $this->output->createProgressBar($count);
}
static $index = 0;
$index++;
if ($index < $count) {
$this->bar->advance();
} else {
$this->bar->finish();
}
$this->progressIndicator->advance();
}
// Tracks the current item for error messages
private $updating;
// An array of errors encountered while parsing
private $errors;
/**
* Log a message to file, configurable by the --log-file parameter.
* If a warning message is passed, we'll spit it to the console as well.

33
app/Importer/Importer.php
View file

@ -21,7 +21,6 @@ abstract class Importer
* Id of User performing import
* @var
*/
protected $created_by;
/**
* Are we updating items in the import
@ -149,17 +148,23 @@ abstract class Importer
{
$headerRow = $this->csv->fetchOne();
$this->csv->setHeaderOffset(0); //explicitly sets the CSV document header record
$results = $this->normalizeInputArray($this->csv->getRecords($headerRow));
$this->populateCustomFields($headerRow);
DB::transaction(function () use (&$results) {
DB::transaction(function () use ($headerRow) {
$importedItemsCount = 0;
Model::unguard();
$resultsCount = count($results);
foreach ($results as $row) {
foreach ($this->csv->getRecords($headerRow) as $row) {
//Lowercase header values to ensure we're comparing values properly.
$row = array_change_key_case($row, CASE_LOWER);
$this->handle($row);
$importedItemsCount++;
if ($this->progressCallback) {
call_user_func($this->progressCallback, $resultsCount);
call_user_func($this->progressCallback, $importedItemsCount);
}
$this->log('------------- Action Summary ----------------');
@ -237,22 +242,6 @@ abstract class Importer
return $key;
}
/**
* Used to lowercase header values to ensure we're comparing values properly.
*
* @param $results
* @return array
*/
public function normalizeInputArray($results)
{
$newArray = [];
foreach ($results as $index => $arrayToNormalize) {
$newArray[$index] = array_change_key_case($arrayToNormalize);
}
return $newArray;
}
/**
* Figure out the fieldname of the custom field
*