Fixed #6634: Asset Import History fixes and optimizations (#6657)

* Starting work on asset history importer.

* Starting work on asset history importer.

* Added checkin target.

* Last change... importing history should also probably be an admin only task.

* Added caching for user and asset queries.

* Updated cache keepalive time to DateTimeInterface

* Updated cache keepalive time to DateTimeInterface
This commit is contained in:
herroworrd 2019-02-08 16:05:56 -08:00 committed by snipe
parent 5624ea14e7
commit 3f7d2aebc7
3 changed files with 162 additions and 240 deletions

View file

@ -22,6 +22,8 @@ use Image;
use Input; use Input;
use Lang; use Lang;
use League\Csv\Reader; use League\Csv\Reader;
use League\Csv\Statement;
use Illuminate\Support\Facades\Cache;
use Log; use Log;
use Mail; use Mail;
use Paginator; use Paginator;
@ -516,7 +518,7 @@ class AssetsController extends Controller
*/ */
public function getImportHistory() public function getImportHistory()
{ {
$this->authorize('checkout', Asset::class); $this->authorize('admin');
return view('hardware/history'); return view('hardware/history');
} }
@ -526,8 +528,8 @@ class AssetsController extends Controller
* This needs a LOT of love. It's done very inelegantly right now, and there are * This needs a LOT of love. It's done very inelegantly right now, and there are
* a ton of optimizations that could (and should) be done. * a ton of optimizations that could (and should) be done.
* *
* @author [A. Gianotto] [<snipe@snipe.net>] * @author [herroworrd]
* @since [v3.3] * @since [v5.0]
* @return View * @return View
*/ */
public function postImportHistory(Request $request) public function postImportHistory(Request $request)
@ -536,128 +538,103 @@ class AssetsController extends Controller
ini_set("auto_detect_line_endings", '1'); ini_set("auto_detect_line_endings", '1');
} }
$requiredcolumns = ['Asset Tag', 'Checkout Date', 'Checkin Date', 'Full Name'];
$csv = Reader::createFromPath(Input::file('user_import_csv')); $csv = Reader::createFromPath(Input::file('user_import_csv'));
$csv->setNewline("\r\n");
//get the first row, usually the CSV header
$csv->setHeaderOffset(0); $csv->setHeaderOffset(0);
$results = $csv->getRecords();
$item = array(); //Stop here if we don't have the columns we need
if(count(array_intersect($requiredcolumns, $csv->getHeader())) != count($requiredcolumns)) {
$status['error'][]['csv'][]['msg'] = 'Headers do not match';
return view('hardware/history')->with('status', $status);
}
$statement = (new Statement())
->orderBy(\Closure::fromCallable([$this, 'sortByName']));
$results = $statement->process($csv);
$status = array(); $status = array();
$status['error'] = array(); $status['error'] = array();
$status['success'] = array(); $status['success'] = array();
$base_username = null;
$cachetime = Carbon::now()->addSeconds(120);
foreach ($results as $record) {
foreach ($results as $row) { $asset_tag = $record['Asset Tag'];
if (is_array($row)) {
$row = array_change_key_case($row, CASE_LOWER);
$asset_tag = Helper::array_smart_fetch($row, "asset tag");
if (!array_key_exists($asset_tag, $item)) {
$item[$asset_tag] = array();
}
$batch_counter = count($item[$asset_tag]);
$item[$asset_tag][$batch_counter]['checkout_date'] = Carbon::parse(Helper::array_smart_fetch($row, "date"))->format('Y-m-d H:i:s'); try {
$checkoutdate = Carbon::parse($record['Checkout Date'])->format('Y-m-d H:i:s');
$item[$asset_tag][$batch_counter]['asset_tag'] = Helper::array_smart_fetch($row, "asset tag"); $checkindate = Carbon::parse($record['Checkin Date'])->format('Y-m-d H:i:s');
$item[$asset_tag][$batch_counter]['name'] = Helper::array_smart_fetch($row, "name"); }
$item[$asset_tag][$batch_counter]['email'] = Helper::array_smart_fetch($row, "email"); catch (\Exception $err) {
$status['error'][]['asset'][$asset_tag]['msg'] = 'Your dates are screwed up. Format needs to be Y-m-d H:i:s';
if ($asset = Asset::where('asset_tag', '=', $asset_tag)->first()) { continue;
$item[$asset_tag][$batch_counter]['asset_id'] = $asset->id;
$base_username = User::generateFormattedNameFromFullName($item[$asset_tag][$batch_counter]['name'], Setting::getSettings()->username_format);
$user = User::where('username', '=', $base_username['username']);
$user_query = ' on username '.$base_username['username'];
if ($request->input('match_firstnamelastname')=='1') {
$firstnamedotlastname = User::generateFormattedNameFromFullName($item[$asset_tag][$batch_counter]['name'], 'firstname.lastname');
$item[$asset_tag][$batch_counter]['username'][] = $firstnamedotlastname['username'];
$user->orWhere('username', '=', $firstnamedotlastname['username']);
$user_query .= ', or on username '.$firstnamedotlastname['username'];
}
if ($request->input('match_flastname')=='1') {
$flastname = User::generateFormattedNameFromFullName( $item[$asset_tag][$batch_counter]['name'], 'filastname');
$item[$asset_tag][$batch_counter]['username'][] = $flastname['username'];
$user->orWhere('username', '=', $flastname['username']);
$user_query .= ', or on username '.$flastname['username'];
}
if ($request->input('match_firstname')=='1') {
$firstname = User::generateFormattedNameFromFullName( $item[$asset_tag][$batch_counter]['name'], 'firstname');
$item[$asset_tag][$batch_counter]['username'][] = $firstname['username'];
$user->orWhere('username', '=', $firstname['username']);
$user_query .= ', or on username '.$firstname['username'];
}
if ($request->input('match_email')=='1') {
if ($item[$asset_tag][$batch_counter]['email']=='') {
$item[$asset_tag][$batch_counter]['username'][] = $user_email = User::generateEmailFromFullName($item[$asset_tag][$batch_counter]['name']);
$user->orWhere('username', '=', $user_email);
$user_query .= ', or on username '.$user_email;
}
}
// A matching user was found
if ($user = $user->first()) {
$item[$asset_tag][$batch_counter]['checkedout_to'] = $user->id;
$item[$asset_tag][$batch_counter]['user_id'] = $user->id;
Actionlog::firstOrCreate(array(
'item_id' => $asset->id,
'item_type' => Asset::class,
'user_id' => Auth::user()->id,
'note' => 'Checkout imported by '.Auth::user()->present()->fullName().' from history importer',
'target_id' => $item[$asset_tag][$batch_counter]['user_id'],
'target_type' => User::class,
'created_at' => $item[$asset_tag][$batch_counter]['checkout_date'],
'action_type' => 'checkout',
));
$asset->assigned_to = $user->id;
if ($asset->save()) {
$status['success'][]['asset'][$asset_tag]['msg'] = 'Asset successfully matched for '.Helper::array_smart_fetch($row, "name").$user_query.' on '.$item[$asset_tag][$batch_counter]['checkout_date'];
} else {
$status['error'][]['asset'][$asset_tag]['msg'] = 'Asset and user was matched but could not be saved.';
}
} else {
$item[$asset_tag][$batch_counter]['checkedout_to'] = null;
$status['error'][]['user'][Helper::array_smart_fetch($row, "name")]['msg'] = 'User does not exist so no checkin log was created.';
}
} else {
$item[$asset_tag][$batch_counter]['asset_id'] = null;
$status['error'][]['asset'][$asset_tag]['msg'] = 'Asset does not exist so no match was attempted.';
}
} }
}
// Loop through and backfill the checkins if($asset = Cache::remember('asset:' . $asset_tag, $cachetime, function () use( &$asset_tag) {
foreach ($item as $key => $asset_batch) { $tocache = Asset::where('asset_tag', '=', $asset_tag)->value('id');
$total_in_batch = count($asset_batch); return is_null($tocache) ? false : $tocache;}))
for ($x = 0; $x < $total_in_batch; $x++) { {
$next = $x + 1; //we've found our asset, now lets look for a user
if($base_username != User::generateFormattedNameFromFullName($record['Full Name'], Setting::getSettings()->username_format)) {
// Only do this if a matching user was found $base_username = User::generateFormattedNameFromFullName($record['Full Name'], Setting::getSettings()->username_format);
if ((array_key_exists('checkedout_to', $asset_batch[$x])) && ($asset_batch[$x]['checkedout_to']!='')) {
if (($total_in_batch > 1) && ($x < $total_in_batch) && (array_key_exists($next, $asset_batch))) { if(!$user = Cache::remember('user:' . $base_username['username'], $cachetime, function () use( &$base_username) {
$checkin_date = Carbon::parse($asset_batch[$next]['checkout_date'])->subDay(1)->format('Y-m-d H:i:s'); $tocache = User::where('username', '=', $base_username['username'])->value('id');
$asset_batch[$x]['real_checkin'] = $checkin_date; return is_null($tocache) ? false : $tocache;}))
{
$status['error'][]['asset'][$asset_tag]['msg'] = 'Asset was found but user (' . $record['Full Name'] . ') not matched';
$base_username = null;
continue;
}
}
if($checkoutdate < $checkindate) {
Actionlog::firstOrCreate(array( Actionlog::firstOrCreate(array(
'item_id' => $asset_batch[$x]['asset_id'], 'item_id' => $asset,
'item_type' => Asset::class, 'item_type' => Asset::class,
'user_id' => Auth::user()->id, 'user_id' => Auth::user()->id,
'note' => 'Checkin imported by ' . Auth::user()->present()->fullName() . ' from history importer', 'note' => 'Historical record added by ' . Auth::user()->present()->fullName(),
'target_id' => null, 'target_id' => $user,
'created_at' => $checkin_date, 'target_type' => User::class,
'created_at' => $checkoutdate,
'action_type' => 'checkout',
));
Actionlog::firstOrCreate(array(
'item_id' => $asset,
'item_type' => Asset::class,
'user_id' => Auth::user()->id,
'note' => 'Historical record added by ' . Auth::user()->present()->fullName(),
'target_id' => $user,
'target_type' => User::class,
'created_at' => $checkindate,
'action_type' => 'checkin' 'action_type' => 'checkin'
)); ));
}
$status['success'][]['asset'][$asset_tag]['msg'] = 'Asset successfully matched for ' . $record['Full Name'] . ' on ' . $checkoutdate;
}
else {
$status['error'][]['asset'][$asset_tag]['msg'] = 'Checkin date needs to be after checkout date.';
} }
} }
else {
$status['error'][]['asset'][$asset_tag]['msg'] = 'Asset not found in Snipe';
}
} }
return view('hardware/history')->with('status', $status); return view('hardware/history')->with('status', $status);
} }
protected function sortByName(array $recordA, array $recordB): int
{
return strcmp($recordB['Full Name'], $recordA['Full Name']);
}
/** /**
* Retore a deleted asset. * Retore a deleted asset.
* *

View file

@ -49,171 +49,114 @@
<div class="box box-default"> <div class="box box-default">
<div class="box-body"> <div class="box-body">
<div class="col-md-12"> <div class="col-md-12">
<form class="form-horizontal" role="form" method="post" enctype="multipart/form-data" action=""> <form class="form-horizontal" role="form" method="post" enctype="multipart/form-data" action="">
<!-- CSRF Token --> <!-- CSRF Token -->
<input type="hidden" name="_token" value="{{ csrf_token() }}" /> <input type="hidden" name="_token" value="{{ csrf_token() }}" />
@if (Session::get('message')) @if (Session::get('message'))
<p class="alert-danger"> <p class="alert-danger">
You have an error in your CSV file:<br /> You have an error in your CSV file:<br />
{{ Session::get('message') }} {{ Session::get('message') }}
</p>
@endif
<p>
Use this tool to import asset history you may have in CSV format. This information likely will be extracted from your previous asset management system.
</p>
<p>
<i>Asset history</i> is defined as a checkout and subsequent checkin that has happened in the past. The assets and users MUST already exist in SNIPE, or they will be skipped. Matching assets for history import happens against the asset tag. We will try to find a matching user based on the user's full name you provide, based on the username format you configured in the Admin &gt; General Settings.
</p> </p>
@endif
<p> <p>
Upload a CSV that contains asset history. The assets and users MUST already exist in the system, or they will be skipped. Matching assets for history import happens against the asset tag. We will try to find a matching user based on the user's name you provide, and the criteria you select below. If you do not select any criteria below, it will simply try to match on the username format you configured in the Admin &gt; General Settings. Fields included in the CSV must match <i>exactly</i> these header values: <strong>Asset Tag, Checkout Date, Checkin Date, Full Name</strong>. Any additional fields will be ignored.
</p> </p>
<p>Fields included in the CSV must match the headers: <strong>Date, Tag, Name</strong>. Any additional fields will be ignored. </p> <div class="form-group">
<label for="first_name" class="col-sm-3 control-label">{{ trans('admin/users/general.usercsv') }}</label>
<p><strong>Date</strong> should be the checkout date. <strong>Tag</strong> should be the asset tag. <strong>Name</strong> should be the user's name (firstname lastname).</p> <div class="col-sm-9">
<input type="file" name="user_import_csv" id="user_import_csv"{{ (config('app.lock_passwords')===true) ? ' disabled' : '' }}>
<p><strong>History should be ordered by date in ascending order.</strong></p> </div>
<div class="form-group">
<label for="first_name" class="col-sm-3 control-label">{{ trans('admin/users/general.usercsv') }}</label>
<div class="col-sm-9">
<input type="file" name="user_import_csv" id="user_import_csv"{{ (config('app.lock_passwords')===true) ? ' disabled' : '' }}>
</div> </div>
</div>
<!-- Form Actions -->
<div class="box-footer text-right">
@if (config('app.lock_passwords')===true)
<div class="col-md-12">
<div class="callout callout-info">
{{ trans('general.feature_disabled') }}
</div>
</div>
@else
<!-- Match firstname.lastname --> <button type="submit" class="btn btn-default">{{ trans('button.submit') }}</button>
<div class="form-group"> @endif
<div class="col-sm-2">
</div>
<div class="col-sm-10">
{{ Form::checkbox('match_firstnamelastname', '1', Input::old('match_firstnamelastname')) }} Try to match users by firstname.lastname (jane.smith) format
</div>
</div>
<!-- Match flastname -->
<div class="form-group">
<div class="col-sm-2">
</div>
<div class="col-sm-10">
{{ Form::checkbox('match_flastname', '1', Input::old('match_flastname')) }} Try to match users by first initial last name (jsmith) format
</div>
</div>
<!-- Match firstname -->
<div class="form-group">
<div class="col-sm-2">
</div>
<div class="col-sm-10">
{{ Form::checkbox('match_firstname', '1', Input::old('match_firstname')) }} Try to match users by first name (jane) format
</div>
</div>
<!-- Match email -->
<div class="form-group">
<div class="col-sm-2">
</div>
<div class="col-sm-10">
{{ Form::checkbox('match_email', '1', Input::old('match_email')) }} Try to match users by email as username
</div>
</div>
</div>
</div>
<!-- Form Actions -->
<div class="box-footer text-right">
@if (config('app.lock_passwords')===true)
<div class="col-md-12">
<div class="callout callout-info">
{{ trans('general.feature_disabled') }}
</div>
</div>
@else
<button type="submit" class="btn btn-default">{{ trans('button.submit') }}</button>
@endif
</div>
</form>
</div>
@if (isset($status))
@if (count($status['error']) > 0)
<div class="row">
<div class="col-md-12">
<div class="box box-default">
<div class="box-header with-border">
<h3 class="box-title"> {{ count($status['error']) }} Error Messages </h3>
</div> </div>
<div class="box-body">
<div style="height : 400px; overflow : auto;">
<table class="table">
@for ($x = 0; $x < count($status['error']); $x++)
@foreach($status['error'][$x] as $object_type => $message)
<tr class="danger">
<td><strong>{{ ucwords($object_type) }} {{ key($message) }}:</strong></td>
<td>{{ $message[key($message)]['msg'] }}</td>
</tr>
@endforeach
@endfor
</table>
</div>
</div>
</div>
</div>
</div>
@endif </form>
@if (count($status['success']) > 0)
@if (isset($status))
@if (count($status['error']) > 0)
<div class="row"> <div class="row">
<div class="col-md-12"> <div class="col-md-12">
<div class="box box-default"> <div class="box box-default">
<div class="box-header with-border"> <div class="box-header with-border">
<h3 class="box-title"> {{ count($status['success']) }} Success Messages </h3> <h3 class="box-title"> {{ count($status['error']) }} Error Messages </h3>
</div> </div>
<div class="box-body"> <div class="box-body">
<div style="height : 400px; overflow : auto;"> <div style="height : 400px; overflow : auto;">
<table class="table"> <table class="table">
@for ($x = 0; $x < count($status['success']); $x++) @for ($x = 0; $x < count($status['error']); $x++)
@foreach($status['success'][$x] as $object_type => $message) @foreach($status['error'][$x] as $object_type => $message)
<tr class="success"> <tr class="danger">
<td><strong>{{ ucwords($object_type) }} {{ key($message) }}:</strong></td> <td><strong>{{ ucwords($object_type) }} {{ key($message) }}:</strong></td>
<td>{{ $message[key($message)]['msg'] }}</td> <td>{{ $message[key($message)]['msg'] }}</td>
</tr> </tr>
@endforeach @endforeach
@endfor @endfor
</table> </table>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
@endif
@endif
</div></div></div> @endif
<script nonce="{{ csrf_token() }}">
$(document).ready(function(){
$('#generate-password').pGenerator({ @if (count($status['success']) > 0)
'bind': 'click', <div class="row">
'passwordElement': '#password', <div class="col-md-12">
'displayElement': '#password-display', <div class="box box-default">
'passwordLength': 10, <div class="box-header with-border">
'uppercase': true, <h3 class="box-title"> {{ count($status['success']) }} Success Messages </h3>
'lowercase': true, </div>
'numbers': true, <div class="box-body">
'specialChars': false, <div style="height : 400px; overflow : auto;">
<table class="table">
@for ($x = 0; $x < count($status['success']); $x++)
@foreach($status['success'][$x] as $object_type => $message)
<tr class="success">
<td><strong>{{ ucwords($object_type) }} {{ key($message) }}:</strong></td>
<td>{{ $message[key($message)]['msg'] }}</td>
</tr>
@endforeach
@endfor
</table>
</div>
</div>
</div>
</div>
</div>
@endif
@endif
}); </div>
}); </div>
</div>
</div>
</div>
</script> @stop
@stop

View file

@ -442,6 +442,8 @@
{{ trans('general.asset_maintenances') }} {{ trans('general.asset_maintenances') }}
</a> </a>
</li> </li>
@endcan
@can('admin')
<li> <li>
<a href="{{ url('hardware/history') }}"> <a href="{{ url('hardware/history') }}">
{{ trans('general.import-history') }} {{ trans('general.import-history') }}