Fix UTF-8 issues in imports

This commit is contained in:
snipe 2016-08-29 22:57:48 -07:00
parent 64cd4fb1c9
commit e4dcd47d6c
3 changed files with 109 additions and 67 deletions

View file

@ -21,6 +21,7 @@ use Illuminate\Database\Eloquent\Model;
use League\Csv\Reader;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption;
use ForceUTF8\Encoding;
ini_set('max_execution_time', 600); //600 seconds = 10 minutes
ini_set('memory_limit', '500M');
@ -284,7 +285,7 @@ class ObjectImportCommand extends Command
*/
public function array_smart_fetch(array $array, $key, $default = '')
{
return array_key_exists($key, $array) ? e(trim($array[ $key ])) : $default;
return array_key_exists(trim($key), $array) ? e(Encoding::fixUTF8(trim($array[ $key ]))) : $default;
}
@ -319,10 +320,12 @@ class ObjectImportCommand extends Command
public function createOrFetchAssetModel(array $row, $category, $manufacturer)
{
$this->log(print_r($row));
$asset_model_name = $this->array_smart_fetch($row, "model name");
$this->log('Raw Model Name: '.$asset_model_name);
$asset_modelno = $this->array_smart_fetch($row, "model number");
if (empty($asset_model_name)) {
$asset_model_name='Unknown';
$asset_model_name ='Unknown';
}
if (empty($asset_modelno)) {
$asset_modelno='';
@ -355,6 +358,7 @@ class ObjectImportCommand extends Command
return $asset_model;
} else {
$this->jsonError('Asset Model "' . $asset_model_name . '"', $asset_model->getErrors());
$this->log('Asset Model "' . $asset_model_name . '"', $asset_model->getErrors());
return $asset_model;
}
} else {

View file

@ -22,7 +22,8 @@
"doctrine/dbal": "v2.4.2",
"barryvdh/laravel-debugbar": "^2.1",
"spatie/laravel-backup": "3.8.1",
"misterphilip/maintenance-mode": "1.0.*"
"misterphilip/maintenance-mode": "1.0.*",
"neitanod/forceutf8": "dev-master"
},
"require-dev": {
"fzaninotto/faker": "~1.4",

165
composer.lock generated
View file

@ -4,21 +4,21 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"hash": "a770010d0ebb93a2e1b5f4acadd5a7f3",
"content-hash": "59772418a4612685eea10dde38ca77b7",
"hash": "ed9f8700f2dcd943ff662a82e4d8314f",
"content-hash": "9c0251ddc1a110d83a762483abeea079",
"packages": [
{
"name": "aws/aws-sdk-php",
"version": "3.18.39",
"version": "3.19.2",
"source": {
"type": "git",
"url": "https://github.com/aws/aws-sdk-php.git",
"reference": "85f1fddaeb40b95106b2a2764268e9c89fc258ce"
"reference": "3cb90413129da42c9d3289d542bee0ae1049892c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/85f1fddaeb40b95106b2a2764268e9c89fc258ce",
"reference": "85f1fddaeb40b95106b2a2764268e9c89fc258ce",
"url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/3cb90413129da42c9d3289d542bee0ae1049892c",
"reference": "3cb90413129da42c9d3289d542bee0ae1049892c",
"shasum": ""
},
"require": {
@ -85,7 +85,7 @@
"s3",
"sdk"
],
"time": "2016-08-11 16:40:35"
"time": "2016-08-23 20:58:48"
},
{
"name": "aws/aws-sdk-php-laravel",
@ -145,21 +145,21 @@
},
{
"name": "barryvdh/laravel-debugbar",
"version": "v2.2.2",
"version": "V2.2.3",
"source": {
"type": "git",
"url": "https://github.com/barryvdh/laravel-debugbar.git",
"reference": "c291e58d0a13953e0f68d99182ee77ebc693edc0"
"reference": "ecd1ce5c4a827e2f6a8fb41bcf67713beb1c1cbd"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/c291e58d0a13953e0f68d99182ee77ebc693edc0",
"reference": "c291e58d0a13953e0f68d99182ee77ebc693edc0",
"url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/ecd1ce5c4a827e2f6a8fb41bcf67713beb1c1cbd",
"reference": "ecd1ce5c4a827e2f6a8fb41bcf67713beb1c1cbd",
"shasum": ""
},
"require": {
"illuminate/support": "5.1.*|5.2.*",
"maximebf/debugbar": "~1.11.0",
"illuminate/support": "5.1.*|5.2.*|5.3.*",
"maximebf/debugbar": "~1.11.0|~1.12.0",
"php": ">=5.5.9",
"symfony/finder": "~2.7|~3.0"
},
@ -195,7 +195,7 @@
"profiler",
"webprofiler"
],
"time": "2016-05-11 13:54:43"
"time": "2016-07-29 15:00:36"
},
{
"name": "classpreloader/classpreloader",
@ -1018,12 +1018,12 @@
"source": {
"type": "git",
"url": "https://github.com/Intervention/image.git",
"reference": "6886d43f5babe6900c29c59640ca81401fe71c80"
"reference": "45a41a38bd1e5290cd51ab773013e6f041b2b711"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/Intervention/image/zipball/6886d43f5babe6900c29c59640ca81401fe71c80",
"reference": "6886d43f5babe6900c29c59640ca81401fe71c80",
"url": "https://api.github.com/repos/Intervention/image/zipball/45a41a38bd1e5290cd51ab773013e6f041b2b711",
"reference": "45a41a38bd1e5290cd51ab773013e6f041b2b711",
"shasum": ""
},
"require": {
@ -1072,7 +1072,7 @@
"thumbnail",
"watermark"
],
"time": "2016-06-22 08:03:11"
"time": "2016-08-19 14:41:12"
},
{
"name": "jakub-onderka/php-console-color",
@ -1221,16 +1221,16 @@
},
{
"name": "laravel/framework",
"version": "v5.2.43",
"version": "v5.2.45",
"source": {
"type": "git",
"url": "https://github.com/laravel/framework.git",
"reference": "5490b8f00564bb60839002f86828e27edd1e5610"
"reference": "2a79f920d5584ec6df7cf996d922a742d11095d1"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/laravel/framework/zipball/5490b8f00564bb60839002f86828e27edd1e5610",
"reference": "5490b8f00564bb60839002f86828e27edd1e5610",
"url": "https://api.github.com/repos/laravel/framework/zipball/2a79f920d5584ec6df7cf996d922a742d11095d1",
"reference": "2a79f920d5584ec6df7cf996d922a742d11095d1",
"shasum": ""
},
"require": {
@ -1311,7 +1311,7 @@
"pusher/pusher-php-server": "Required to use the Pusher broadcast driver (~2.0).",
"symfony/css-selector": "Required to use some of the crawler integration testing tools (2.8.*|3.0.*).",
"symfony/dom-crawler": "Required to use most of the crawler integration testing tools (2.8.*|3.0.*).",
"symfony/psr-http-message-bridge": "Required to psr7 bridging features (0.2.*)."
"symfony/psr-http-message-bridge": "Required to use psr7 bridging features (0.2.*)."
},
"type": "library",
"extra": {
@ -1347,7 +1347,7 @@
"framework",
"laravel"
],
"time": "2016-08-10 12:23:59"
"time": "2016-08-26 11:44:52"
},
{
"name": "laravelcollective/html",
@ -1595,16 +1595,16 @@
},
{
"name": "maximebf/debugbar",
"version": "v1.11.1",
"version": "v1.12.0",
"source": {
"type": "git",
"url": "https://github.com/maximebf/php-debugbar.git",
"reference": "d9302891c1f0a0ac5a4f66725163a00537c6359f"
"reference": "e634fbd32cd6bc3fa0e8c972b52d4bf49bab3988"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/d9302891c1f0a0ac5a4f66725163a00537c6359f",
"reference": "d9302891c1f0a0ac5a4f66725163a00537c6359f",
"url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/e634fbd32cd6bc3fa0e8c972b52d4bf49bab3988",
"reference": "e634fbd32cd6bc3fa0e8c972b52d4bf49bab3988",
"shasum": ""
},
"require": {
@ -1623,7 +1623,7 @@
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.11-dev"
"dev-master": "1.12-dev"
}
},
"autoload": {
@ -1652,7 +1652,7 @@
"debug",
"debugbar"
],
"time": "2016-01-22 12:22:23"
"time": "2016-05-15 13:11:34"
},
{
"name": "misterphilip/maintenance-mode",
@ -1876,6 +1876,40 @@
],
"time": "2016-01-05 18:25:05"
},
{
"name": "neitanod/forceutf8",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/neitanod/forceutf8.git",
"reference": "2c1b21e00ed16b2b083ae4e27901cb5f2856db90"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/neitanod/forceutf8/zipball/2c1b21e00ed16b2b083ae4e27901cb5f2856db90",
"reference": "2c1b21e00ed16b2b083ae4e27901cb5f2856db90",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"type": "library",
"autoload": {
"psr-0": {
"ForceUTF8\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"authors": [
{
"name": "Sebastián Grignoli",
"email": "grignoli@gmail.com"
}
],
"description": "PHP Class Encoding featuring popular Encoding::toUTF8() function --formerly known as forceUTF8()-- that fixes mixed encoded strings.",
"homepage": "https://github.com/neitanod/forceutf8",
"time": "2015-05-07 16:37:23"
},
{
"name": "nesbot/carbon",
"version": "1.21.0",
@ -3132,17 +3166,17 @@
"source": {
"type": "git",
"url": "https://github.com/tecnickcom/tc-lib-barcode.git",
"reference": "220728e5f659b935348442e8d1d3e46fd5f9e178"
"reference": "df69541618a0ebc24bc8f938e52f76a471f2e018"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/tecnickcom/tc-lib-barcode/zipball/220728e5f659b935348442e8d1d3e46fd5f9e178",
"reference": "220728e5f659b935348442e8d1d3e46fd5f9e178",
"url": "https://api.github.com/repos/tecnickcom/tc-lib-barcode/zipball/df69541618a0ebc24bc8f938e52f76a471f2e018",
"reference": "df69541618a0ebc24bc8f938e52f76a471f2e018",
"shasum": ""
},
"require": {
"php": ">=5.4",
"tecnickcom/tc-lib-color": "^1.11.0"
"tecnickcom/tc-lib-color": "^1.12.0"
},
"require-dev": {
"apigen/apigen": "^4.1.2",
@ -3210,20 +3244,20 @@
"tc-lib-barcode",
"upc"
],
"time": "2016-07-10 18:29:15"
"time": "2016-08-25 12:36:23"
},
{
"name": "tecnickcom/tc-lib-color",
"version": "1.11.0",
"version": "1.12.0",
"source": {
"type": "git",
"url": "https://github.com/tecnickcom/tc-lib-color.git",
"reference": "6a000b658758e271bf4c41bbc1ce4c685d8a7160"
"reference": "176464ae7ad0256c1dfd9d742ee2461d0b660f7c"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/tecnickcom/tc-lib-color/zipball/6a000b658758e271bf4c41bbc1ce4c685d8a7160",
"reference": "6a000b658758e271bf4c41bbc1ce4c685d8a7160",
"url": "https://api.github.com/repos/tecnickcom/tc-lib-color/zipball/176464ae7ad0256c1dfd9d742ee2461d0b660f7c",
"reference": "176464ae7ad0256c1dfd9d742ee2461d0b660f7c",
"shasum": ""
},
"require": {
@ -3272,7 +3306,7 @@
"tc-lib-color",
"web"
],
"time": "2016-06-13 14:31:19"
"time": "2016-08-25 11:56:01"
},
{
"name": "vlucas/phpdotenv",
@ -3326,24 +3360,24 @@
},
{
"name": "watson/validating",
"version": "2.2.1",
"version": "2.2.2",
"source": {
"type": "git",
"url": "https://github.com/dwightwatson/validating.git",
"reference": "64dc3d211372576d468e2bfaf3c7b7ace66ee970"
"reference": "8f37e416aaf02129c8ad57a446a6ef7080019687"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/dwightwatson/validating/zipball/64dc3d211372576d468e2bfaf3c7b7ace66ee970",
"reference": "64dc3d211372576d468e2bfaf3c7b7ace66ee970",
"url": "https://api.github.com/repos/dwightwatson/validating/zipball/8f37e416aaf02129c8ad57a446a6ef7080019687",
"reference": "8f37e416aaf02129c8ad57a446a6ef7080019687",
"shasum": ""
},
"require": {
"illuminate/contracts": "~5.0",
"illuminate/database": "~5.0 || >=5.1.27",
"illuminate/events": "~5.0",
"illuminate/support": "~5.0",
"illuminate/validation": "~5.0",
"illuminate/contracts": "~5.0 <5.3",
"illuminate/database": "~5.0 <5.3 || >=5.1.27",
"illuminate/events": "~5.0 <5.3",
"illuminate/support": "~5.0 <5.3",
"illuminate/validation": "~5.0 <5.3",
"php": ">=5.4.0"
},
"require-dev": {
@ -3377,7 +3411,7 @@
"laravel",
"validation"
],
"time": "2016-04-07 14:59:06"
"time": "2016-08-28 07:54:32"
}
],
"packages-dev": [
@ -3571,16 +3605,16 @@
},
{
"name": "facebook/webdriver",
"version": "1.1.2",
"version": "1.1.3",
"source": {
"type": "git",
"url": "https://github.com/facebook/php-webdriver.git",
"reference": "0b889d7de7461439f8a3bbcca46e0f696cb27986"
"reference": "b7186fb1bcfda956d237f59face250d06ef47253"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/facebook/php-webdriver/zipball/0b889d7de7461439f8a3bbcca46e0f696cb27986",
"reference": "0b889d7de7461439f8a3bbcca46e0f696cb27986",
"url": "https://api.github.com/repos/facebook/php-webdriver/zipball/b7186fb1bcfda956d237f59face250d06ef47253",
"reference": "b7186fb1bcfda956d237f59face250d06ef47253",
"shasum": ""
},
"require": {
@ -3588,7 +3622,9 @@
"php": ">=5.3.19"
},
"require-dev": {
"phpunit/phpunit": "4.6.*"
"friendsofphp/php-cs-fixer": "^1.11",
"phpunit/phpunit": "4.6.* || ~5.0",
"squizlabs/php_codesniffer": "^2.6"
},
"suggest": {
"phpdocumentor/phpdocumentor": "2.*"
@ -3611,7 +3647,7 @@
"selenium",
"webdriver"
],
"time": "2016-06-04 00:02:34"
"time": "2016-08-10 00:44:08"
},
{
"name": "fzaninotto/faker",
@ -4580,23 +4616,23 @@
},
{
"name": "sebastian/environment",
"version": "1.3.7",
"version": "1.3.8",
"source": {
"type": "git",
"url": "https://github.com/sebastianbergmann/environment.git",
"reference": "4e8f0da10ac5802913afc151413bc8c53b6c2716"
"reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/4e8f0da10ac5802913afc151413bc8c53b6c2716",
"reference": "4e8f0da10ac5802913afc151413bc8c53b6c2716",
"url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/be2c607e43ce4c89ecd60e75c6a85c126e754aea",
"reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea",
"shasum": ""
},
"require": {
"php": ">=5.3.3"
"php": "^5.3.3 || ^7.0"
},
"require-dev": {
"phpunit/phpunit": "~4.4"
"phpunit/phpunit": "^4.8 || ^5.0"
},
"type": "library",
"extra": {
@ -4626,7 +4662,7 @@
"environment",
"hhvm"
],
"time": "2016-05-17 03:18:57"
"time": "2016-08-18 05:49:44"
},
{
"name": "sebastian/exporter",
@ -5184,7 +5220,8 @@
"intervention/image": 20,
"maknz/slack": 20,
"erusev/parsedown": 20,
"tecnickcom/tc-lib-barcode": 20
"tecnickcom/tc-lib-barcode": 20,
"neitanod/forceutf8": 20
},
"prefer-stable": false,
"prefer-lowest": false,