Fix UTF-8 issues in imports

This commit is contained in:
snipe 2016-08-29 22:57:48 -07:00
parent 64cd4fb1c9
commit e4dcd47d6c
3 changed files with 109 additions and 67 deletions

View file

@ -21,6 +21,7 @@ use Illuminate\Database\Eloquent\Model;
use League\Csv\Reader; use League\Csv\Reader;
use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Input\InputOption;
use ForceUTF8\Encoding;
ini_set('max_execution_time', 600); //600 seconds = 10 minutes ini_set('max_execution_time', 600); //600 seconds = 10 minutes
ini_set('memory_limit', '500M'); ini_set('memory_limit', '500M');
@ -284,7 +285,7 @@ class ObjectImportCommand extends Command
*/ */
public function array_smart_fetch(array $array, $key, $default = '') public function array_smart_fetch(array $array, $key, $default = '')
{ {
return array_key_exists($key, $array) ? e(trim($array[ $key ])) : $default; return array_key_exists(trim($key), $array) ? e(Encoding::fixUTF8(trim($array[ $key ]))) : $default;
} }
@ -319,10 +320,12 @@ class ObjectImportCommand extends Command
public function createOrFetchAssetModel(array $row, $category, $manufacturer) public function createOrFetchAssetModel(array $row, $category, $manufacturer)
{ {
$this->log(print_r($row));
$asset_model_name = $this->array_smart_fetch($row, "model name"); $asset_model_name = $this->array_smart_fetch($row, "model name");
$this->log('Raw Model Name: '.$asset_model_name);
$asset_modelno = $this->array_smart_fetch($row, "model number"); $asset_modelno = $this->array_smart_fetch($row, "model number");
if (empty($asset_model_name)) { if (empty($asset_model_name)) {
$asset_model_name='Unknown'; $asset_model_name ='Unknown';
} }
if (empty($asset_modelno)) { if (empty($asset_modelno)) {
$asset_modelno=''; $asset_modelno='';
@ -355,6 +358,7 @@ class ObjectImportCommand extends Command
return $asset_model; return $asset_model;
} else { } else {
$this->jsonError('Asset Model "' . $asset_model_name . '"', $asset_model->getErrors()); $this->jsonError('Asset Model "' . $asset_model_name . '"', $asset_model->getErrors());
$this->log('Asset Model "' . $asset_model_name . '"', $asset_model->getErrors());
return $asset_model; return $asset_model;
} }
} else { } else {

View file

@ -22,7 +22,8 @@
"doctrine/dbal": "v2.4.2", "doctrine/dbal": "v2.4.2",
"barryvdh/laravel-debugbar": "^2.1", "barryvdh/laravel-debugbar": "^2.1",
"spatie/laravel-backup": "3.8.1", "spatie/laravel-backup": "3.8.1",
"misterphilip/maintenance-mode": "1.0.*" "misterphilip/maintenance-mode": "1.0.*",
"neitanod/forceutf8": "dev-master"
}, },
"require-dev": { "require-dev": {
"fzaninotto/faker": "~1.4", "fzaninotto/faker": "~1.4",

165
composer.lock generated
View file

@ -4,21 +4,21 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically" "This file is @generated automatically"
], ],
"hash": "a770010d0ebb93a2e1b5f4acadd5a7f3", "hash": "ed9f8700f2dcd943ff662a82e4d8314f",
"content-hash": "59772418a4612685eea10dde38ca77b7", "content-hash": "9c0251ddc1a110d83a762483abeea079",
"packages": [ "packages": [
{ {
"name": "aws/aws-sdk-php", "name": "aws/aws-sdk-php",
"version": "3.18.39", "version": "3.19.2",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/aws/aws-sdk-php.git", "url": "https://github.com/aws/aws-sdk-php.git",
"reference": "85f1fddaeb40b95106b2a2764268e9c89fc258ce" "reference": "3cb90413129da42c9d3289d542bee0ae1049892c"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/85f1fddaeb40b95106b2a2764268e9c89fc258ce", "url": "https://api.github.com/repos/aws/aws-sdk-php/zipball/3cb90413129da42c9d3289d542bee0ae1049892c",
"reference": "85f1fddaeb40b95106b2a2764268e9c89fc258ce", "reference": "3cb90413129da42c9d3289d542bee0ae1049892c",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -85,7 +85,7 @@
"s3", "s3",
"sdk" "sdk"
], ],
"time": "2016-08-11 16:40:35" "time": "2016-08-23 20:58:48"
}, },
{ {
"name": "aws/aws-sdk-php-laravel", "name": "aws/aws-sdk-php-laravel",
@ -145,21 +145,21 @@
}, },
{ {
"name": "barryvdh/laravel-debugbar", "name": "barryvdh/laravel-debugbar",
"version": "v2.2.2", "version": "V2.2.3",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/barryvdh/laravel-debugbar.git", "url": "https://github.com/barryvdh/laravel-debugbar.git",
"reference": "c291e58d0a13953e0f68d99182ee77ebc693edc0" "reference": "ecd1ce5c4a827e2f6a8fb41bcf67713beb1c1cbd"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/c291e58d0a13953e0f68d99182ee77ebc693edc0", "url": "https://api.github.com/repos/barryvdh/laravel-debugbar/zipball/ecd1ce5c4a827e2f6a8fb41bcf67713beb1c1cbd",
"reference": "c291e58d0a13953e0f68d99182ee77ebc693edc0", "reference": "ecd1ce5c4a827e2f6a8fb41bcf67713beb1c1cbd",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"illuminate/support": "5.1.*|5.2.*", "illuminate/support": "5.1.*|5.2.*|5.3.*",
"maximebf/debugbar": "~1.11.0", "maximebf/debugbar": "~1.11.0|~1.12.0",
"php": ">=5.5.9", "php": ">=5.5.9",
"symfony/finder": "~2.7|~3.0" "symfony/finder": "~2.7|~3.0"
}, },
@ -195,7 +195,7 @@
"profiler", "profiler",
"webprofiler" "webprofiler"
], ],
"time": "2016-05-11 13:54:43" "time": "2016-07-29 15:00:36"
}, },
{ {
"name": "classpreloader/classpreloader", "name": "classpreloader/classpreloader",
@ -1018,12 +1018,12 @@
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/Intervention/image.git", "url": "https://github.com/Intervention/image.git",
"reference": "6886d43f5babe6900c29c59640ca81401fe71c80" "reference": "45a41a38bd1e5290cd51ab773013e6f041b2b711"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/Intervention/image/zipball/6886d43f5babe6900c29c59640ca81401fe71c80", "url": "https://api.github.com/repos/Intervention/image/zipball/45a41a38bd1e5290cd51ab773013e6f041b2b711",
"reference": "6886d43f5babe6900c29c59640ca81401fe71c80", "reference": "45a41a38bd1e5290cd51ab773013e6f041b2b711",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -1072,7 +1072,7 @@
"thumbnail", "thumbnail",
"watermark" "watermark"
], ],
"time": "2016-06-22 08:03:11" "time": "2016-08-19 14:41:12"
}, },
{ {
"name": "jakub-onderka/php-console-color", "name": "jakub-onderka/php-console-color",
@ -1221,16 +1221,16 @@
}, },
{ {
"name": "laravel/framework", "name": "laravel/framework",
"version": "v5.2.43", "version": "v5.2.45",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/laravel/framework.git", "url": "https://github.com/laravel/framework.git",
"reference": "5490b8f00564bb60839002f86828e27edd1e5610" "reference": "2a79f920d5584ec6df7cf996d922a742d11095d1"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/laravel/framework/zipball/5490b8f00564bb60839002f86828e27edd1e5610", "url": "https://api.github.com/repos/laravel/framework/zipball/2a79f920d5584ec6df7cf996d922a742d11095d1",
"reference": "5490b8f00564bb60839002f86828e27edd1e5610", "reference": "2a79f920d5584ec6df7cf996d922a742d11095d1",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -1311,7 +1311,7 @@
"pusher/pusher-php-server": "Required to use the Pusher broadcast driver (~2.0).", "pusher/pusher-php-server": "Required to use the Pusher broadcast driver (~2.0).",
"symfony/css-selector": "Required to use some of the crawler integration testing tools (2.8.*|3.0.*).", "symfony/css-selector": "Required to use some of the crawler integration testing tools (2.8.*|3.0.*).",
"symfony/dom-crawler": "Required to use most of the crawler integration testing tools (2.8.*|3.0.*).", "symfony/dom-crawler": "Required to use most of the crawler integration testing tools (2.8.*|3.0.*).",
"symfony/psr-http-message-bridge": "Required to psr7 bridging features (0.2.*)." "symfony/psr-http-message-bridge": "Required to use psr7 bridging features (0.2.*)."
}, },
"type": "library", "type": "library",
"extra": { "extra": {
@ -1347,7 +1347,7 @@
"framework", "framework",
"laravel" "laravel"
], ],
"time": "2016-08-10 12:23:59" "time": "2016-08-26 11:44:52"
}, },
{ {
"name": "laravelcollective/html", "name": "laravelcollective/html",
@ -1595,16 +1595,16 @@
}, },
{ {
"name": "maximebf/debugbar", "name": "maximebf/debugbar",
"version": "v1.11.1", "version": "v1.12.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/maximebf/php-debugbar.git", "url": "https://github.com/maximebf/php-debugbar.git",
"reference": "d9302891c1f0a0ac5a4f66725163a00537c6359f" "reference": "e634fbd32cd6bc3fa0e8c972b52d4bf49bab3988"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/d9302891c1f0a0ac5a4f66725163a00537c6359f", "url": "https://api.github.com/repos/maximebf/php-debugbar/zipball/e634fbd32cd6bc3fa0e8c972b52d4bf49bab3988",
"reference": "d9302891c1f0a0ac5a4f66725163a00537c6359f", "reference": "e634fbd32cd6bc3fa0e8c972b52d4bf49bab3988",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -1623,7 +1623,7 @@
"type": "library", "type": "library",
"extra": { "extra": {
"branch-alias": { "branch-alias": {
"dev-master": "1.11-dev" "dev-master": "1.12-dev"
} }
}, },
"autoload": { "autoload": {
@ -1652,7 +1652,7 @@
"debug", "debug",
"debugbar" "debugbar"
], ],
"time": "2016-01-22 12:22:23" "time": "2016-05-15 13:11:34"
}, },
{ {
"name": "misterphilip/maintenance-mode", "name": "misterphilip/maintenance-mode",
@ -1876,6 +1876,40 @@
], ],
"time": "2016-01-05 18:25:05" "time": "2016-01-05 18:25:05"
}, },
{
"name": "neitanod/forceutf8",
"version": "dev-master",
"source": {
"type": "git",
"url": "https://github.com/neitanod/forceutf8.git",
"reference": "2c1b21e00ed16b2b083ae4e27901cb5f2856db90"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/neitanod/forceutf8/zipball/2c1b21e00ed16b2b083ae4e27901cb5f2856db90",
"reference": "2c1b21e00ed16b2b083ae4e27901cb5f2856db90",
"shasum": ""
},
"require": {
"php": ">=5.3.0"
},
"type": "library",
"autoload": {
"psr-0": {
"ForceUTF8\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"authors": [
{
"name": "Sebastián Grignoli",
"email": "grignoli@gmail.com"
}
],
"description": "PHP Class Encoding featuring popular Encoding::toUTF8() function --formerly known as forceUTF8()-- that fixes mixed encoded strings.",
"homepage": "https://github.com/neitanod/forceutf8",
"time": "2015-05-07 16:37:23"
},
{ {
"name": "nesbot/carbon", "name": "nesbot/carbon",
"version": "1.21.0", "version": "1.21.0",
@ -3132,17 +3166,17 @@
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/tecnickcom/tc-lib-barcode.git", "url": "https://github.com/tecnickcom/tc-lib-barcode.git",
"reference": "220728e5f659b935348442e8d1d3e46fd5f9e178" "reference": "df69541618a0ebc24bc8f938e52f76a471f2e018"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/tecnickcom/tc-lib-barcode/zipball/220728e5f659b935348442e8d1d3e46fd5f9e178", "url": "https://api.github.com/repos/tecnickcom/tc-lib-barcode/zipball/df69541618a0ebc24bc8f938e52f76a471f2e018",
"reference": "220728e5f659b935348442e8d1d3e46fd5f9e178", "reference": "df69541618a0ebc24bc8f938e52f76a471f2e018",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=5.4", "php": ">=5.4",
"tecnickcom/tc-lib-color": "^1.11.0" "tecnickcom/tc-lib-color": "^1.12.0"
}, },
"require-dev": { "require-dev": {
"apigen/apigen": "^4.1.2", "apigen/apigen": "^4.1.2",
@ -3210,20 +3244,20 @@
"tc-lib-barcode", "tc-lib-barcode",
"upc" "upc"
], ],
"time": "2016-07-10 18:29:15" "time": "2016-08-25 12:36:23"
}, },
{ {
"name": "tecnickcom/tc-lib-color", "name": "tecnickcom/tc-lib-color",
"version": "1.11.0", "version": "1.12.0",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/tecnickcom/tc-lib-color.git", "url": "https://github.com/tecnickcom/tc-lib-color.git",
"reference": "6a000b658758e271bf4c41bbc1ce4c685d8a7160" "reference": "176464ae7ad0256c1dfd9d742ee2461d0b660f7c"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/tecnickcom/tc-lib-color/zipball/6a000b658758e271bf4c41bbc1ce4c685d8a7160", "url": "https://api.github.com/repos/tecnickcom/tc-lib-color/zipball/176464ae7ad0256c1dfd9d742ee2461d0b660f7c",
"reference": "6a000b658758e271bf4c41bbc1ce4c685d8a7160", "reference": "176464ae7ad0256c1dfd9d742ee2461d0b660f7c",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -3272,7 +3306,7 @@
"tc-lib-color", "tc-lib-color",
"web" "web"
], ],
"time": "2016-06-13 14:31:19" "time": "2016-08-25 11:56:01"
}, },
{ {
"name": "vlucas/phpdotenv", "name": "vlucas/phpdotenv",
@ -3326,24 +3360,24 @@
}, },
{ {
"name": "watson/validating", "name": "watson/validating",
"version": "2.2.1", "version": "2.2.2",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/dwightwatson/validating.git", "url": "https://github.com/dwightwatson/validating.git",
"reference": "64dc3d211372576d468e2bfaf3c7b7ace66ee970" "reference": "8f37e416aaf02129c8ad57a446a6ef7080019687"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/dwightwatson/validating/zipball/64dc3d211372576d468e2bfaf3c7b7ace66ee970", "url": "https://api.github.com/repos/dwightwatson/validating/zipball/8f37e416aaf02129c8ad57a446a6ef7080019687",
"reference": "64dc3d211372576d468e2bfaf3c7b7ace66ee970", "reference": "8f37e416aaf02129c8ad57a446a6ef7080019687",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"illuminate/contracts": "~5.0", "illuminate/contracts": "~5.0 <5.3",
"illuminate/database": "~5.0 || >=5.1.27", "illuminate/database": "~5.0 <5.3 || >=5.1.27",
"illuminate/events": "~5.0", "illuminate/events": "~5.0 <5.3",
"illuminate/support": "~5.0", "illuminate/support": "~5.0 <5.3",
"illuminate/validation": "~5.0", "illuminate/validation": "~5.0 <5.3",
"php": ">=5.4.0" "php": ">=5.4.0"
}, },
"require-dev": { "require-dev": {
@ -3377,7 +3411,7 @@
"laravel", "laravel",
"validation" "validation"
], ],
"time": "2016-04-07 14:59:06" "time": "2016-08-28 07:54:32"
} }
], ],
"packages-dev": [ "packages-dev": [
@ -3571,16 +3605,16 @@
}, },
{ {
"name": "facebook/webdriver", "name": "facebook/webdriver",
"version": "1.1.2", "version": "1.1.3",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/facebook/php-webdriver.git", "url": "https://github.com/facebook/php-webdriver.git",
"reference": "0b889d7de7461439f8a3bbcca46e0f696cb27986" "reference": "b7186fb1bcfda956d237f59face250d06ef47253"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/facebook/php-webdriver/zipball/0b889d7de7461439f8a3bbcca46e0f696cb27986", "url": "https://api.github.com/repos/facebook/php-webdriver/zipball/b7186fb1bcfda956d237f59face250d06ef47253",
"reference": "0b889d7de7461439f8a3bbcca46e0f696cb27986", "reference": "b7186fb1bcfda956d237f59face250d06ef47253",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
@ -3588,7 +3622,9 @@
"php": ">=5.3.19" "php": ">=5.3.19"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "4.6.*" "friendsofphp/php-cs-fixer": "^1.11",
"phpunit/phpunit": "4.6.* || ~5.0",
"squizlabs/php_codesniffer": "^2.6"
}, },
"suggest": { "suggest": {
"phpdocumentor/phpdocumentor": "2.*" "phpdocumentor/phpdocumentor": "2.*"
@ -3611,7 +3647,7 @@
"selenium", "selenium",
"webdriver" "webdriver"
], ],
"time": "2016-06-04 00:02:34" "time": "2016-08-10 00:44:08"
}, },
{ {
"name": "fzaninotto/faker", "name": "fzaninotto/faker",
@ -4580,23 +4616,23 @@
}, },
{ {
"name": "sebastian/environment", "name": "sebastian/environment",
"version": "1.3.7", "version": "1.3.8",
"source": { "source": {
"type": "git", "type": "git",
"url": "https://github.com/sebastianbergmann/environment.git", "url": "https://github.com/sebastianbergmann/environment.git",
"reference": "4e8f0da10ac5802913afc151413bc8c53b6c2716" "reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea"
}, },
"dist": { "dist": {
"type": "zip", "type": "zip",
"url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/4e8f0da10ac5802913afc151413bc8c53b6c2716", "url": "https://api.github.com/repos/sebastianbergmann/environment/zipball/be2c607e43ce4c89ecd60e75c6a85c126e754aea",
"reference": "4e8f0da10ac5802913afc151413bc8c53b6c2716", "reference": "be2c607e43ce4c89ecd60e75c6a85c126e754aea",
"shasum": "" "shasum": ""
}, },
"require": { "require": {
"php": ">=5.3.3" "php": "^5.3.3 || ^7.0"
}, },
"require-dev": { "require-dev": {
"phpunit/phpunit": "~4.4" "phpunit/phpunit": "^4.8 || ^5.0"
}, },
"type": "library", "type": "library",
"extra": { "extra": {
@ -4626,7 +4662,7 @@
"environment", "environment",
"hhvm" "hhvm"
], ],
"time": "2016-05-17 03:18:57" "time": "2016-08-18 05:49:44"
}, },
{ {
"name": "sebastian/exporter", "name": "sebastian/exporter",
@ -5184,7 +5220,8 @@
"intervention/image": 20, "intervention/image": 20,
"maknz/slack": 20, "maknz/slack": 20,
"erusev/parsedown": 20, "erusev/parsedown": 20,
"tecnickcom/tc-lib-barcode": 20 "tecnickcom/tc-lib-barcode": 20,
"neitanod/forceutf8": 20
}, },
"prefer-stable": false, "prefer-stable": false,
"prefer-lowest": false, "prefer-lowest": false,