From 14e22bd443a69941de7664d821c6d6f03889196c Mon Sep 17 00:00:00 2001 From: srowen Date: Sun, 2 Nov 2008 12:22:33 +0000 Subject: [PATCH] Revamp the character encoding detection to use a more complicated but still crude algorithm that detects more Shift_JIS sequences correctly. git-svn-id: https://zxing.googlecode.com/svn/trunk@664 59b500cc-1b3d-0410-9834-0bbf25fbcc57 --- .../decoder/DecodedBitStreamParser.java | 52 +++++++++++-------- .../zxing/qrcode/QRCodeBlackBox2TestCase.java | 8 +-- .../DecodedBitStreamParserTestCase.java | 15 +++++- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java b/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java index 287e56cd6..f9489801f 100644 --- a/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java +++ b/core/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParser.java @@ -254,14 +254,23 @@ final class DecodedBitStreamParser { // that it's UTF-8. int length = bytes.length; boolean canBeISO88591 = true; + boolean canBeShiftJIS = true; + boolean sawDoubleByteStart = false; + int maybeSingleByteKatakanaCount = 0; boolean lastWasPossibleDoubleByteStart = false; - for (int i = 0; i < length; i++) { + for (int i = 0; i < length && (canBeISO88591 || canBeShiftJIS); i++) { int value = bytes[i] & 0xFF; - if (value >= 0x80 && value <= 0x9F && i < length - 1) { + if (value >= 0x7F && value <= 0x9F) { canBeISO88591 = false; - // ISO-8859-1 shouldn't use this, but before we decide it is Shift_JIS, - // just double check that it is followed by a byte that's valid in - // the Shift_JIS encoding + } + if (value >= 0xA1 && value <= 0xDF) { + // count the number of characters that might be a Shift_JIS single-byte Katakana character + maybeSingleByteKatakanaCount++; + } + if (((value >= 0x81 && value <= 0x9F) || (value >= 0xE0 && value <= 0xEF)) && i < length - 1) { + // These start double-byte characters in Shift_JIS. Let's see if it's followed by a valid + // second byte. + sawDoubleByteStart = true; if (lastWasPossibleDoubleByteStart) { // If we just checked this and the last byte for being a valid double-byte // char, don't check starting on this byte. If this and the last byte @@ -274,30 +283,27 @@ final class DecodedBitStreamParser { lastWasPossibleDoubleByteStart = true; int nextValue = bytes[i + 1] & 0xFF; if (nextValue < 0x40 || nextValue > 0xFC) { - return UTF8; + canBeShiftJIS = false; } // There is some conflicting information out there about which bytes can follow which in // double-byte Shift_JIS characters. The rule above seems to be the one that matches practice. - // The stricter rule below, however, is given by other resources. - /* - if ((value & 0x1) == 0) { - // if even, next value should be in [0x9F,0xFC] - // if not, we'll guess UTF-8 - if (nextValue < 0x9F || nextValue > 0xFC) { - return UTF8; - } - } else { - // if odd, next value should be in [0x40,0x9E] - // if not, we'll guess UTF-8 - if (nextValue < 0x40 || nextValue > 0x9E) { - return UTF8; - } - } - */ } } } - return canBeISO88591 ? ISO88591 : SHIFT_JIS; + // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough. The crude heuristic is: + // - If we saw + // - at least one byte that starts a double-byte value (bytes that are rare in ISO-8859-1), or + // - over 5% of bytes that could be single-byte Katakana (also rare in ISO-8859-1), + // - and, saw no sequences that are invalid in Shift_JIS, then we conclude Shift_JIS + if ((sawDoubleByteStart || 20 * maybeSingleByteKatakanaCount > length) && canBeShiftJIS) { + return SHIFT_JIS; + } + // Otherwise, we default to ISO-8859-1 unless we know it can't be + if (canBeISO88591) { + return ISO88591; + } + // Otherwise, we take a wild guess with UTF-8 + return UTF8; } private static int parseECIValue(BitSource bits) { diff --git a/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java b/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java index a5d946fcf..82bfa4ec8 100644 --- a/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java +++ b/core/test/src/com/google/zxing/qrcode/QRCodeBlackBox2TestCase.java @@ -29,10 +29,10 @@ public final class QRCodeBlackBox2TestCase extends AbstractBlackBoxTestCase { public QRCodeBlackBox2TestCase() { super(new File("test/data/blackbox/qrcode-2"), new MultiFormatReader(), BarcodeFormat.QR_CODE); - addTest(14, 14, 0.0f); - addTest(10, 10, 90.0f); - addTest(13, 13, 180.0f); - addTest(10, 10, 270.0f); + addTest(23, 23, 0.0f); + addTest(18, 18, 90.0f); + addTest(22, 22, 180.0f); + addTest(17, 17, 270.0f); } } \ No newline at end of file diff --git a/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java b/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java index 8b4136687..76be3902e 100644 --- a/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java +++ b/core/test/src/com/google/zxing/qrcode/decoder/DecodedBitStreamParserTestCase.java @@ -28,6 +28,17 @@ import junit.framework.TestCase; public final class DecodedBitStreamParserTestCase extends TestCase { public void testSimpleByteMode() throws ReaderException { + BitSourceBuilder builder = new BitSourceBuilder(); + builder.write(0x04, 4); // Byte mode + builder.write(0x03, 8); // 3 bytes + builder.write(0xF1, 8); + builder.write(0xF2, 8); + builder.write(0xF3, 8); + String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1)); + assertEquals("\u00f1\u00f2\u00f3", result); + } + + public void testSimpleSJIS() throws ReaderException { BitSourceBuilder builder = new BitSourceBuilder(); builder.write(0x04, 4); // Byte mode builder.write(0x03, 8); // 3 bytes @@ -35,7 +46,7 @@ public final class DecodedBitStreamParserTestCase extends TestCase { builder.write(0xA2, 8); builder.write(0xA3, 8); String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1)); - assertEquals("\u00a1\u00a2\u00a3", result); // this should be "¡¢£" if your editor character encoding matches mine! + assertEquals("\uff61\uff62\uff63", result); } public void testECI() throws ReaderException { @@ -48,7 +59,7 @@ public final class DecodedBitStreamParserTestCase extends TestCase { builder.write(0xA2, 8); builder.write(0xA3, 8); String result = DecodedBitStreamParser.decode(builder.toByteArray(), Version.getVersionForNumber(1)); - assertEquals("\u00ed\u00f3\u00fa", result); // should be like "íóú" + assertEquals("\u00ed\u00f3\u00fa", result); } // TODO definitely need more tests here