Wrote a quick command line app to calculate how many total images are passing our blackbox unit tests. This could use some improvement but it's a start.

git-svn-id: https://zxing.googlecode.com/svn/trunk@971 59b500cc-1b3d-0410-9834-0bbf25fbcc57

core/test/src/com/google/zxing/AllPositiveBlackBoxTester.java

@@ -0,0 +1,177 @@
+/*
+ * Copyright 2008 ZXing authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.zxing;
+
+import com.google.zxing.common.AbstractBlackBoxTestCase;
+import com.google.zxing.datamatrix.DataMatrixBlackBox1TestCase;
+import com.google.zxing.datamatrix.DataMatrixBlackBox2TestCase;
+import com.google.zxing.oned.Code128BlackBox1TestCase;
+import com.google.zxing.oned.Code128BlackBox2TestCase;
+import com.google.zxing.oned.Code128BlackBox3TestCase;
+import com.google.zxing.oned.Code39BlackBox1TestCase;
+import com.google.zxing.oned.Code39BlackBox3TestCase;
+import com.google.zxing.oned.Code39ExtendedBlackBox2TestCase;
+import com.google.zxing.oned.EAN13BlackBox1TestCase;
+import com.google.zxing.oned.EAN13BlackBox2TestCase;
+import com.google.zxing.oned.EAN13BlackBox3TestCase;
+import com.google.zxing.oned.EAN13BlackBox4TestCase;
+import com.google.zxing.oned.EAN8BlackBox1TestCase;
+import com.google.zxing.oned.ITFBlackBox1TestCase;
+import com.google.zxing.oned.ITFBlackBox2TestCase;
+import com.google.zxing.oned.UPCABlackBox1TestCase;
+import com.google.zxing.oned.UPCABlackBox2TestCase;
+import com.google.zxing.oned.UPCABlackBox3ReflectiveTestCase;
+import com.google.zxing.oned.UPCABlackBox4TestCase;
+import com.google.zxing.oned.UPCEBlackBox1TestCase;
+import com.google.zxing.oned.UPCEBlackBox2TestCase;
+import com.google.zxing.oned.UPCEBlackBox3ReflectiveTestCase;
+import com.google.zxing.qrcode.QRCodeBlackBox1TestCase;
+import com.google.zxing.qrcode.QRCodeBlackBox2TestCase;
+import com.google.zxing.qrcode.QRCodeBlackBox3TestCase;
+import com.google.zxing.qrcode.QRCodeBlackBox4TestCase;
+import com.google.zxing.qrcode.QRCodeBlackBox5TestCase;
+
+/**
+ * This is a quick and dirty way to get totals across all the positive black box tests. It is
+ * necessary because we spawn multiple processes when using the standard test-blackbox Ant target.
+ * It would be a shame to change that because it does help with performance. Perhaps we can find a
+ * way to unify these in the future.
+ *
+ * @author dswitkin@google.com (Daniel Switkin)
+ */
+public final class AllPositiveBlackBoxTester {
+
+  // This list has to be manually kept up to date. I don't know any automatic way to include every
+  // subclass of AbstractBlackBoxTestCase, and furthermore to exclude subclasses of
+  // AbstractNegativeBlackBoxTestCase which derives from it.
+  private static final AbstractBlackBoxTestCase[] TESTS = {
+    new DataMatrixBlackBox1TestCase(),
+    new DataMatrixBlackBox2TestCase(),
+    new Code128BlackBox1TestCase(),
+    new Code128BlackBox2TestCase(),
+    new Code128BlackBox3TestCase(),
+    new Code39BlackBox1TestCase(),
+    new Code39ExtendedBlackBox2TestCase(),
+    new Code39BlackBox3TestCase(),
+    new EAN13BlackBox1TestCase(),
+    new EAN13BlackBox2TestCase(),
+    new EAN13BlackBox3TestCase(),
+    new EAN13BlackBox4TestCase(),
+    new EAN8BlackBox1TestCase(),
+    new ITFBlackBox1TestCase(),
+    new ITFBlackBox2TestCase(),
+    new UPCABlackBox1TestCase(),
+    new UPCABlackBox2TestCase(),
+    new UPCABlackBox3ReflectiveTestCase(),
+    new UPCABlackBox4TestCase(),
+    new UPCEBlackBox1TestCase(),
+    new UPCEBlackBox2TestCase(),
+    new UPCEBlackBox3ReflectiveTestCase(),
+    new QRCodeBlackBox1TestCase(),
+    new QRCodeBlackBox2TestCase(),
+    new QRCodeBlackBox3TestCase(),
+    new QRCodeBlackBox4TestCase(),
+    new QRCodeBlackBox5TestCase()
+  };
+
+//  private static final AbstractBlackBoxTestCase[] TESTS1 = {
+//    new DataMatrixBlackBox1TestCase(),
+//    new DataMatrixBlackBox2TestCase(),
+//    new Code128BlackBox1TestCase(),
+//    new Code128BlackBox2TestCase(),
+//    new Code128BlackBox3TestCase(),
+//    new Code39BlackBox1TestCase(),
+//    new Code39ExtendedBlackBox2TestCase(),
+//    new Code39BlackBox3TestCase(),
+//    new EAN13BlackBox1TestCase(),
+//    new EAN13BlackBox2TestCase(),
+//    new EAN13BlackBox3TestCase(),
+//    new EAN13BlackBox4TestCase(),
+//    new EAN8BlackBox1TestCase(),
+//    new ITFBlackBox1TestCase(),
+//    new ITFBlackBox2TestCase(),
+//  };
+//
+//  private static final AbstractBlackBoxTestCase[] TESTS2 = {
+//    new UPCABlackBox1TestCase(),
+//    new UPCABlackBox2TestCase(),
+//    new UPCABlackBox3ReflectiveTestCase(),
+//    new UPCABlackBox4TestCase(),
+//    new UPCEBlackBox1TestCase(),
+//    new UPCEBlackBox2TestCase(),
+//    new UPCEBlackBox3ReflectiveTestCase(),
+//    new QRCodeBlackBox1TestCase(),
+//    new QRCodeBlackBox2TestCase(),
+//    new QRCodeBlackBox3TestCase(),
+//    new QRCodeBlackBox4TestCase(),
+//    new QRCodeBlackBox5TestCase()
+//  };
+//
+//  private static class WorkerThread extends Thread {
+//
+//    private AbstractBlackBoxTestCase[] testCases;
+//    private AbstractBlackBoxTestCase.SummaryResults results;
+//
+//    public WorkerThread(AbstractBlackBoxTestCase[] tests) {
+//      testCases = tests;
+//    }
+//
+//    @Override
+//    public void run() {
+//      try {
+//        results = new AbstractBlackBoxTestCase.SummaryResults();
+//        for (int x = 0; x < testCases.length; x++) {
+//          results.add(testCases[x].testBlackBoxCountingResults());
+//        }
+//      } catch (IOException e) {
+//
+//      }
+//    }
+//
+//    public AbstractBlackBoxTestCase.SummaryResults getResults() {
+//      return results;
+//    }
+//  }
+
+  private AllPositiveBlackBoxTester() {
+
+  }
+
+  public static void main(String[] args) throws Exception {
+    long now = System.currentTimeMillis();
+    AbstractBlackBoxTestCase.SummaryResults results = new AbstractBlackBoxTestCase.SummaryResults();
+
+    for (int x = 0; x < TESTS.length; x++) {
+      results.add(TESTS[x].testBlackBoxCountingResults());
+    }
+
+    // This threaded version can't be used yet because BlackPointEstimator (and possibly other code)
+    // is not thread-safe. It's also pretty lame. It would be much better to spawn a group of
+    // threads which pull tests from a shared, locked list as needed.
+//    WorkerThread thread1 = new WorkerThread(TESTS1);
+//    WorkerThread thread2 = new WorkerThread(TESTS2);
+//    thread1.start();
+//    thread2.start();
+//    thread1.join();
+//    thread2.join();
+//    results.add(thread1.getResults());
+//    results.add(thread2.getResults());
+
+    now = System.currentTimeMillis() - now;
+    System.out.println(results.toString() + "\n  Total time: " + now + " ms");
+  }
+}

core/test/src/com/google/zxing/common/AbstractBlackBoxTestCase.java

@@ -23,9 +23,9 @@ import com.google.zxing.Reader;
 import com.google.zxing.ReaderException;
 import com.google.zxing.Result;
 import com.google.zxing.client.j2se.BufferedImageMonochromeBitmapSource;
+
 import junit.framework.TestCase;
 
-import javax.imageio.ImageIO;
 import java.awt.geom.AffineTransform;
 import java.awt.image.AffineTransformOp;
 import java.awt.image.BufferedImage;
@@ -35,10 +35,12 @@ import java.io.FileInputStream;
 import java.io.FilenameFilter;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
 import java.util.Hashtable;
 import java.util.List;
-import java.util.ArrayList;
-import java.nio.charset.Charset;
+
+import javax.imageio.ImageIO;
 
 /**
  * @author Sean Owen
@@ -60,6 +62,35 @@ public abstract class AbstractBlackBoxTestCase extends TestCase {
     }
   };
 
+  public static class SummaryResults {
+    private int totalFound;
+    private int totalMustPass;
+    private int totalTests;
+
+    public SummaryResults() {
+      totalFound = 0;
+      totalMustPass = 0;
+      totalTests = 0;
+    }
+
+    public SummaryResults(int found, int mustPass, int total) {
+      totalFound = found;
+      totalMustPass = mustPass;
+      totalTests = total;
+    }
+
+    public void add(SummaryResults other) {
+      totalFound += other.totalFound;
+      totalMustPass += other.totalMustPass;
+      totalTests += other.totalTests;
+    }
+
+    public String toString() {
+      return "\nSUMMARY RESULTS:\n  Decoded " + totalFound + " images out of " + totalTests +
+        " (" + (totalFound * 100 / totalTests) + "%, " + totalMustPass + " required)";
+    }
+  }
+
   private static class TestResult {
     private final int mustPassCount;
     private final int tryHarderCount;
@@ -125,7 +156,13 @@ public abstract class AbstractBlackBoxTestCase extends TestCase {
     return null;
   }
 
+  // This workaround is used because AbstractNegativeBlackBoxTestCase overrides this method but does
+  // not return SummaryResults.
   public void testBlackBox() throws IOException {
+    testBlackBoxCountingResults();
+  }
+
+  public SummaryResults testBlackBoxCountingResults() throws IOException {
     assertFalse(testResults.isEmpty());
 
     File[] imageFiles = getImageFiles();
@@ -187,6 +224,7 @@ public abstract class AbstractBlackBoxTestCase extends TestCase {
           " degrees: Too many images failed",
           tryHarderCounts[x] >= testResults.get(x).getTryHarderCount());
     }
+    return new SummaryResults(totalFound, totalMustPass, totalTests);
   }
 
   private boolean decode(MonochromeBitmapSource source, float rotation, String expectedText,

core/test/src/com/google/zxing/oned/EAN13BlackBox4TestCase.java

@@ -28,7 +28,7 @@ public final class EAN13BlackBox4TestCase extends AbstractBlackBoxTestCase {
 
   public EAN13BlackBox4TestCase() {
     super("test/data/blackbox/ean13-4", new MultiFormatReader(), BarcodeFormat.EAN_13);
-    addTest(6, 12, 0.0f);
+    addTest(6, 13, 0.0f);
     addTest(7, 13, 180.0f);
   }