Expand parser and Unicode test coverage

test/run-test

@@ -16,15 +16,12 @@ run_testprog() {
     esac
 }
 
-if [ ! -f $TESTFILE ]; then
-    echo "$TESTFILE cannot be found" >&2
-    exit 1
-fi
-
-mkdir -p $TMPDIR
-${srcdir}/split-testfile.py $TESTFILE $TMPDIR | \
-while read input output; do
-    run_test load_dump $input $output
-    run_test loadf_dumpf $input $output
-    run_test loads_dumps $input $output
+for testfile in $TESTFILES; do
+    mkdir -p $TMPDIR
+    ${srcdir}/split-testfile.py $testfile $TMPDIR | while read input output; do
+        run_test load_dump $input $output
+        run_test loadf_dumpf $input $output
+        run_test loads_dumps $input $output
+    done || exit 1
+    rm -rf $TMPDIR
 done

test/split-testfile.py

@@ -17,8 +17,8 @@ def main():
         print 'usage: %s input-file output-directory' % sys.argv[0]
         return 2
 
-    infile = sys.argv[1]
-    outdir = sys.argv[2]
+    infile = os.path.normpath(sys.argv[1])
+    outdir = os.path.normpath(sys.argv[2])
 
     if not os.path.exists(outdir):
         print >>sys.stderr, 'output directory %r does not exist!' % outdir
@@ -40,7 +40,7 @@ def main():
             current.write(line)
 
     close_files(input, output)
-    print >>sys.stderr, "%d test cases" % (i + 1)
+    print >>sys.stderr, "%s: %d test cases" % (infile, i + 1)
 
 if __name__ == '__main__':
     sys.exit(main() or 0)

test/test-invalid

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-TESTFILE="${srcdir}/testdata/invalid"
+TESTFILES="${srcdir}/testdata/invalid ${srcdir}/testdata/invalid-unicode"
 TMPDIR="tmp"
 
 run_test() {

test/test-valid

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-TESTFILE="${srcdir}/testdata/valid"
+TESTFILES="${srcdir}/testdata/valid"
 TMPDIR="tmp"
 
 run_test() {

test/testdata/invalid

@@ -77,6 +77,11 @@ unexpected token near end of file
 1
 unexpected newline near '"a'
 ========
+{"a":"a" 123}
+====
+1
+'}' expected near '123'
+========
 {[
 ====
 1
@@ -132,6 +137,16 @@ invalid token near '0'
 1
 invalid token near '-0'
 ========
+[troo
+====
+1
+invalid token near 'troo'
+========
+["\a <-- invalid escape"]
+====
+1
+invalid escape near '"\'
+========
 ["	 <-- tab character"]
 ====
 1
@@ -156,3 +171,8 @@ invalid Unicode '\uD888\u3210'
 ====
 1
 invalid Unicode '\uDFAA'
+========
+å
+====
+1
+'[' or '{' expected near 'å'

test/testdata/invalid-unicode

@@ -0,0 +1,89 @@
+<EFBFBD>
+====
+-1
+unable to decode byte 0xe5 at position 0
+========
+["<22> <-- invalid UTF-8"]
+====
+-1
+unable to decode byte 0xe5 at position 2
+========
+[<5B>]
+====
+-1
+unable to decode byte 0xe5 at position 1
+========
+[a<>]
+====
+-1
+unable to decode byte 0xe5 at position 2
+========
+["\u<>"]
+====
+-1
+unable to decode byte 0xe5 at position 4
+========
+["\<5C>"]
+====
+-1
+unable to decode byte 0xe5 at position 3
+========
+[0<>]
+====
+-1
+unable to decode byte 0xe5 at position 2
+========
+[123<32>]
+====
+-1
+unable to decode byte 0xe5 at position 4
+========
+[1e<31>]
+====
+-1
+unable to decode byte 0xe5 at position 3
+========
+[1e1<65>]
+====
+-1
+unable to decode byte 0xe5 at position 4
+========
+["<22>"]
+====
+-1
+unable to decode byte 0x81 at position 2
+========
+["<22>"]
+====
+-1
+unable to decode byte 0xc1 at position 2
+========
+["<22>"]
+====
+-1
+unable to decode byte 0xfd at position 2
+========
+["<22><><EFBFBD><EFBFBD>"]
+====
+-1
+unable to decode byte 0xf4 at position 2
+========
+["<22><><EFBFBD> <-- overlong encoding"]
+====
+-1
+unable to decode byte 0xe0 at position 2
+========
+["<22><><EFBFBD><EFBFBD> <-- overlong encoding"]
+====
+-1
+unable to decode byte 0xf0 at position 2
+========
+["<22><> <-- truncated UTF-8"]
+====
+-1
+unable to decode byte 0xe0 at position 2
+========
+["<22><><EFBFBD> <-- encoded surrogate half"]
+====
+-1
+unable to decode byte 0xed at position 2

test/testdata/valid

@@ -4,7 +4,7 @@
 ========
 ["abcdefghijklmnopqrstuvwxyz1234567890 "]
 ========
-["€þıœəßð some utf-8 ĸʒ×ŋµåäö"]
+["€þıœəßð some utf-8 ĸʒ×ŋµåäö𝄞"]
 ========
 ["\"\\\/\b\f\n\r\t"]
 ========