Run lax suite with the simpler runner script

3 files changed, 52 insertions, 21 deletions

diff --git a/test/lax/README.md b/test/lax/README.md
new file mode 100644
index 00000000..d1b5d375
--- /dev/null
+++ b/test/lax/README.md
@@ -0,0 +1,5 @@
+Lax Test Suite
+==============
+
+This suite contains negative tests that can be successfully read (with
+warnings) if and only if lax parsing is enabled.
diff --git a/test/meson.build b/test/meson.build
index e4850f0c..f5282672 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -195,6 +195,14 @@ ns_serdtest = 'http://drobilla.net/sw/serd/test/'
 ns_w3 = 'http://www.w3.org/2013/'
 
 test_suites = {
+  'lax.lax': [
+    '--lax',
+    files('lax/manifest.ttl'), ns_serdtest + 'lax/',
+    '--', '-l',
+  ],
+  'lax.strict': [
+    files('lax/manifest.ttl'), ns_serdtest + 'lax/',
+  ],
   'prefix.add': [
     '--reverse',
     files('prefix/manifest.ttl'), ns_serdtest + 'prefix/',
@@ -253,24 +261,6 @@ if not get_option('tools').disabled()
        suite: ['rdf', 'serd'],
        timeout: 240)
 
-  ### The lax suite is special because it is run twice...
-  lax_manifest = files('lax/manifest.ttl')
-  lax_base_uri = ns_serdtest + 'lax/'
-
-  ### ... once with strict parsing to test the hard errors
-  test('lax.strict', run_test_suite,
-       args: script_args + [lax_manifest, lax_base_uri],
-       is_parallel: false,
-       suite: ['rdf', 'serd'],
-       timeout: 240)
-
-  ### ... and once with lax parsing to tolerate them
-  test('lax.lax', run_test_suite,
-       args: script_args + [lax_manifest, lax_base_uri, '--', '-l'],
-       is_parallel: false,
-       suite: ['rdf', 'serd'],
-       timeout: 240)
-
   ## Standard W3C test suites
 
   w3c_suites = ['Turtle', 'NTriples', 'NQuads', 'TriG']
diff --git a/test/run_suite.py b/test/run_suite.py
index 5c74c9c7..6426b0c2 100755
--- a/test/run_suite.py
+++ b/test/run_suite.py
@@ -15,13 +15,16 @@ import tempfile
 import serd_test_util as util
 
 NS_MF = "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#"
+NS_RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
 NS_RDFT = "http://www.w3.org/ns/rdftest#"
 
 DEVNULL = subprocess.DEVNULL
 PIPE = subprocess.PIPE
 
 TEST_TYPES = [
+    NS_RDFT + "TestNQuadsNegativeSyntax",
     NS_RDFT + "TestNQuadsPositiveSyntax",
+    NS_RDFT + "TestNTriplesNegativeSyntax",
     NS_RDFT + "TestNTriplesPositiveSyntax",
     NS_RDFT + "TestTrigEval",
     NS_RDFT + "TestTrigNegativeEval",
@@ -49,16 +52,48 @@ def run_eval_test(base_uri, command, in_path, good_path, out_path):
         return util.lines_equal(list(good), out, good_path, out_path)
 
 
+def run_positive_test(base_uri, command, in_path):
+    """Run a positive syntax test and ensure no errors occur."""
+
+    command = command + [in_path, base_uri]
+    subprocess.check_call(command, encoding="utf-8", stdout=DEVNULL)
+    return True
+
+
+def run_negative_test(base_uri, command, in_path):
+    """Run a negative syntax test and return whether the error was detected."""
+
+    if not os.path.exists(in_path):
+        raise RuntimeError("Input file missing: " + in_path)
+
+    command = command + [in_path, base_uri]
+    proc = subprocess.run(command, check=False, stderr=PIPE, stdout=DEVNULL)
+
+    if proc.returncode == 0:
+        util.error("Unexpected successful return: " + in_path)
+        return False
+
+    if len(proc.stderr) == 0:
+        util.error("Command failed with no error output: " + in_path)
+        return False
+
+    return True
+
+
 def run_entry(args, entry, command, out_dir, suite_dir):
     """Run a single test entry from the manifest."""
 
     in_path = util.file_path(suite_dir, entry[NS_MF + "action"][0])
     base = args.base_uri + os.path.basename(in_path)
 
-    good_path = in_path
-    if NS_MF + "result" in entry:
-        good_path = util.file_path(suite_dir, entry[NS_MF + "result"][0])
+    negative = "Negative" in entry[NS_RDF + "type"][0]
+    if negative and not args.lax:
+        return run_negative_test(base, command, in_path)
+
+    if NS_MF + "result" not in entry:
+        return run_positive_test(base, command, in_path)
 
+    good_path = util.file_path(suite_dir, entry[NS_MF + "result"][0])
     if args.reverse:
         in_path, good_path = good_path, in_path
 
@@ -109,6 +144,7 @@ def main():
         description=__doc__,
     )
 
+    parser.add_argument("--lax", action="store_true", help="tolerate errors")
     parser.add_argument("--reverse", action="store_true", help="reverse test")
     parser.add_argument("--serdi", default="serdi", help="path to serdi")
     parser.add_argument("--wrapper", default="", help="executable wrapper")