Replace complicated test support code

Everything covered by the old hairy runner script is now covered by more
focused test suites, so the "thru" pass can be eliminated without losing
significant coverage.

3 files changed, 116 insertions, 416 deletions

diff --git a/test/meson.build b/test/meson.build
index 8f8e9e20..0f2268f6 100644
--- a/test/meson.build
+++ b/test/meson.build
@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: 0BSD OR ISC
 
 run_suite = find_program('run_suite.py')
-run_test_suite = files('run_test_suite.py')
 
 wrapper = meson.get_external_property('exe_wrapper', '')
 
@@ -22,9 +21,12 @@ if get_option('strict')
   # Check licensing metadata
   reuse = find_program('reuse', required: false)
   if reuse.found()
-    test('REUSE', reuse,
-         args: ['--root', serd_src_root, 'lint'],
-         suite: 'data')
+    test(
+      'REUSE',
+      reuse,
+      args: ['--root', serd_src_root, 'lint'],
+      suite: 'data',
+    )
   endif
 endif
 
@@ -35,7 +37,6 @@ endif
 python_scripts = files(
   '../scripts/serd_bench.py',
   'run_suite.py',
-  'run_test_suite.py',
   'serd_test_util/__init__.py',
   'test_quiet.py',
   'test_stdin.py',
@@ -96,7 +97,29 @@ if wrapper != ''
   common_script_args += ['--wrapper', wrapper]
 endif
 
-if not get_option('tools').disabled()
+simple_command_tests = {
+  'serdi': {
+    'bad': [
+      ['-c'],
+      ['-fi'],
+      ['-i', 'turtle'],
+      ['-i', 'unknown'],
+      ['-i'],
+      ['-o', 'unknown'],
+      ['-o'],
+      ['-p'],
+      ['-r'],
+      ['-z'],
+    ],
+    'good': [
+      ['-h'],
+      ['-s', '<urn:eg:s> a <urn:eg:T> .'],
+      ['-v'],
+    ],
+  },
+}
+
+if is_variable('serdi')
   script_args = common_script_args + ['--serdi', serdi]
   serd_ttl = files('../serd.ttl')[0]
 
@@ -104,87 +127,63 @@ if not get_option('tools').disabled()
 
   # Command line options
 
-  good_args = [
-    ['-v'],
-    ['-h'],
-    ['-s', '<urn:eg:s> a <urn:eg:T> .'],
-  ]
-
-  foreach args : good_args
-    test(args[0], serdi, args: args, suite: ['serdi', 'options'])
-  endforeach
-
-  bad_args = [
-    ['/no/such/file'],
-    ['ftp://unsupported.org'],
-    ['-c'],
-    ['-i', 'unknown'],
-    ['-i', 'turtle'],
-    ['-i'],
-    ['-fi'],
-    ['-o', 'unknown'],
-    ['-o'],
-    ['-p'],
-    ['-r'],
-    ['-z'],
-    ['-s', '<foo> a <Bar> .'],
-  ]
-
-  foreach args : bad_args
-    name = ' '.join(args).underscorify()
-    test(name, serdi,
-         args: args,
-         should_fail: true,
-         suite: ['serdi', 'options'])
+  cmd_suite = ['serdi', 'options']
+
+  foreach kind, cases : simple_command_tests['serdi']
+    foreach args : cases
+      test(
+        ' '.join(args).underscorify(),
+        serdi,
+        args: args,
+        should_fail: kind == 'bad',
+        suite: cmd_suite,
+      )
+    endforeach
   endforeach
 
-  test('none', serdi, should_fail: true, suite: ['serdi', 'options'])
+  test('none', serdi, should_fail: true, suite: cmd_suite)
 
-  test('quiet', files('test_quiet.py'),
-       args: script_args + files('bad/bad-base.ttl'),
-       suite: ['serdi', 'options'])
+  test(
+    'quiet',
+    files('test_quiet.py'),
+    args: script_args + files('bad/bad-base.ttl'),
+    suite: cmd_suite,
+  )
 
   # Inputs
 
-  test('stdin', files('test_stdin.py'),
-       args: script_args,
-       suite: ['serdi', 'input'])
-
-  test('string', serdi,
-       args: ['-s', '<foo> a <Bar> .'],
-       should_fail: true,
-       suite: ['serdi', 'input'])
+  input_suite = ['serdi', 'input']
 
-  test('missing', serdi,
-       args: ['-i', 'turtle'],
-       should_fail: true,
-       suite: ['serdi', 'input'])
+  bad_input_tests = {
+    'string': ['-s', '<foo> a <Bar> .'],
+    'no_such_file': ['no_such_file'],
+    'remote': ['ftp://example.org/unsupported.ttl'],
+  }
 
-  test('no_such_file', serdi,
-       args: ['no_such_file'],
-       should_fail: true,
-       suite: ['serdi', 'input'])
+  foreach name, args : bad_input_tests
+    test(name, serdi, args: args, should_fail: true, suite: input_suite)
+  endforeach
 
-  test('remote', serdi,
-       args: ['ftp://example.org/unsupported.ttl'],
-       should_fail: true,
-       suite: ['serdi', 'input'])
+  test('stdin', files('test_stdin.py'), args: script_args, suite: input_suite)
 
   # IO errors
 
-  test('read_dir', serdi,
-       args: ['-e', 'file://@0@/'.format(serd_src_root)],
-       should_fail: true,
-       suite: 'io_errors')
+  io_error_tests = {
+    'read_dir_bulk': [serd_src_root],
+    'read_dir_bytes': ['-e', serd_src_root],
+    'read_dir_uri': ['file://@0@/'.format(serd_src_root)],
+  }
 
-  test('bulk_read_dir', serdi,
-       args: ['file://@0@/'.format(serd_src_root)],
-       should_fail: true,
-       suite: 'io_errors')
+  foreach name, args : io_error_tests
+    test(name, serdi, args: args, should_fail: true, suite: 'io')
+  endforeach
 
-  test('write_error', files('test_write_error.py'),
-       args: script_args + [serd_ttl],
-       suite: 'io_errors')
+  test(
+    'write_error',
+    files('test_write_error.py'),
+    args: script_args + [serd_ttl],
+    suite: 'io',
+  )
 endif
 
 ###########################
@@ -195,10 +194,36 @@ ns_serdtest = 'http://drobilla.net/sw/serd/test/'
 ns_w3 = 'http://www.w3.org/2013/'
 
 test_suites = {
+  'NQuads': [
+    files('NQuadsTests/manifest.ttl'), ns_w3 + 'NQuadsTests/',
+    '--', '-a', '-i', 'NQuads',
+  ],
+  'NTriples': [
+    files('NTriplesTests/manifest.ttl'), ns_w3 + 'NTriplesTests/',
+    '--', '-a', '-i', 'NTriples',
+  ],
+  'TriG': [
+    files('TriGTests/manifest.ttl'), ns_w3 + 'TriGTests/',
+    '--', '-a', '-f', '-i', 'TriG',
+  ],
+  'Turtle': [
+    files('TurtleTests/manifest.ttl'), ns_w3 + 'TurtleTests/',
+    '--', '-a', '-i', 'Turtle',
+  ],
+  'bad': [
+    files('bad/manifest.ttl'), ns_serdtest + 'bad/',
+  ],
+  'bulk': [
+    files('good/manifest.ttl'), ns_serdtest + 'good/',
+    '--', '-b',
+  ],
   'full': [
     files('full/manifest.ttl'), ns_serdtest + 'full/',
     '--', '-f',
   ],
+  'good': [
+    files('good/manifest.ttl'), ns_serdtest + 'good/',
+  ],
   'lax.lax': [
     '--lax',
     files('lax/manifest.ttl'), ns_serdtest + 'lax/',
@@ -221,6 +246,7 @@ test_suites = {
   ],
   'qualify': [
     files('qualify/manifest.ttl'), ns_serdtest + 'qualify/',
+    '--', '-i', 'turtle', # Just for coverage
   ],
   'root': [
     files('root/manifest.ttl'), ns_serdtest + 'root/',
@@ -228,59 +254,16 @@ test_suites = {
   ],
 }
 
-if not get_option('tools').disabled()
+# Run every test suite with serdi
+if is_variable('serdi')
   script_args = common_script_args + ['--serdi', serdi]
 
   foreach name, args : test_suites
     test(
       name, run_suite,
       args: script_args + args,
-      suite: ['rdf'],
+      suite: ['suite'],
       timeout: 240,
     )
   endforeach
-
-  ## Serd-specific test suites
-
-  serd_suites = ['good', 'bad']
-
-  ### Run all suites with no special arguments
-  foreach name : serd_suites
-    manifest = files(name / 'manifest.ttl')
-    base_uri = ns_serdtest + name + '/'
-    test(name, run_test_suite,
-         args: script_args + [manifest, base_uri],
-         suite: ['rdf', 'serd'],
-         timeout: 240)
-  endforeach
-
-  test('good.bulk', run_test_suite,
-       args: script_args + [
-         files('good/manifest.ttl'),
-         ns_serdtest + 'good/',
-         '--',
-         '-b'
-       ],
-       is_parallel: false,
-       suite: ['rdf', 'serd'],
-       timeout: 240)
-
-  ## Standard W3C test suites
-
-  w3c_suites = ['Turtle', 'NTriples', 'NQuads', 'TriG']
-
-  foreach syntax : w3c_suites
-    manifest = files(syntax + 'Tests' / 'manifest.ttl')
-    base_uri = ns_w3 + syntax + 'Tests/'
-
-    args = [manifest, base_uri]
-    if syntax == 'TriG'
-      args += ['--', '-a']
-    endif
-
-    test(syntax, run_test_suite,
-         args: script_args + args,
-         suite: ['rdf', 'w3c'],
-         timeout: 240)
-  endforeach
 endif
diff --git a/test/run_suite.py b/test/run_suite.py
index 6426b0c2..9c3d27ee 100755
--- a/test/run_suite.py
+++ b/test/run_suite.py
@@ -119,13 +119,21 @@ def run_suite(args, command, out_dir):
         if klass not in TEST_TYPES:
             raise RuntimeError("Unknown manifest entry type: " + klass)
 
-        for instance in instances:
+        for uri in instances:
             try:
-                entry = model[instance]
+                entry = model[uri]
                 if check and NS_MF + "result" not in entry:
-                    raise RuntimeError("Eval test missing result: " + instance)
+                    raise RuntimeError("Eval test missing result: " + uri)
 
-                results.check(run_entry(args, entry, command, out_dir, top))
+                # Run test and record result
+                passed = run_entry(args, entry, command, out_dir, top)
+                results.check(passed)
+
+                # Write test report entry
+                if args.report:
+                    with open(args.report, "a", encoding="utf-8") as report:
+                        text = util.earl_assertion(uri, passed, args.asserter)
+                        report.write(text)
 
             except subprocess.CalledProcessError as exception:
                 if exception.stderr is not None:
@@ -144,7 +152,9 @@ def main():
         description=__doc__,
     )
 
+    parser.add_argument("--asserter", help="asserter URI for test report")
     parser.add_argument("--lax", action="store_true", help="tolerate errors")
+    parser.add_argument("--report", help="path to write result report to")
     parser.add_argument("--reverse", action="store_true", help="reverse test")
     parser.add_argument("--serdi", default="serdi", help="path to serdi")
     parser.add_argument("--wrapper", default="", help="executable wrapper")
diff --git a/test/run_test_suite.py b/test/run_test_suite.py
deleted file mode 100755
index a3211644..00000000
--- a/test/run_test_suite.py
+++ /dev/null
@@ -1,293 +0,0 @@
-#!/usr/bin/env python3
-
-# Copyright 2022-2023 David Robillard <d@drobilla.net>
-# SPDX-License-Identifier: ISC
-
-"""Run an RDF test suite with serdi."""
-
-# pylint: disable=too-many-arguments
-# pylint: disable=too-many-locals
-# pylint: disable=too-many-statements
-
-import argparse
-import itertools
-import os
-import shlex
-import subprocess
-import sys
-import tempfile
-
-import serd_test_util as util
-
-NS_MF = "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#"
-NS_RDFT = "http://www.w3.org/ns/rdftest#"
-
-
-def test_thru(
-    base_uri,
-    path,
-    check_lines,
-    check_path,
-    out_test_dir,
-    flags,
-    isyntax,
-    osyntax,
-    command_prefix,
-):
-    """Test lossless round-tripping through two different syntaxes."""
-
-    assert isyntax is not None
-    assert osyntax is not None
-
-    test_name = os.path.basename(path)
-    out_path = os.path.join(out_test_dir, test_name + ".pass")
-    thru_path = os.path.join(out_test_dir, test_name + ".thru")
-
-    out_opts = itertools.chain(
-        ["-i", isyntax],
-        ["-o", isyntax],
-        ["-p", "serd_test"],
-    )
-
-    out_cmd = (
-        command_prefix
-        + [f for sublist in flags for f in sublist]
-        + list(out_opts)
-        + [path, base_uri]
-    )
-
-    with open(out_path, "wb") as out:
-        subprocess.run(out_cmd, check=True, stdout=out)
-
-    thru_opts = itertools.chain(
-        ["-c", "serd_test"],
-        ["-i", isyntax],
-        ["-o", osyntax],
-    )
-
-    thru_cmd = command_prefix + list(thru_opts) + [out_path, base_uri]
-
-    proc = subprocess.run(
-        thru_cmd, check=True, capture_output=True, encoding="utf-8"
-    )
-
-    return util.lines_equal(
-        check_lines,
-        proc.stdout.splitlines(True),
-        check_path,
-        thru_path,
-    )
-
-
-def _test_input_syntax(test_class):
-    """Return the output syntax use for a given test class."""
-
-    if "NTriples" in test_class:
-        return "NTriples"
-
-    if "Turtle" in test_class:
-        return "Turtle"
-
-    if "NQuads" in test_class:
-        return "NQuads"
-
-    if "Trig" in test_class:
-        return "Trig"
-
-    raise RuntimeError("Unknown test class: " + test_class)
-
-
-def _test_output_syntax(test_class):
-    """Return the output syntax use for a given test class."""
-
-    if "NTriples" in test_class or "Turtle" in test_class:
-        return "NTriples"
-
-    if "NQuads" in test_class or "Trig" in test_class:
-        return "NQuads"
-
-    raise RuntimeError("Unknown test class: " + test_class)
-
-
-def _option_combinations(options):
-    """Return an iterator that cycles through all combinations of options."""
-
-    combinations = []
-    for count in range(len(options) + 1):
-        combinations += list(itertools.combinations(options, count))
-
-    return itertools.cycle(combinations)
-
-
-def run_suite(
-    manifest_path,
-    base_uri,
-    report_filename,
-    input_syntax,
-    command_prefix,
-    out_test_dir,
-):
-    """Run all tests in a test suite manifest."""
-
-    test_dir = os.path.dirname(manifest_path)
-    model, instances = util.load_rdf(manifest_path, base_uri, command_prefix)
-
-    asserter = ""
-    if os.getenv("USER") == "drobilla":
-        asserter = "http://drobilla.net/drobilla#me"
-
-    def run_tests(test_class, tests, expected_return, results):
-        thru_flags = [["-e"], ["-f"], ["-b"], ["-r", "http://example.org/"]]
-        osyntax = _test_output_syntax(test_class)
-        thru_options_iter = _option_combinations(thru_flags)
-
-        if input_syntax is not None:
-            isyntax = input_syntax
-        else:
-            isyntax = _test_input_syntax(test_class)
-
-        for test in sorted(tests):
-            test_uri = model[test][NS_MF + "action"][0]
-            test_uri_path = util.uri_path(test_uri)
-            test_name = os.path.basename(test_uri_path)
-            test_path = os.path.join(test_dir, test_name)
-
-            command = command_prefix + ["-f", test_path, test_uri]
-            command_string = " ".join(shlex.quote(c) for c in command)
-            out_filename = os.path.join(out_test_dir, test_name + ".out")
-
-            if expected_return == 0:  # Positive test
-                with tempfile.TemporaryFile("w+", encoding="utf-8") as out:
-                    proc = subprocess.run(command, check=False, stdout=out)
-                    passed = proc.returncode == 0
-                    results.check(
-                        passed, "Unexpected failure: " + command_string
-                    )
-
-                    if (
-                        proc.returncode == 0
-                        and NS_MF + "result" in model[test]
-                    ):
-                        # Check output against expected output from test suite
-                        check_uri = model[test][NS_MF + "result"][0]
-                        check_filename = os.path.basename(
-                            util.uri_path(check_uri)
-                        )
-                        check_path = os.path.join(test_dir, check_filename)
-
-                        with open(check_path, "r", encoding="utf-8") as check:
-                            check_lines = check.readlines()
-
-                            out.seek(0)
-                            results.check(
-                                util.lines_equal(
-                                    check_lines,
-                                    list(out),
-                                    check_path,
-                                    out_filename,
-                                )
-                            )
-
-                            # Run round-trip test
-                            check.seek(0)
-                            results.check(
-                                test_thru(
-                                    test_uri,
-                                    test_path,
-                                    check_lines,
-                                    check_path,
-                                    out_test_dir,
-                                    list(next(thru_options_iter)),
-                                    isyntax,
-                                    osyntax,
-                                    command_prefix,
-                                ),
-                                "Corrupted round-trip: " + test_uri,
-                            )
-
-            else:  # Negative test
-                with tempfile.TemporaryFile() as stderr:
-                    proc = subprocess.run(
-                        command,
-                        check=False,
-                        stdout=subprocess.DEVNULL,
-                        stderr=stderr,
-                    )
-
-                    passed = proc.returncode != 0
-                    results.check(
-                        passed, "Unexpected success: " + command_string
-                    )
-
-                    # Check that an error message was printed
-                    stderr.seek(0, 2)  # Seek to end
-                    results.check(
-                        stderr.tell() > 0,
-                        "No error message printed: " + command_string,
-                    )
-
-            # Write test report entry
-            if report_filename:
-                with open(report_filename, "a", encoding="utf-8") as report:
-                    report.write(util.earl_assertion(test, passed, asserter))
-
-    # Run all test types in the test suite
-    results = util.Results()
-    for test_class, instances in instances.items():
-        if test_class.startswith(NS_RDFT):
-            expected = (
-                1
-                if "-l" not in command_prefix and "Negative" in test_class
-                else 0
-            )
-            run_tests(test_class, instances, expected, results)
-
-    return util.print_result_summary(results)
-
-
-def main():
-    """Run the command line tool."""
-
-    parser = argparse.ArgumentParser(
-        usage="%(prog)s [OPTION]... MANIFEST BASE_URI -- [SERDI_OPTION]...",
-        description=__doc__,
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-    )
-
-    parser.add_argument("--report", help="path to write result report to")
-    parser.add_argument("--serdi", default="serdi", help="path to serdi")
-    parser.add_argument("--syntax", default=None, help="input syntax")
-    parser.add_argument("--wrapper", default="", help="executable wrapper")
-    parser.add_argument("manifest", help="test suite manifest.ttl file")
-    parser.add_argument("base_uri", help="base URI for tests")
-    parser.add_argument(
-        "serdi_option", nargs=argparse.REMAINDER, help="option for serdi"
-    )
-
-    args = parser.parse_args(sys.argv[1:])
-    command_prefix = (
-        shlex.split(args.wrapper) + [args.serdi] + args.serdi_option
-    )
-
-    with tempfile.TemporaryDirectory() as test_out_dir:
-        return run_suite(
-            args.manifest,
-            args.base_uri,
-            args.report,
-            args.syntax,
-            command_prefix,
-            test_out_dir,
-        )
-
-
-if __name__ == "__main__":
-    try:
-        sys.exit(main())
-    except subprocess.CalledProcessError as e:
-        if e.stderr is not None:
-            sys.stderr.write(e.stderr)
-
-        sys.stderr.write("error: ")
-        sys.stderr.write(str(e))
-        sys.stderr.write("\n")
-        sys.exit(e.returncode)