diff options
| -rw-r--r-- | .gitlab-ci.yml | 2 | ||||
| -rw-r--r-- | test/meson.build | 221 | ||||
| -rwxr-xr-x | test/run_suite.py | 18 | ||||
| -rwxr-xr-x | test/run_test_suite.py | 293 |
4 files changed, 117 insertions, 417 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c19cfcb2..2f327470 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -84,7 +84,7 @@ mingw32: image: lv2plugin/debian-mingw32 script: - meson setup build --cross-file=/usr/share/meson/cross/i686-w64-mingw32.ini -Dc_std=c11 -Dbuildtype=debug -Ddocs=disabled -Dstrict=true -Dwerror=true - - ninja -C build + - ninja -C build test - meson configure -Dbuildtype=release build - ninja -C build test variables: diff --git a/test/meson.build b/test/meson.build index 8f8e9e20..0f2268f6 100644 --- a/test/meson.build +++ b/test/meson.build @@ -2,7 +2,6 @@ # SPDX-License-Identifier: 0BSD OR ISC run_suite = find_program('run_suite.py') -run_test_suite = files('run_test_suite.py') wrapper = meson.get_external_property('exe_wrapper', '') @@ -22,9 +21,12 @@ if get_option('strict') # Check licensing metadata reuse = find_program('reuse', required: false) if reuse.found() - test('REUSE', reuse, - args: ['--root', serd_src_root, 'lint'], - suite: 'data') + test( + 'REUSE', + reuse, + args: ['--root', serd_src_root, 'lint'], + suite: 'data', + ) endif endif @@ -35,7 +37,6 @@ endif python_scripts = files( '../scripts/serd_bench.py', 'run_suite.py', - 'run_test_suite.py', 'serd_test_util/__init__.py', 'test_quiet.py', 'test_stdin.py', @@ -96,7 +97,29 @@ if wrapper != '' common_script_args += ['--wrapper', wrapper] endif -if not get_option('tools').disabled() +simple_command_tests = { + 'serdi': { + 'bad': [ + ['-c'], + ['-fi'], + ['-i', 'turtle'], + ['-i', 'unknown'], + ['-i'], + ['-o', 'unknown'], + ['-o'], + ['-p'], + ['-r'], + ['-z'], + ], + 'good': [ + ['-h'], + ['-s', '<urn:eg:s> a <urn:eg:T> .'], + ['-v'], + ], + }, +} + +if is_variable('serdi') script_args = common_script_args + ['--serdi', serdi] serd_ttl = files('../serd.ttl')[0] @@ -104,87 +127,63 @@ if not get_option('tools').disabled() # Command line options - good_args = [ - ['-v'], - ['-h'], - ['-s', '<urn:eg:s> a <urn:eg:T> .'], - ] - - foreach args : good_args - test(args[0], serdi, args: args, suite: ['serdi', 'options']) - endforeach - - bad_args = [ - ['/no/such/file'], - ['ftp://unsupported.org'], - ['-c'], - ['-i', 'unknown'], - ['-i', 'turtle'], - ['-i'], - ['-fi'], - ['-o', 'unknown'], - ['-o'], - ['-p'], - ['-r'], - ['-z'], - ['-s', '<foo> a <Bar> .'], - ] - - foreach args : bad_args - name = ' '.join(args).underscorify() - test(name, serdi, - args: args, - should_fail: true, - suite: ['serdi', 'options']) + cmd_suite = ['serdi', 'options'] + + foreach kind, cases : simple_command_tests['serdi'] + foreach args : cases + test( + ' '.join(args).underscorify(), + serdi, + args: args, + should_fail: kind == 'bad', + suite: cmd_suite, + ) + endforeach endforeach - test('none', serdi, should_fail: true, suite: ['serdi', 'options']) + test('none', serdi, should_fail: true, suite: cmd_suite) - test('quiet', files('test_quiet.py'), - args: script_args + files('bad/bad-base.ttl'), - suite: ['serdi', 'options']) + test( + 'quiet', + files('test_quiet.py'), + args: script_args + files('bad/bad-base.ttl'), + suite: cmd_suite, + ) # Inputs - test('stdin', files('test_stdin.py'), - args: script_args, - suite: ['serdi', 'input']) - - test('string', serdi, - args: ['-s', '<foo> a <Bar> .'], - should_fail: true, - suite: ['serdi', 'input']) + input_suite = ['serdi', 'input'] - test('missing', serdi, - args: ['-i', 'turtle'], - should_fail: true, - suite: ['serdi', 'input']) + bad_input_tests = { + 'string': ['-s', '<foo> a <Bar> .'], + 'no_such_file': ['no_such_file'], + 'remote': ['ftp://example.org/unsupported.ttl'], + } - test('no_such_file', serdi, - args: ['no_such_file'], - should_fail: true, - suite: ['serdi', 'input']) + foreach name, args : bad_input_tests + test(name, serdi, args: args, should_fail: true, suite: input_suite) + endforeach - test('remote', serdi, - args: ['ftp://example.org/unsupported.ttl'], - should_fail: true, - suite: ['serdi', 'input']) + test('stdin', files('test_stdin.py'), args: script_args, suite: input_suite) # IO errors - test('read_dir', serdi, - args: ['-e', 'file://@0@/'.format(serd_src_root)], - should_fail: true, - suite: 'io_errors') + io_error_tests = { + 'read_dir_bulk': [serd_src_root], + 'read_dir_bytes': ['-e', serd_src_root], + 'read_dir_uri': ['file://@0@/'.format(serd_src_root)], + } - test('bulk_read_dir', serdi, - args: ['file://@0@/'.format(serd_src_root)], - should_fail: true, - suite: 'io_errors') + foreach name, args : io_error_tests + test(name, serdi, args: args, should_fail: true, suite: 'io') + endforeach - test('write_error', files('test_write_error.py'), - args: script_args + [serd_ttl], - suite: 'io_errors') + test( + 'write_error', + files('test_write_error.py'), + args: script_args + [serd_ttl], + suite: 'io', + ) endif ########################### @@ -195,10 +194,36 @@ ns_serdtest = 'http://drobilla.net/sw/serd/test/' ns_w3 = 'http://www.w3.org/2013/' test_suites = { + 'NQuads': [ + files('NQuadsTests/manifest.ttl'), ns_w3 + 'NQuadsTests/', + '--', '-a', '-i', 'NQuads', + ], + 'NTriples': [ + files('NTriplesTests/manifest.ttl'), ns_w3 + 'NTriplesTests/', + '--', '-a', '-i', 'NTriples', + ], + 'TriG': [ + files('TriGTests/manifest.ttl'), ns_w3 + 'TriGTests/', + '--', '-a', '-f', '-i', 'TriG', + ], + 'Turtle': [ + files('TurtleTests/manifest.ttl'), ns_w3 + 'TurtleTests/', + '--', '-a', '-i', 'Turtle', + ], + 'bad': [ + files('bad/manifest.ttl'), ns_serdtest + 'bad/', + ], + 'bulk': [ + files('good/manifest.ttl'), ns_serdtest + 'good/', + '--', '-b', + ], 'full': [ files('full/manifest.ttl'), ns_serdtest + 'full/', '--', '-f', ], + 'good': [ + files('good/manifest.ttl'), ns_serdtest + 'good/', + ], 'lax.lax': [ '--lax', files('lax/manifest.ttl'), ns_serdtest + 'lax/', @@ -221,6 +246,7 @@ test_suites = { ], 'qualify': [ files('qualify/manifest.ttl'), ns_serdtest + 'qualify/', + '--', '-i', 'turtle', # Just for coverage ], 'root': [ files('root/manifest.ttl'), ns_serdtest + 'root/', @@ -228,59 +254,16 @@ test_suites = { ], } -if not get_option('tools').disabled() +# Run every test suite with serdi +if is_variable('serdi') script_args = common_script_args + ['--serdi', serdi] foreach name, args : test_suites test( name, run_suite, args: script_args + args, - suite: ['rdf'], + suite: ['suite'], timeout: 240, ) endforeach - - ## Serd-specific test suites - - serd_suites = ['good', 'bad'] - - ### Run all suites with no special arguments - foreach name : serd_suites - manifest = files(name / 'manifest.ttl') - base_uri = ns_serdtest + name + '/' - test(name, run_test_suite, - args: script_args + [manifest, base_uri], - suite: ['rdf', 'serd'], - timeout: 240) - endforeach - - test('good.bulk', run_test_suite, - args: script_args + [ - files('good/manifest.ttl'), - ns_serdtest + 'good/', - '--', - '-b' - ], - is_parallel: false, - suite: ['rdf', 'serd'], - timeout: 240) - - ## Standard W3C test suites - - w3c_suites = ['Turtle', 'NTriples', 'NQuads', 'TriG'] - - foreach syntax : w3c_suites - manifest = files(syntax + 'Tests' / 'manifest.ttl') - base_uri = ns_w3 + syntax + 'Tests/' - - args = [manifest, base_uri] - if syntax == 'TriG' - args += ['--', '-a'] - endif - - test(syntax, run_test_suite, - args: script_args + args, - suite: ['rdf', 'w3c'], - timeout: 240) - endforeach endif diff --git a/test/run_suite.py b/test/run_suite.py index 6426b0c2..9c3d27ee 100755 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -119,13 +119,21 @@ def run_suite(args, command, out_dir): if klass not in TEST_TYPES: raise RuntimeError("Unknown manifest entry type: " + klass) - for instance in instances: + for uri in instances: try: - entry = model[instance] + entry = model[uri] if check and NS_MF + "result" not in entry: - raise RuntimeError("Eval test missing result: " + instance) + raise RuntimeError("Eval test missing result: " + uri) - results.check(run_entry(args, entry, command, out_dir, top)) + # Run test and record result + passed = run_entry(args, entry, command, out_dir, top) + results.check(passed) + + # Write test report entry + if args.report: + with open(args.report, "a", encoding="utf-8") as report: + text = util.earl_assertion(uri, passed, args.asserter) + report.write(text) except subprocess.CalledProcessError as exception: if exception.stderr is not None: @@ -144,7 +152,9 @@ def main(): description=__doc__, ) + parser.add_argument("--asserter", help="asserter URI for test report") parser.add_argument("--lax", action="store_true", help="tolerate errors") + parser.add_argument("--report", help="path to write result report to") parser.add_argument("--reverse", action="store_true", help="reverse test") parser.add_argument("--serdi", default="serdi", help="path to serdi") parser.add_argument("--wrapper", default="", help="executable wrapper") diff --git a/test/run_test_suite.py b/test/run_test_suite.py deleted file mode 100755 index a3211644..00000000 --- a/test/run_test_suite.py +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2022-2023 David Robillard <d@drobilla.net> -# SPDX-License-Identifier: ISC - -"""Run an RDF test suite with serdi.""" - -# pylint: disable=too-many-arguments -# pylint: disable=too-many-locals -# pylint: disable=too-many-statements - -import argparse -import itertools -import os -import shlex -import subprocess -import sys -import tempfile - -import serd_test_util as util - -NS_MF = "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#" -NS_RDFT = "http://www.w3.org/ns/rdftest#" - - -def test_thru( - base_uri, - path, - check_lines, - check_path, - out_test_dir, - flags, - isyntax, - osyntax, - command_prefix, -): - """Test lossless round-tripping through two different syntaxes.""" - - assert isyntax is not None - assert osyntax is not None - - test_name = os.path.basename(path) - out_path = os.path.join(out_test_dir, test_name + ".pass") - thru_path = os.path.join(out_test_dir, test_name + ".thru") - - out_opts = itertools.chain( - ["-i", isyntax], - ["-o", isyntax], - ["-p", "serd_test"], - ) - - out_cmd = ( - command_prefix - + [f for sublist in flags for f in sublist] - + list(out_opts) - + [path, base_uri] - ) - - with open(out_path, "wb") as out: - subprocess.run(out_cmd, check=True, stdout=out) - - thru_opts = itertools.chain( - ["-c", "serd_test"], - ["-i", isyntax], - ["-o", osyntax], - ) - - thru_cmd = command_prefix + list(thru_opts) + [out_path, base_uri] - - proc = subprocess.run( - thru_cmd, check=True, capture_output=True, encoding="utf-8" - ) - - return util.lines_equal( - check_lines, - proc.stdout.splitlines(True), - check_path, - thru_path, - ) - - -def _test_input_syntax(test_class): - """Return the output syntax use for a given test class.""" - - if "NTriples" in test_class: - return "NTriples" - - if "Turtle" in test_class: - return "Turtle" - - if "NQuads" in test_class: - return "NQuads" - - if "Trig" in test_class: - return "Trig" - - raise RuntimeError("Unknown test class: " + test_class) - - -def _test_output_syntax(test_class): - """Return the output syntax use for a given test class.""" - - if "NTriples" in test_class or "Turtle" in test_class: - return "NTriples" - - if "NQuads" in test_class or "Trig" in test_class: - return "NQuads" - - raise RuntimeError("Unknown test class: " + test_class) - - -def _option_combinations(options): - """Return an iterator that cycles through all combinations of options.""" - - combinations = [] - for count in range(len(options) + 1): - combinations += list(itertools.combinations(options, count)) - - return itertools.cycle(combinations) - - -def run_suite( - manifest_path, - base_uri, - report_filename, - input_syntax, - command_prefix, - out_test_dir, -): - """Run all tests in a test suite manifest.""" - - test_dir = os.path.dirname(manifest_path) - model, instances = util.load_rdf(manifest_path, base_uri, command_prefix) - - asserter = "" - if os.getenv("USER") == "drobilla": - asserter = "http://drobilla.net/drobilla#me" - - def run_tests(test_class, tests, expected_return, results): - thru_flags = [["-e"], ["-f"], ["-b"], ["-r", "http://example.org/"]] - osyntax = _test_output_syntax(test_class) - thru_options_iter = _option_combinations(thru_flags) - - if input_syntax is not None: - isyntax = input_syntax - else: - isyntax = _test_input_syntax(test_class) - - for test in sorted(tests): - test_uri = model[test][NS_MF + "action"][0] - test_uri_path = util.uri_path(test_uri) - test_name = os.path.basename(test_uri_path) - test_path = os.path.join(test_dir, test_name) - - command = command_prefix + ["-f", test_path, test_uri] - command_string = " ".join(shlex.quote(c) for c in command) - out_filename = os.path.join(out_test_dir, test_name + ".out") - - if expected_return == 0: # Positive test - with tempfile.TemporaryFile("w+", encoding="utf-8") as out: - proc = subprocess.run(command, check=False, stdout=out) - passed = proc.returncode == 0 - results.check( - passed, "Unexpected failure: " + command_string - ) - - if ( - proc.returncode == 0 - and NS_MF + "result" in model[test] - ): - # Check output against expected output from test suite - check_uri = model[test][NS_MF + "result"][0] - check_filename = os.path.basename( - util.uri_path(check_uri) - ) - check_path = os.path.join(test_dir, check_filename) - - with open(check_path, "r", encoding="utf-8") as check: - check_lines = check.readlines() - - out.seek(0) - results.check( - util.lines_equal( - check_lines, - list(out), - check_path, - out_filename, - ) - ) - - # Run round-trip test - check.seek(0) - results.check( - test_thru( - test_uri, - test_path, - check_lines, - check_path, - out_test_dir, - list(next(thru_options_iter)), - isyntax, - osyntax, - command_prefix, - ), - "Corrupted round-trip: " + test_uri, - ) - - else: # Negative test - with tempfile.TemporaryFile() as stderr: - proc = subprocess.run( - command, - check=False, - stdout=subprocess.DEVNULL, - stderr=stderr, - ) - - passed = proc.returncode != 0 - results.check( - passed, "Unexpected success: " + command_string - ) - - # Check that an error message was printed - stderr.seek(0, 2) # Seek to end - results.check( - stderr.tell() > 0, - "No error message printed: " + command_string, - ) - - # Write test report entry - if report_filename: - with open(report_filename, "a", encoding="utf-8") as report: - report.write(util.earl_assertion(test, passed, asserter)) - - # Run all test types in the test suite - results = util.Results() - for test_class, instances in instances.items(): - if test_class.startswith(NS_RDFT): - expected = ( - 1 - if "-l" not in command_prefix and "Negative" in test_class - else 0 - ) - run_tests(test_class, instances, expected, results) - - return util.print_result_summary(results) - - -def main(): - """Run the command line tool.""" - - parser = argparse.ArgumentParser( - usage="%(prog)s [OPTION]... MANIFEST BASE_URI -- [SERDI_OPTION]...", - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - - parser.add_argument("--report", help="path to write result report to") - parser.add_argument("--serdi", default="serdi", help="path to serdi") - parser.add_argument("--syntax", default=None, help="input syntax") - parser.add_argument("--wrapper", default="", help="executable wrapper") - parser.add_argument("manifest", help="test suite manifest.ttl file") - parser.add_argument("base_uri", help="base URI for tests") - parser.add_argument( - "serdi_option", nargs=argparse.REMAINDER, help="option for serdi" - ) - - args = parser.parse_args(sys.argv[1:]) - command_prefix = ( - shlex.split(args.wrapper) + [args.serdi] + args.serdi_option - ) - - with tempfile.TemporaryDirectory() as test_out_dir: - return run_suite( - args.manifest, - args.base_uri, - args.report, - args.syntax, - command_prefix, - test_out_dir, - ) - - -if __name__ == "__main__": - try: - sys.exit(main()) - except subprocess.CalledProcessError as e: - if e.stderr is not None: - sys.stderr.write(e.stderr) - - sys.stderr.write("error: ") - sys.stderr.write(str(e)) - sys.stderr.write("\n") - sys.exit(e.returncode) |