diff options
| -rw-r--r-- | test/lax/README.md | 5 | ||||
| -rw-r--r-- | test/meson.build | 26 | ||||
| -rwxr-xr-x | test/run_suite.py | 42 |
3 files changed, 52 insertions, 21 deletions
diff --git a/test/lax/README.md b/test/lax/README.md new file mode 100644 index 00000000..d1b5d375 --- /dev/null +++ b/test/lax/README.md @@ -0,0 +1,5 @@ +Lax Test Suite +============== + +This suite contains negative tests that can be successfully read (with +warnings) if and only if lax parsing is enabled. diff --git a/test/meson.build b/test/meson.build index e4850f0c..f5282672 100644 --- a/test/meson.build +++ b/test/meson.build @@ -195,6 +195,14 @@ ns_serdtest = 'http://drobilla.net/sw/serd/test/' ns_w3 = 'http://www.w3.org/2013/' test_suites = { + 'lax.lax': [ + '--lax', + files('lax/manifest.ttl'), ns_serdtest + 'lax/', + '--', '-l', + ], + 'lax.strict': [ + files('lax/manifest.ttl'), ns_serdtest + 'lax/', + ], 'prefix.add': [ '--reverse', files('prefix/manifest.ttl'), ns_serdtest + 'prefix/', @@ -253,24 +261,6 @@ if not get_option('tools').disabled() suite: ['rdf', 'serd'], timeout: 240) - ### The lax suite is special because it is run twice... - lax_manifest = files('lax/manifest.ttl') - lax_base_uri = ns_serdtest + 'lax/' - - ### ... once with strict parsing to test the hard errors - test('lax.strict', run_test_suite, - args: script_args + [lax_manifest, lax_base_uri], - is_parallel: false, - suite: ['rdf', 'serd'], - timeout: 240) - - ### ... and once with lax parsing to tolerate them - test('lax.lax', run_test_suite, - args: script_args + [lax_manifest, lax_base_uri, '--', '-l'], - is_parallel: false, - suite: ['rdf', 'serd'], - timeout: 240) - ## Standard W3C test suites w3c_suites = ['Turtle', 'NTriples', 'NQuads', 'TriG'] diff --git a/test/run_suite.py b/test/run_suite.py index 5c74c9c7..6426b0c2 100755 --- a/test/run_suite.py +++ b/test/run_suite.py @@ -15,13 +15,16 @@ import tempfile import serd_test_util as util NS_MF = "http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#" +NS_RDF = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" NS_RDFT = "http://www.w3.org/ns/rdftest#" DEVNULL = subprocess.DEVNULL PIPE = subprocess.PIPE TEST_TYPES = [ + NS_RDFT + "TestNQuadsNegativeSyntax", NS_RDFT + "TestNQuadsPositiveSyntax", + NS_RDFT + "TestNTriplesNegativeSyntax", NS_RDFT + "TestNTriplesPositiveSyntax", NS_RDFT + "TestTrigEval", NS_RDFT + "TestTrigNegativeEval", @@ -49,16 +52,48 @@ def run_eval_test(base_uri, command, in_path, good_path, out_path): return util.lines_equal(list(good), out, good_path, out_path) +def run_positive_test(base_uri, command, in_path): + """Run a positive syntax test and ensure no errors occur.""" + + command = command + [in_path, base_uri] + subprocess.check_call(command, encoding="utf-8", stdout=DEVNULL) + return True + + +def run_negative_test(base_uri, command, in_path): + """Run a negative syntax test and return whether the error was detected.""" + + if not os.path.exists(in_path): + raise RuntimeError("Input file missing: " + in_path) + + command = command + [in_path, base_uri] + proc = subprocess.run(command, check=False, stderr=PIPE, stdout=DEVNULL) + + if proc.returncode == 0: + util.error("Unexpected successful return: " + in_path) + return False + + if len(proc.stderr) == 0: + util.error("Command failed with no error output: " + in_path) + return False + + return True + + def run_entry(args, entry, command, out_dir, suite_dir): """Run a single test entry from the manifest.""" in_path = util.file_path(suite_dir, entry[NS_MF + "action"][0]) base = args.base_uri + os.path.basename(in_path) - good_path = in_path - if NS_MF + "result" in entry: - good_path = util.file_path(suite_dir, entry[NS_MF + "result"][0]) + negative = "Negative" in entry[NS_RDF + "type"][0] + if negative and not args.lax: + return run_negative_test(base, command, in_path) + + if NS_MF + "result" not in entry: + return run_positive_test(base, command, in_path) + good_path = util.file_path(suite_dir, entry[NS_MF + "result"][0]) if args.reverse: in_path, good_path = good_path, in_path @@ -109,6 +144,7 @@ def main(): description=__doc__, ) + parser.add_argument("--lax", action="store_true", help="tolerate errors") parser.add_argument("--reverse", action="store_true", help="reverse test") parser.add_argument("--serdi", default="serdi", help="path to serdi") parser.add_argument("--wrapper", default="", help="executable wrapper") |