From d23dbcc92592fb68f3294889d2b8fac9511094ca Mon Sep 17 00:00:00 2001 From: David Robillard Date: Fri, 15 Mar 2019 23:57:07 +0100 Subject: WIP: Rewrite test framework --- wscript | 274 ++++++++++++++++++++++++++-------------------------------------- 1 file changed, 109 insertions(+), 165 deletions(-) (limited to 'wscript') diff --git a/wscript b/wscript index 425ca787..2c084733 100644 --- a/wscript +++ b/wscript @@ -3,6 +3,7 @@ import glob import io import os + from waflib import Logs, Options from waflib.extras import autowaf @@ -216,18 +217,6 @@ def upload_docs(ctx): os.system('soelim %s | pre-grohtml troff -man -wall -Thtml | post-grohtml > build/%s.html' % (page, page)) os.system('rsync -avz --delete -e ssh build/%s.html drobilla@drobilla.net:~/drobilla.net/man/' % page) -def file_equals(patha, pathb): - import filecmp - - if filecmp.cmp(patha, pathb, shallow=False): - return True - - with io.open(patha, 'rU', encoding='utf-8') as fa: - with io.open(pathb, 'rU', encoding='utf-8') as fb: - show_diff(fa.readlines(), fb.readlines(), patha, pathb) - - return False - def earl_assertion(test, passed, asserter): import datetime @@ -235,10 +224,6 @@ def earl_assertion(test, passed, asserter): if asserter is not None: asserter_str = '\n\tearl:assertedBy <%s> ;' % asserter - passed_str = 'earl:failed' - if passed: - passed_str = 'earl:passed' - return ''' [] a earl:Assertion ;%s @@ -251,49 +236,33 @@ def earl_assertion(test, passed, asserter): ] . ''' % (asserter_str, test, - passed_str, + 'earl:passed' if passed else 'earl:failed', datetime.datetime.now().replace(microsecond=0).isoformat()) -def build_path(ctx, path): - return os.path.relpath(path, os.getcwd()) - -def show_diff(from_lines, to_lines, from_filename, to_filename): - import difflib - import sys - - for line in difflib.unified_diff( - from_lines, to_lines, - fromfile=os.path.abspath(from_filename), - tofile=os.path.abspath(to_filename)): - sys.stderr.write(line) - -def check_output(out_filename, check_filename): - if not os.access(out_filename, os.F_OK): - Logs.pprint('RED', 'error: missing output file %s' % out_filename) - return False - - return file_equals(check_filename, out_filename) - -def test_thru(ctx, base, path, check_filename, flags, isyntax, osyntax, - options='', quiet=False): - in_filename = build_path(ctx, os.path.join(ctx.path.abspath(), path)) - out_filename = build_path(ctx, path + '.thru') - - command = ('serdi_static %s %s -i %s -o %s -p foo "%s" "%s" | ' - 'serdi_static %s -i %s -o %s -c foo - "%s" > %s') % ( - options, flags.ljust(5), - isyntax, isyntax, in_filename, base, - options, isyntax, osyntax, base, out_filename) - - if autowaf.run_test(ctx, APPNAME, command, 0, name=out_filename, quiet=quiet): - autowaf.run_test( - ctx, APPNAME, - lambda: check_output(out_filename, check_filename), - True, - name=out_filename, - quiet=quiet) - else: - Logs.pprint('RED', 'FAIL: error running %s' % command) +serdi = './serdi_static' + +def test_thru(group, base, path, check_path, flags, isyntax, osyntax, opts=[]): + out_path = path + '.out' + out_cmd = [serdi] + opts + [f for sublist in flags for f in sublist] + [ + '-i', isyntax, + '-o', isyntax, + '-p', 'foo', + group.tst.src_path(path), base] + + thru_path = path + '.thru' + thru_cmd = [serdi] + opts + [ + '-i', isyntax, + '-o', osyntax, + '-c', 'foo', + out_path, + base] + + return (group.run(out_cmd, stdout=out_path, verbosity=0, name=out_path) and + group.run(thru_cmd, stdout=thru_path, verbosity=0, name=thru_path) and + group.run(lambda: autowaf.test_file_equals(thru_path, check_path), + expected=True, + verbosity=0, + name='%s == %s' % (thru_path, check_path))) def file_uri_to_path(uri): try: @@ -305,7 +274,7 @@ def file_uri_to_path(uri): drive = os.path.splitdrive(path[1:])[0] return path if not drive else path[1:] -def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''): +def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=[]): import itertools srcdir = ctx.path.abspath() @@ -315,7 +284,7 @@ def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''): import subprocess import re model = {} - proc = subprocess.Popen(['./serdi_static', filename], stdout=subprocess.PIPE) + proc = subprocess.Popen([serdi, filename], stdout=subprocess.PIPE) for line in proc.communicate()[0].splitlines(): matches = re.match('<([^ ]*)> <([^ ]*)> <([^ ]*)> \.', line.decode('utf-8')) if matches: @@ -333,14 +302,7 @@ def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''): if os.getenv('USER') == 'drobilla': asserter = 'http://drobilla.net/drobilla#me' - def run_test(command, expected_return, name, quiet=False): - header = Options.options.verbose - result = autowaf.run_test(ctx, APPNAME, command, expected_return, name=name, header=header, quiet=quiet) - if expected_return is not None and expected_return != 0: - autowaf.run_test(ctx, APPNAME, - lambda: bool(result[1][1]), - True, name=name + ' prints error message', quiet=True) - return result + verbosity = 2 if Options.options.verbose else 0 def run_tests(test_class, expected_return): tests = [] @@ -350,68 +312,66 @@ def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''): if len(tests) == 0: return - thru_flags = ['-e', '-f', '-b', '-r http://example.org/'] + thru_flags = [['-e'], ['-f'], ['-b'], ['-r', 'http://example.org/']] thru_options = [] for n in range(len(thru_flags) + 1): thru_options += list(itertools.combinations(thru_flags, n)) thru_options_iter = itertools.cycle(thru_options) - quiet = not Options.options.verbose tests_name = '%s.%s' % (testdir, test_class[test_class.find('#') + 1:]) - with autowaf.begin_tests(ctx, APPNAME, tests_name): + with ctx.test_group(tests_name) as group: for (num, test) in enumerate(sorted(tests)): action_node = model[test][mf + 'action'][0] action = os.path.join('tests', testdir, os.path.basename(action_node)) rel_action = os.path.join(os.path.relpath(srcdir), action) - abs_action = os.path.join(srcdir, action) uri = base_uri + os.path.basename(action) - command = 'serdi_static %s -f %s "%s" > %s' % ( - options, rel_action, uri, action + '.out') + command = [serdi] + options + ['-f', rel_action, uri] # Run strict test - result = run_test(command, expected_return, action, quiet=quiet) - if result[0] and ((mf + 'result') in model[test]): + if expected_return == 0: + result = group.run(command, stdout=action + '.out', name=action) + else: + result = group.run(command, + stdout=action + '.out', + stderr=autowaf.NONEMPTY, + expected=expected_return, + name=action) + + if result and ((mf + 'result') in model[test]): # Check output against test suite check_uri = model[test][mf + 'result'][0] - check_path = build_path(ctx, file_uri_to_path(check_uri)) - result = autowaf.run_test( - ctx, APPNAME, - lambda: check_output(action + '.out', check_path), - True, name=action, quiet=True) + check_path = ctx.src_path(file_uri_to_path(check_uri)) + result = group.run( + lambda: autowaf.test_file_equals(action + '.out', check_path), + expected=True, name=action) # Run round-trip tests - if result[0]: - test_thru(ctx, uri, action, check_path, - ' '.join(next(thru_options_iter)), - isyntax, osyntax, options, quiet=True) + if result: + test_thru(group, uri, action, check_path, + list(next(thru_options_iter)), + isyntax, osyntax, options) # Write test report entry if report is not None: - report.write(earl_assertion(test, result[0], asserter)) + report.write(earl_assertion(test, result, asserter)) # Run lax test - run_test(command.replace('serdi_static', 'serdi_static -l'), - None, action + ' lax', True) - - def test_types(): - types = [] - for lang in ['Turtle', 'NTriples', 'Trig', 'NQuads']: - types += [['http://www.w3.org/ns/rdftest#Test%sPositiveSyntax' % lang, 0], - ['http://www.w3.org/ns/rdftest#Test%sNegativeSyntax' % lang, 1], - ['http://www.w3.org/ns/rdftest#Test%sNegativeEval' % lang, 1], - ['http://www.w3.org/ns/rdftest#Test%sEval' % lang, 0]] - return types - - for i in test_types(): + group.run([command[0]] + ['-l'] + command[1:], + expected=None, name=action + ' lax') + + for i in (('http://www.w3.org/ns/rdftest#Test%sPositiveSyntax' % isyntax, 0), + ('http://www.w3.org/ns/rdftest#Test%sEval' % isyntax, 0), + ('http://www.w3.org/ns/rdftest#Test%sNegativeSyntax' % isyntax, 1), + ('http://www.w3.org/ns/rdftest#Test%sNegativeEval' % isyntax, 1)): run_tests(i[0], i[1]) def test(ctx): - "runs test suite" - + import tempfile + # Create test output directories for i in ['bad', 'good', 'TurtleTests', 'NTriplesTests', 'NQuadsTests', 'TriGTests']: try: - test_dir = os.path.join(autowaf.build_dir(APPNAME, 'tests'), i) + test_dir = os.path.join('tests', i) os.makedirs(test_dir) for i in glob.glob(test_dir + '/*.*'): os.remove(i) @@ -419,70 +379,57 @@ def test(ctx): pass srcdir = ctx.path.abspath() - os.environ['PATH'] = '.' + os.pathsep + os.getenv('PATH') - autowaf.pre_test(ctx, APPNAME) - autowaf.run_tests(ctx, APPNAME, ['serd_test'], name='Unit') + with ctx.test_group('Unit') as tests: + tests.run(['./serd_test']) - def test_syntax_io(in_name, expected_name, lang): + def test_syntax_io(test, in_name, check_name, lang): in_path = 'tests/good/%s' % in_name - autowaf.run_test( - ctx, APPNAME, - 'serdi_static -o %s "%s/%s" "%s" > %s.out' % ( - lang, srcdir, in_path, in_path, in_path), - 0, name=in_name) - - autowaf.run_test( - ctx, APPNAME, - lambda: file_equals('%s/tests/good/%s' % (srcdir, expected_name), - '%s.out' % in_path), - True, quiet=True, name=in_name + '-check') - - with autowaf.begin_tests(ctx, APPNAME, 'ThroughSyntax'): - test_syntax_io('base.ttl', 'base.ttl', 'turtle') - test_syntax_io('qualify-in.ttl', 'qualify-out.ttl', 'turtle') - - nul = os.devnull - autowaf.run_tests(ctx, APPNAME, [ - 'serdi_static %s/tests/good/manifest.ttl > %s' % (srcdir, nul), - 'serdi_static -v > %s' % nul, - 'serdi_static -h > %s' % nul, - 'serdi_static -s " a <#Thingie> ." > %s' % nul, - 'serdi_static %s > %s' % (nul, nul) - ], 0, name='GoodCommands') - - autowaf.run_tests(ctx, APPNAME, [ - 'serdi_static -q %s/tests/bad/bad-id-clash.ttl > %s' % (srcdir, nul), - 'serdi_static > %s' % nul, - 'serdi_static ftp://example.org/unsupported.ttl > %s' % nul, - 'serdi_static -i > %s' % nul, - 'serdi_static -o > %s' % nul, - 'serdi_static -z > %s' % nul, - 'serdi_static -p > %s' % nul, - 'serdi_static -c > %s' % nul, - 'serdi_static -r > %s' % nul, - 'serdi_static -i illegal > %s' % nul, - 'serdi_static -o illegal > %s' % nul, - 'serdi_static -i turtle > %s' % nul, - 'serdi_static /no/such/file > %s' % nul], - 1, name='BadCommands') - - with autowaf.begin_tests(ctx, APPNAME, 'IoErrors'): - # Test read error by reading a directory - autowaf.run_test(ctx, APPNAME, 'serdi_static -e "file://%s/"' % srcdir, - 1, name='read_error') - - # Test read error with bulk input by reading a directory - autowaf.run_test(ctx, APPNAME, 'serdi_static "file://%s/"' % srcdir, - 1, name='read_error_bulk') - - # Test write error by writing to /dev/full + out_path = in_path + '.out' + check_path = '%s/tests/good/%s' % (srcdir, check_name) + + test.run([serdi, '-o', lang, '%s/%s' % (srcdir, in_path), in_path], + stdout=out_path, name=in_name) + + test.run(lambda: autowaf.test_file_equals(check_path, out_path), + expected=True, name='%s check' % in_name) + + with ctx.test_group('ThroughSyntax') as tests: + test_syntax_io(tests, 'base.ttl', 'base.ttl', 'turtle') + test_syntax_io(tests, 'qualify-in.ttl', 'qualify-out.ttl', 'turtle') + + with ctx.test_group('GoodCommands') as tests: + tests.run([serdi, '%s/tests/good/manifest.ttl' % srcdir]) + tests.run([serdi, '-v']) + tests.run([serdi, '-h']) + tests.run([serdi, '-s', ' a <#Thingie> .']) + tests.run([serdi, os.devnull]) + with tempfile.TemporaryFile(mode='r') as stdin: + tests.run([serdi, '-'], stdin=stdin) + + with ctx.test_group('BadCommands', expected=1) as tests: + tests.run([serdi]) + tests.run([serdi, '/no/such/file']) + tests.run([serdi, 'ftp://example.org/unsupported.ttl']) + tests.run([serdi, '-c']) + tests.run([serdi, '-i', 'illegal']) + tests.run([serdi, '-i', 'turtle']) + tests.run([serdi, '-i']) + tests.run([serdi, '-o', 'illegal']) + tests.run([serdi, '-o']) + tests.run([serdi, '-p']) + tests.run([serdi, '-q', '%s/tests/bad/bad-id-clash.ttl' % srcdir]) + tests.run([serdi, '-r']) + tests.run([serdi, '-z']) + + with ctx.test_group('IoErrors', expected=1) as tests: + tests.run([serdi, '-e', 'file://%s/' % srcdir], name='Read directory') + tests.run([serdi, 'file://%s/' % srcdir], name='Bulk read directory') if os.path.exists('/dev/full'): - autowaf.run_test(ctx, APPNAME, - 'serdi_static "file://%s/tests/good/manifest.ttl" > /dev/full' % srcdir, - 1, name='write_error') + tests.run([serdi, 'file://%s/tests/good/manifest.ttl' % srcdir], + stdout='/dev/full', name='Write error') - # Serd-specific test cases + # Serd-specific test suites serd_base = 'http://drobilla.net/sw/serd/tests/' test_suite(ctx, serd_base + 'good/', 'good', None, 'Turtle', 'NTriples') test_suite(ctx, serd_base + 'bad/', 'bad', None, 'Turtle', 'NTriples') @@ -493,8 +440,7 @@ def test(ctx): '@prefix dc: .\n') with open(os.path.join(srcdir, 'serd.ttl')) as serd_ttl: - for line in serd_ttl: - report.write(line) + report.writelines(serd_ttl) w3c_base = 'http://www.w3.org/2013/' test_suite(ctx, w3c_base + 'TurtleTests/', @@ -504,9 +450,7 @@ def test(ctx): test_suite(ctx, w3c_base + 'NQuadsTests/', 'NQuadsTests', report, 'NQuads', 'NQuads') test_suite(ctx, w3c_base + 'TriGTests/', - 'TriGTests', report, 'TriG', 'NQuads', '-a') - - autowaf.post_test(ctx, APPNAME) + 'TriGTests', report, 'Trig', 'NQuads', ['-a']) def posts(ctx): path = str(ctx.path.abspath()) -- cgit v1.2.1