WIP: Rewrite test frameworktest-performance

1 files changed, 109 insertions, 165 deletions

diff --git a/wscript b/wscript
index 425ca787..2c084733 100644
--- a/wscript
+++ b/wscript
@@ -3,6 +3,7 @@
 import glob
 import io
 import os
+
 from waflib import Logs, Options
 from waflib.extras import autowaf
 
@@ -216,18 +217,6 @@ def upload_docs(ctx):
         os.system('soelim %s | pre-grohtml troff -man -wall -Thtml | post-grohtml > build/%s.html' % (page, page))
         os.system('rsync -avz --delete -e ssh build/%s.html drobilla@drobilla.net:~/drobilla.net/man/' % page)
 
-def file_equals(patha, pathb):
-    import filecmp
-
-    if filecmp.cmp(patha, pathb, shallow=False):
-        return True
-
-    with io.open(patha, 'rU', encoding='utf-8') as fa:
-        with io.open(pathb, 'rU', encoding='utf-8') as fb:
-            show_diff(fa.readlines(), fb.readlines(), patha, pathb)
-
-    return False
-
 def earl_assertion(test, passed, asserter):
     import datetime
 
@@ -235,10 +224,6 @@ def earl_assertion(test, passed, asserter):
     if asserter is not None:
         asserter_str = '\n\tearl:assertedBy <%s> ;' % asserter
 
-    passed_str = 'earl:failed'
-    if passed:
-        passed_str = 'earl:passed'
-
     return '''
 []
 	a earl:Assertion ;%s
@@ -251,49 +236,33 @@ def earl_assertion(test, passed, asserter):
 	] .
 ''' % (asserter_str,
        test,
-       passed_str,
+       'earl:passed' if passed else 'earl:failed',
        datetime.datetime.now().replace(microsecond=0).isoformat())
 
-def build_path(ctx, path):
-    return os.path.relpath(path, os.getcwd())
-
-def show_diff(from_lines, to_lines, from_filename, to_filename):
-    import difflib
-    import sys
-
-    for line in difflib.unified_diff(
-            from_lines, to_lines,
-            fromfile=os.path.abspath(from_filename),
-            tofile=os.path.abspath(to_filename)):
-        sys.stderr.write(line)
-
-def check_output(out_filename, check_filename):
-    if not os.access(out_filename, os.F_OK):
-        Logs.pprint('RED', 'error: missing output file %s' % out_filename)
-        return False
-
-    return file_equals(check_filename, out_filename)
-
-def test_thru(ctx, base, path, check_filename, flags, isyntax, osyntax,
-              options='', quiet=False):
-    in_filename = build_path(ctx, os.path.join(ctx.path.abspath(), path))
-    out_filename = build_path(ctx, path + '.thru')
-
-    command = ('serdi_static %s %s -i %s -o %s -p foo "%s" "%s" | '
-               'serdi_static %s -i %s -o %s -c foo - "%s" > %s') % (
-                   options, flags.ljust(5),
-                   isyntax, isyntax, in_filename, base,
-                   options, isyntax, osyntax, base, out_filename)
-
-    if autowaf.run_test(ctx, APPNAME, command, 0, name=out_filename, quiet=quiet):
-        autowaf.run_test(
-            ctx, APPNAME,
-            lambda: check_output(out_filename, check_filename),
-            True,
-            name=out_filename,
-            quiet=quiet)
-    else:
-        Logs.pprint('RED', 'FAIL: error running %s' % command)
+serdi = './serdi_static'
+
+def test_thru(group, base, path, check_path, flags, isyntax, osyntax, opts=[]):
+    out_path = path + '.out'
+    out_cmd = [serdi] + opts + [f for sublist in flags for f in sublist] + [
+        '-i', isyntax,
+        '-o', isyntax,
+        '-p', 'foo',
+        group.tst.src_path(path), base]
+
+    thru_path = path + '.thru'
+    thru_cmd = [serdi] + opts + [
+        '-i', isyntax,
+        '-o', osyntax,
+        '-c', 'foo',
+        out_path,
+        base]
+
+    return (group.run(out_cmd, stdout=out_path, verbosity=0, name=out_path) and
+            group.run(thru_cmd, stdout=thru_path, verbosity=0, name=thru_path) and
+            group.run(lambda: autowaf.test_file_equals(thru_path, check_path),
+                      expected=True,
+                      verbosity=0,
+                      name='%s == %s' % (thru_path, check_path)))
 
 def file_uri_to_path(uri):
     try:
@@ -305,7 +274,7 @@ def file_uri_to_path(uri):
     drive = os.path.splitdrive(path[1:])[0]
     return path if not drive else path[1:]
 
-def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''):
+def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=[]):
     import itertools
 
     srcdir = ctx.path.abspath()
@@ -315,7 +284,7 @@ def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''):
         import subprocess
         import re
         model = {}
-        proc  = subprocess.Popen(['./serdi_static', filename], stdout=subprocess.PIPE)
+        proc  = subprocess.Popen([serdi, filename], stdout=subprocess.PIPE)
         for line in proc.communicate()[0].splitlines():
             matches = re.match('<([^ ]*)> <([^ ]*)> <([^ ]*)> \.', line.decode('utf-8'))
             if matches:
@@ -333,14 +302,7 @@ def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''):
     if os.getenv('USER') == 'drobilla':
         asserter = 'http://drobilla.net/drobilla#me'
 
-    def run_test(command, expected_return, name, quiet=False):
-        header = Options.options.verbose
-        result = autowaf.run_test(ctx, APPNAME, command, expected_return, name=name, header=header, quiet=quiet)
-        if expected_return is not None and expected_return != 0:
-            autowaf.run_test(ctx, APPNAME,
-                             lambda: bool(result[1][1]),
-                             True, name=name + ' prints error message', quiet=True)
-        return result
+    verbosity = 2 if Options.options.verbose else 0
 
     def run_tests(test_class, expected_return):
         tests = []
@@ -350,68 +312,66 @@ def test_suite(ctx, base_uri, testdir, report, isyntax, osyntax, options=''):
         if len(tests) == 0:
             return
 
-        thru_flags   = ['-e', '-f', '-b', '-r http://example.org/']
+        thru_flags   = [['-e'], ['-f'], ['-b'], ['-r', 'http://example.org/']]
         thru_options = []
         for n in range(len(thru_flags) + 1):
             thru_options += list(itertools.combinations(thru_flags, n))
         thru_options_iter = itertools.cycle(thru_options)
 
-        quiet = not Options.options.verbose
         tests_name = '%s.%s' % (testdir, test_class[test_class.find('#') + 1:])
-        with autowaf.begin_tests(ctx, APPNAME, tests_name):
+        with ctx.test_group(tests_name) as group:
             for (num, test) in enumerate(sorted(tests)):
                 action_node = model[test][mf + 'action'][0]
                 action      = os.path.join('tests', testdir, os.path.basename(action_node))
                 rel_action  = os.path.join(os.path.relpath(srcdir), action)
-                abs_action  = os.path.join(srcdir, action)
                 uri         = base_uri + os.path.basename(action)
-                command     = 'serdi_static %s -f %s "%s" > %s' % (
-                    options, rel_action, uri, action + '.out')
+                command     = [serdi] + options + ['-f', rel_action, uri]
 
                 # Run strict test
-                result = run_test(command, expected_return, action, quiet=quiet)
-                if result[0] and ((mf + 'result') in model[test]):
+                if expected_return == 0:
+                    result = group.run(command, stdout=action + '.out', name=action)
+                else:
+                    result = group.run(command,
+                                       stdout=action + '.out',
+                                       stderr=autowaf.NONEMPTY,
+                                       expected=expected_return,
+                                       name=action)
+
+                if result and ((mf + 'result') in model[test]):
                     # Check output against test suite
                     check_uri  = model[test][mf + 'result'][0]
-                    check_path = build_path(ctx, file_uri_to_path(check_uri))
-                    result     = autowaf.run_test(
-                        ctx, APPNAME,
-                        lambda: check_output(action + '.out', check_path),
-                        True, name=action, quiet=True)
+                    check_path = ctx.src_path(file_uri_to_path(check_uri))
+                    result     = group.run(
+                        lambda: autowaf.test_file_equals(action + '.out', check_path),
+                        expected=True, name=action)
 
                     # Run round-trip tests
-                    if result[0]:
-                        test_thru(ctx, uri, action, check_path,
-                                  ' '.join(next(thru_options_iter)),
-                                  isyntax, osyntax, options, quiet=True)
+                    if result:
+                        test_thru(group, uri, action, check_path,
+                                  list(next(thru_options_iter)),
+                                  isyntax, osyntax, options)
 
                 # Write test report entry
                 if report is not None:
-                    report.write(earl_assertion(test, result[0], asserter))
+                    report.write(earl_assertion(test, result, asserter))
 
                 # Run lax test
-                run_test(command.replace('serdi_static', 'serdi_static -l'),
-                         None, action + ' lax', True)
-
-    def test_types():
-        types = []
-        for lang in ['Turtle', 'NTriples', 'Trig', 'NQuads']:
-            types += [['http://www.w3.org/ns/rdftest#Test%sPositiveSyntax' % lang, 0],
-                      ['http://www.w3.org/ns/rdftest#Test%sNegativeSyntax' % lang, 1],
-                      ['http://www.w3.org/ns/rdftest#Test%sNegativeEval' % lang, 1],
-                      ['http://www.w3.org/ns/rdftest#Test%sEval' % lang, 0]]
-        return types
-
-    for i in test_types():
+                group.run([command[0]] + ['-l'] + command[1:],
+                          expected=None, name=action + ' lax')
+
+    for i in (('http://www.w3.org/ns/rdftest#Test%sPositiveSyntax' % isyntax, 0),
+              ('http://www.w3.org/ns/rdftest#Test%sEval' % isyntax, 0),
+              ('http://www.w3.org/ns/rdftest#Test%sNegativeSyntax' % isyntax, 1),
+              ('http://www.w3.org/ns/rdftest#Test%sNegativeEval' % isyntax, 1)):
         run_tests(i[0], i[1])
 
 def test(ctx):
-    "runs test suite"
-
+    import tempfile
+    
     # Create test output directories
     for i in ['bad', 'good', 'TurtleTests', 'NTriplesTests', 'NQuadsTests', 'TriGTests']:
         try:
-            test_dir = os.path.join(autowaf.build_dir(APPNAME, 'tests'), i)
+            test_dir = os.path.join('tests', i)
             os.makedirs(test_dir)
             for i in glob.glob(test_dir + '/*.*'):
                 os.remove(i)
@@ -419,70 +379,57 @@ def test(ctx):
             pass
 
     srcdir = ctx.path.abspath()
-    os.environ['PATH'] = '.' + os.pathsep + os.getenv('PATH')
 
-    autowaf.pre_test(ctx, APPNAME)
-    autowaf.run_tests(ctx, APPNAME, ['serd_test'], name='Unit')
+    with ctx.test_group('Unit') as tests:
+        tests.run(['./serd_test'])
 
-    def test_syntax_io(in_name, expected_name, lang):
+    def test_syntax_io(test, in_name, check_name, lang):
         in_path = 'tests/good/%s' % in_name
-        autowaf.run_test(
-            ctx, APPNAME,
-            'serdi_static -o %s "%s/%s" "%s" > %s.out' % (
-                lang, srcdir, in_path, in_path, in_path),
-            0, name=in_name)
-
-        autowaf.run_test(
-            ctx, APPNAME,
-            lambda: file_equals('%s/tests/good/%s' % (srcdir, expected_name),
-                                '%s.out' % in_path),
-            True, quiet=True, name=in_name + '-check')
-
-    with autowaf.begin_tests(ctx, APPNAME, 'ThroughSyntax'):
-        test_syntax_io('base.ttl',       'base.ttl',        'turtle')
-        test_syntax_io('qualify-in.ttl', 'qualify-out.ttl', 'turtle')
-
-    nul = os.devnull
-    autowaf.run_tests(ctx, APPNAME, [
-            'serdi_static %s/tests/good/manifest.ttl > %s' % (srcdir, nul),
-            'serdi_static -v > %s' % nul,
-            'serdi_static -h > %s' % nul,
-            'serdi_static -s "<foo> a <#Thingie> ." > %s' % nul,
-            'serdi_static %s > %s' % (nul, nul)
-    ], 0, name='GoodCommands')
-
-    autowaf.run_tests(ctx, APPNAME, [
-            'serdi_static -q %s/tests/bad/bad-id-clash.ttl > %s' % (srcdir, nul),
-            'serdi_static > %s' % nul,
-            'serdi_static ftp://example.org/unsupported.ttl > %s' % nul,
-            'serdi_static -i > %s' % nul,
-            'serdi_static -o > %s' % nul,
-            'serdi_static -z > %s' % nul,
-            'serdi_static -p > %s' % nul,
-            'serdi_static -c > %s' % nul,
-            'serdi_static -r > %s' % nul,
-            'serdi_static -i illegal > %s' % nul,
-            'serdi_static -o illegal > %s' % nul,
-            'serdi_static -i turtle > %s' % nul,
-            'serdi_static /no/such/file > %s' % nul],
-                      1, name='BadCommands')
-
-    with autowaf.begin_tests(ctx, APPNAME, 'IoErrors'):
-        # Test read error by reading a directory
-        autowaf.run_test(ctx, APPNAME, 'serdi_static -e "file://%s/"' % srcdir,
-                         1, name='read_error')
-
-        # Test read error with bulk input by reading a directory
-        autowaf.run_test(ctx, APPNAME, 'serdi_static "file://%s/"' % srcdir,
-                         1, name='read_error_bulk')
-
-        # Test write error by writing to /dev/full
+        out_path = in_path + '.out'
+        check_path = '%s/tests/good/%s' % (srcdir, check_name)
+
+        test.run([serdi, '-o', lang, '%s/%s' % (srcdir, in_path), in_path],
+                 stdout=out_path, name=in_name)
+
+        test.run(lambda: autowaf.test_file_equals(check_path, out_path),
+                 expected=True, name='%s check' % in_name)
+
+    with ctx.test_group('ThroughSyntax') as tests:
+        test_syntax_io(tests, 'base.ttl',       'base.ttl',        'turtle')
+        test_syntax_io(tests, 'qualify-in.ttl', 'qualify-out.ttl', 'turtle')
+
+    with ctx.test_group('GoodCommands') as tests:
+        tests.run([serdi, '%s/tests/good/manifest.ttl' % srcdir])
+        tests.run([serdi, '-v'])
+        tests.run([serdi, '-h'])
+        tests.run([serdi, '-s', '<foo> a <#Thingie> .'])
+        tests.run([serdi, os.devnull])
+        with tempfile.TemporaryFile(mode='r') as stdin:
+            tests.run([serdi, '-'], stdin=stdin)
+
+    with ctx.test_group('BadCommands', expected=1) as tests:
+        tests.run([serdi])
+        tests.run([serdi, '/no/such/file'])
+        tests.run([serdi, 'ftp://example.org/unsupported.ttl'])
+        tests.run([serdi, '-c'])
+        tests.run([serdi, '-i', 'illegal'])
+        tests.run([serdi, '-i', 'turtle'])
+        tests.run([serdi, '-i'])
+        tests.run([serdi, '-o', 'illegal'])
+        tests.run([serdi, '-o'])
+        tests.run([serdi, '-p'])
+        tests.run([serdi, '-q', '%s/tests/bad/bad-id-clash.ttl' % srcdir])
+        tests.run([serdi, '-r'])
+        tests.run([serdi, '-z'])
+
+    with ctx.test_group('IoErrors', expected=1) as tests:
+        tests.run([serdi, '-e', 'file://%s/' % srcdir], name='Read directory')
+        tests.run([serdi, 'file://%s/' % srcdir], name='Bulk read directory')
         if os.path.exists('/dev/full'):
-            autowaf.run_test(ctx, APPNAME,
-                             'serdi_static "file://%s/tests/good/manifest.ttl" > /dev/full' % srcdir,
-                             1, name='write_error')
+            tests.run([serdi, 'file://%s/tests/good/manifest.ttl' % srcdir],
+                      stdout='/dev/full', name='Write error')
 
-    # Serd-specific test cases
+    # Serd-specific test suites
     serd_base = 'http://drobilla.net/sw/serd/tests/'
     test_suite(ctx, serd_base + 'good/', 'good', None, 'Turtle', 'NTriples')
     test_suite(ctx, serd_base + 'bad/', 'bad', None, 'Turtle', 'NTriples')
@@ -493,8 +440,7 @@ def test(ctx):
                      '@prefix dc: <http://purl.org/dc/elements/1.1/> .\n')
 
         with open(os.path.join(srcdir, 'serd.ttl')) as serd_ttl:
-            for line in serd_ttl:
-                report.write(line)
+            report.writelines(serd_ttl)
 
         w3c_base = 'http://www.w3.org/2013/'
         test_suite(ctx, w3c_base + 'TurtleTests/',
@@ -504,9 +450,7 @@ def test(ctx):
         test_suite(ctx, w3c_base + 'NQuadsTests/',
                    'NQuadsTests', report, 'NQuads', 'NQuads')
         test_suite(ctx, w3c_base + 'TriGTests/',
-                   'TriGTests', report, 'TriG', 'NQuads', '-a')
-
-    autowaf.post_test(ctx, APPNAME)
+                   'TriGTests', report, 'Trig', 'NQuads', ['-a'])
 
 def posts(ctx):
     path = str(ctx.path.abspath())