Squashed 'waflib/' changes from d7a7ca4..27a69a7

27a69a7 Add option to filter tests by regular expression
ac29b74 Fix configuring with root prefix
6134be6 Print test coverage summary
6d7dd59 Don't attempt to clear coverage when coverage is disabled
cddbcb5 Pass file comparison test if difflib finds no changes
6e4e92b Seek to end before checking the size of stderr output
e7a29b6 Upgrade to waf 2.0.15
8280f9d Add command for running executables from the build directory
8073c1a Make make_simple_dox() safe in case of exception
70d03b8 Avoid use of global counter hacks for configuration display
b7d689a Rewrite test framework
94deadf Automatically add options and move add_flags() to options context
f4259ee Reduce system include path noise
927b608 Automatically display configuration header
c44b8f3 Set line justification from a constant in the wscript
a48e26f Automatically detect if wscript has a test hook
ef66724 Save runtime variables in the environment
63bcbcd Clean up TestContext
b1d9505 Add ExecutionContext for setting runtime environment
387c1df Add show_diff() and test_file_equals() utilities
29d4d29 Fix in-tree library paths
9fde01f Add custom configuration context
6d3612f Add lib_path_name constant
915dcb1 Report test time correctly
26f0a2b Remove redundant verbose-tests option
fc35c2c Fix MSVC CFLAGS
5ea8f99 Improve test output spacing
0e23b29 Raise exception when test suite fails to ensure non-zero exit status
d6de073 Show run time of unit tests
5b65554 Add short configure option for ultra-strict flags
4687ba6 Use gtest-like test output
258903d Fix failure count in test group summaries
da07e73 Fix verbose tests with Python 3
6e726eb Add support for suppressing warnings from system libraries
3e43d09 Add short forms for common options
5db477c Automatically define version for child libraries
982416b Fix version definition for projects with special characters
9eb0cf8 Remove blank line padding around configuration summary

git-subtree-dir: waflib
git-subtree-split: 27a69a76d1f625ad86b60b5dbd12368bc25cae25

20 files changed, 663 insertions, 344 deletions

diff --git a/extras/autowaf.py b/extras/autowaf.py
index a844bf5..5feef03 100644
--- a/extras/autowaf.py
+++ b/extras/autowaf.py
@@ -2,41 +2,48 @@ import glob
 import os
 import subprocess
 import sys
+import time
 
-from waflib import Build, Context, Logs, Options, Utils
+from waflib import Configure, ConfigSet, Build, Context, Logs, Options, Utils
 from waflib.TaskGen import feature, before, after
 
 global g_is_child
 g_is_child = False
 
-# Only run autowaf hooks once (even if sub projects call several times)
-global g_step
-g_step = 0
+NONEMPTY = -10
 
-global line_just
-line_just = 40
+if sys.platform == 'win32':
+    lib_path_name = 'PATH'
+elif sys.platform == 'darwin':
+    lib_path_name = 'DYLD_LIBRARY_PATH'
+else:
+    lib_path_name = 'LD_LIBRARY_PATH'
 
 # Compute dependencies globally
 # import preproc
 # preproc.go_absolute = True
 
-# Test context that inherits build context to make configuration available
-class TestContext(Build.BuildContext):
-    "Run tests"
-    cmd = 'test'
-    fun = 'test'
-
 @feature('c', 'cxx')
 @after('apply_incpaths')
 def include_config_h(self):
     self.env.append_value('INCPATHS', self.bld.bldnode.abspath())
 
-def set_options(opt, debug_by_default=False, test=False):
-    "Add standard autowaf options if they havn't been added yet"
-    global g_step
-    if g_step > 0:
-        return
+class OptionsContext(Options.OptionsContext):
+    def __init__(self, **kwargs):
+        super(OptionsContext, self).__init__(**kwargs)
+        set_options(self)
 
+    def configuration_options(self):
+        return self.get_option_group('Configuration options')
+
+    def add_flags(self, group, flags):
+        """Tersely add flags (a dictionary of longname:desc) to a group"""
+        for name, desc in flags.items():
+            group.add_option('--' + name, action='store_true',
+                             dest=name.replace('-', '_'), help=desc)
+
+def set_options(opt, debug_by_default=False):
+    "Add standard autowaf options"
     opts = opt.get_option_group('Configuration options')
 
     # Standard directory options
@@ -60,42 +67,71 @@ def set_options(opt, debug_by_default=False, test=False):
         opts.add_option('--optimize', action='store_false', default=True,
                         dest='debug', help="build optimized binaries")
     else:
-        opts.add_option('--debug', action='store_true', default=False,
+        opts.add_option('-d', '--debug', action='store_true', default=False,
                         dest='debug', help="build debuggable binaries")
         opts.add_option('--pardebug', action='store_true', default=False,
                         dest='pardebug',
                         help="build debug libraries with D suffix")
 
-    opts.add_option('--strict', action='store_true', default=False,
+    opts.add_option('-s', '--strict', action='store_true', default=False,
                     dest='strict',
                     help="use strict compiler flags and show all warnings")
-    opts.add_option('--ultra-strict', action='store_true', default=False,
+    opts.add_option('-S', '--ultra-strict', action='store_true', default=False,
                     dest='ultra_strict',
                     help="use extremely strict compiler flags (likely noisy)")
     opts.add_option('--docs', action='store_true', default=False, dest='docs',
                     help="build documentation (requires doxygen)")
 
     # Test options
-    if test:
+    if hasattr(Context.g_module, 'test'):
         test_opts = opt.add_option_group('Test options', '')
-        opts.add_option('--test', action='store_true', dest='build_tests',
+        opts.add_option('-T', '--test', action='store_true', dest='build_tests',
                         help='build unit tests')
         opts.add_option('--no-coverage', action='store_true',
                         dest='no_coverage',
                         help='do not instrument code for test coverage')
-        test_opts.add_option('--test-wrapper', type='string',
+        test_opts.add_option('--wrapper', type='string',
                              dest='test_wrapper',
                              help='command prefix for tests (e.g. valgrind)')
-        test_opts.add_option('--verbose-tests', action='store_true',
-                             default=False, dest='verbose_tests',
-                             help='always show test output')
-
-    g_step = 1
-
-def add_flags(opt, flags):
-    for name, desc in flags.items():
-        opt.add_option('--' + name, action='store_true',
-                       dest=name.replace('-', '_'), help=desc)
+        test_opts.add_option('--test-filter', type='string',
+                             dest='test_filter',
+                             help='regular expression for tests to run')
+
+    # Run options
+    run_opts = opt.add_option_group('Run options')
+    run_opts.add_option('--cmd', type='string', dest='cmd',
+                        help='command to run from build directory')
+
+class ConfigureContext(Configure.ConfigurationContext):
+    """configures the project"""
+
+    def __init__(self, **kwargs):
+        self.line_just = 45
+        if hasattr(Context.g_module, 'line_just'):
+            self.line_just = Context.g_module.line_just
+
+        super(ConfigureContext, self).__init__(**kwargs)
+        self.run_env = ConfigSet.ConfigSet()
+        self.system_include_paths = set()
+
+    def pre_recurse(self, node):
+        if len(self.stack_path) == 1:
+            Logs.pprint('BOLD', 'Configuring %s' % node.parent.srcpath())
+        super(ConfigureContext, self).pre_recurse(node)
+
+    def store(self):
+        self.env.AUTOWAF_RUN_ENV = self.run_env.get_merged_dict()
+        for path in sorted(self.system_include_paths):
+            if 'COMPILER_CC' in self.env:
+                self.env.append_value('CFLAGS', ['-isystem', path])
+            if 'COMPILER_CXX' in self.env:
+                self.env.append_value('CXXFLAGS', ['-isystem', path])
+
+        super(ConfigureContext, self).store()
+
+    def build_path(self, path='.'):
+        """Return `path` within the build directory"""
+        return str(self.path.get_bld().find_node(path))
 
 def get_check_func(conf, lang):
     if lang == 'c':
@@ -180,6 +216,10 @@ def check_pkg(conf, name, **args):
     else:
         conf.env[var_name] = CheckType.OPTIONAL
 
+    if not conf.env.MSVC_COMPILER and 'system' in args and args['system']:
+        conf.system_include_paths.update(
+            conf.env['INCLUDES_' + nameify(args['uselib_store'])])
+
 def normpath(path):
     if sys.platform == 'win32':
         return os.path.normpath(path).replace('\\', '/')
@@ -187,10 +227,6 @@ def normpath(path):
         return os.path.normpath(path)
 
 def configure(conf):
-    global g_step
-    if g_step > 1:
-        return
-
     def append_cxx_flags(flags):
         conf.env.append_value('CFLAGS', flags)
         conf.env.append_value('CXXFLAGS', flags)
@@ -238,16 +274,37 @@ def configure(conf):
             conf.env['CXXFLAGS'] = ['-O0', '-g']
     else:
         if conf.env['MSVC_COMPILER']:
-            conf.env['CFLAGS']   = ['/MD', '/FS', '/DNDEBUG']
-            conf.env['CXXFLAGS'] = ['/MD', '/FS', '/DNDEBUG']
+            append_cxx_flags(['/MD', '/FS', '/DNDEBUG'])
         else:
             append_cxx_flags(['-DNDEBUG'])
 
     if conf.env.MSVC_COMPILER:
         Options.options.no_coverage = True
-        if Options.options.strict:
-            conf.env.append_value('CFLAGS', ['/Wall'])
-            conf.env.append_value('CXXFLAGS', ['/Wall'])
+        append_cxx_flags(['/nologo',
+                          '/FS',
+                          '/DNDEBUG',
+                          '/D_CRT_SECURE_NO_WARNINGS',
+                          '/experimental:external',
+                          '/external:W0',
+                          '/external:anglebrackets'])
+        conf.env.append_value('LINKFLAGS', '/nologo')
+        if Options.options.strict or Options.options.ultra_strict:
+            ms_strict_flags = ['/Wall',
+                               '/wd4061',
+                               '/wd4200',
+                               '/wd4514',
+                               '/wd4571',
+                               '/wd4625',
+                               '/wd4626',
+                               '/wd4706',
+                               '/wd4710',
+                               '/wd4820',
+                               '/wd5026',
+                               '/wd5027',
+                               '/wd5045']
+            conf.env.append_value('CFLAGS', ms_strict_flags)
+            conf.env.append_value('CXXFLAGS', ms_strict_flags)
+            conf.env.append_value('CXXFLAGS', ['/EHsc'])
     else:
         if Options.options.ultra_strict:
             Options.options.strict = True
@@ -310,16 +367,14 @@ def configure(conf):
     # Define version in configuration
     appname = getattr(Context.g_module, Context.APPNAME, 'noname')
     version = getattr(Context.g_module, Context.VERSION, '0.0.0')
-    define(conf, appname.upper() + '_VERSION', version)
+    defname = appname.upper().replace('-', '_').replace('.', '_')
+    define(conf, defname + '_VERSION', version)
 
     conf.env.prepend_value('CFLAGS', '-I' + os.path.abspath('.'))
     conf.env.prepend_value('CXXFLAGS', '-I' + os.path.abspath('.'))
-    g_step = 2
 
 def display_summary(conf, msgs=None):
-    global g_is_child
-    Logs.pprint('', '')
-    if not g_is_child:
+    if len(conf.stack_path) == 1:
         display_msg(conf, "Install prefix", conf.env['PREFIX'])
         if 'COMPILER_CC' in conf.env:
             display_msg(conf, "C Flags", ' '.join(conf.env['CFLAGS']))
@@ -331,13 +386,11 @@ def display_summary(conf, msgs=None):
     if msgs is not None:
         display_msgs(conf, msgs)
 
-    Logs.pprint('', '')
-
 def set_c_lang(conf, lang):
     "Set a specific C language standard, like 'c99' or 'c11'"
     if conf.env.MSVC_COMPILER:
         # MSVC has no hope or desire to compile C99, just compile as C++
-        conf.env.append_unique('CFLAGS', ['-TP'])
+        conf.env.append_unique('CFLAGS', ['/TP'])
     else:
         flag = '-std=%s' % lang
         conf.check(cflags=['-Werror', flag],
@@ -361,7 +414,7 @@ def set_modern_c_flags(conf):
     if 'COMPILER_CC' in conf.env:
         if conf.env.MSVC_COMPILER:
             # MSVC has no hope or desire to compile C99, just compile as C++
-            conf.env.append_unique('CFLAGS', ['-TP'])
+            conf.env.append_unique('CFLAGS', ['/TP'])
         else:
             for flag in ['-std=c11', '-std=c99']:
                 if conf.check(cflags=['-Werror', flag], mandatory=False,
@@ -436,22 +489,16 @@ def set_lib_env(conf, name, version):
     major_ver    = version.split('.')[0]
     pkg_var_name = 'PKG_' + name.replace('-', '_') + '_' + major_ver
     lib_name     = '%s-%s' % (name, major_ver)
+    lib_path     = [str(conf.path.get_bld())]
     if conf.env.PARDEBUG:
         lib_name += 'D'
     conf.env[pkg_var_name]       = lib_name
     conf.env['INCLUDES_' + NAME] = ['${INCLUDEDIR}/%s-%s' % (name, major_ver)]
-    conf.env['LIBPATH_' + NAME]  = [conf.env.LIBDIR]
+    conf.env['LIBPATH_' + NAME]  = lib_path
     conf.env['LIB_' + NAME]      = [lib_name]
 
-def set_line_just(conf, width):
-    global line_just
-    line_just = max(line_just, width)
-    conf.line_just = line_just
-
-def display_header(title):
-    global g_is_child
-    if g_is_child:
-        Logs.pprint('BOLD', title)
+    conf.run_env.append_unique(lib_path_name, lib_path)
+    conf.define(NAME + '_VERSION', version)
 
 def display_msg(conf, msg, status=None, color=None):
     color = 'CYAN'
@@ -495,7 +542,7 @@ def build_pc(bld, name, version, version_suffix, libs, subst_dict={}):
     """
 
     pkg_prefix       = bld.env['PREFIX']
-    if pkg_prefix[-1] == '/':
+    if len(pkg_prefix) > 1 and pkg_prefix[-1] == '/':
         pkg_prefix = pkg_prefix[:-1]
 
     target = name.lower()
@@ -584,7 +631,8 @@ def make_simple_dox(name):
         os.chdir(top)
     except Exception as e:
         Logs.error("Failed to fix up %s documentation: %s" % (name, e))
-
+    finally:
+        os.chdir(top)
 
 def build_dox(bld, name, version, srcdir, blddir, outdir='', versioned=True):
     """Build Doxygen API documentation"""
@@ -715,211 +763,415 @@ def build_i18n(bld, srcdir, dir, name, sources, copyright_holder=None):
     build_i18n_po(bld, srcdir, dir, name, sources, copyright_holder)
     build_i18n_mo(bld, srcdir, dir, name, sources, copyright_holder)
 
-def cd_to_build_dir(ctx, appname):
-    top_level = (len(ctx.stack_path) > 1)
-    if top_level:
-        os.chdir(os.path.join('build', appname))
-    else:
-        os.chdir('build')
-    Logs.pprint('GREEN', ("Waf: Entering directory `%s'" %
-                          os.path.abspath(os.getcwd())))
+class ExecutionEnvironment:
+    """Context that sets system environment variables for program execution"""
+    def __init__(self, changes):
+        self.original_environ = os.environ.copy()
+
+        self.diff = {}
+        for path_name, paths in changes.items():
+            value = os.pathsep.join(paths)
+            if path_name in os.environ:
+                value += os.pathsep + os.environ[path_name]
+
+            self.diff[path_name] = value
+
+        os.environ.update(self.diff)
+
+    def __str__(self):
+        return '\n'.join({'%s="%s"' % (k, v) for k, v in self.diff.items()})
+
+    def __enter__(self):
+        return self
 
-def cd_to_orig_dir(ctx, child):
-    if child:
-        os.chdir(os.path.join('..', '..'))
+    def __exit__(self, type, value, traceback):
+        os.environ = self.original_environ
+
+class RunContext(Build.BuildContext):
+    "runs an executable from the build directory"
+    cmd = 'run'
+
+    def execute(self):
+        self.restore()
+        if not self.all_envs:
+            self.load_envs()
+
+        with ExecutionEnvironment(self.env.AUTOWAF_RUN_ENV) as env:
+            if Options.options.verbose:
+                Logs.pprint('GREEN', str(env) + '\n')
+
+            if Options.options.cmd:
+                Logs.pprint('GREEN', 'Running %s' % Options.options.cmd)
+                subprocess.call(Options.options.cmd, shell=True)
+            else:
+                Logs.error("error: Missing --cmd option for run command")
+
+def show_diff(from_lines, to_lines, from_filename, to_filename):
+    import difflib
+    import sys
+
+    same = True
+    for line in difflib.unified_diff(
+            from_lines, to_lines,
+            fromfile=os.path.abspath(from_filename),
+            tofile=os.path.abspath(to_filename)):
+        sys.stderr.write(line)
+        same = False
+
+    return same
+
+def test_file_equals(patha, pathb):
+    import filecmp
+    import io
+
+    for path in (patha, pathb):
+        if not os.access(path, os.F_OK):
+            Logs.pprint('RED', 'error: missing file %s' % path)
+            return False
+
+    if filecmp.cmp(patha, pathb, shallow=False):
+        return True
+
+    with io.open(patha, 'rU', encoding='utf-8') as fa:
+        with io.open(pathb, 'rU', encoding='utf-8') as fb:
+            return show_diff(fa.readlines(), fb.readlines(), patha, pathb)
+
+def bench_time():
+    if hasattr(time, 'perf_counter'): # Added in Python 3.3
+        return time.perf_counter()
     else:
-        os.chdir('..')
-
-def pre_test(ctx, appname, dirs=['src']):
-    if not hasattr(ctx, 'autowaf_tests_total'):
-        ctx.autowaf_tests_total        = 0
-        ctx.autowaf_tests_failed       = 0
-        ctx.autowaf_local_tests_total  = 0
-        ctx.autowaf_local_tests_failed = 0
-        ctx.autowaf_tests              = {}
-
-    ctx.autowaf_tests[appname] = {'total': 0, 'failed': 0}
-
-    cd_to_build_dir(ctx, appname)
-    if not ctx.env.NO_COVERAGE:
-        diropts  = ''
-        for i in dirs:
-            diropts += ' -d ' + i
-        clear_log = open('lcov-clear.log', 'w')
-        try:
-            try:
-                # Clear coverage data
-                subprocess.call(('lcov %s -z' % diropts).split(),
-                                stdout=clear_log, stderr=clear_log)
-            except Exception:
-                Logs.warn('Failed to run lcov, no coverage report generated')
-        finally:
-            clear_log.close()
-
-def post_test(ctx, appname, dirs=['src'], remove=['*boost*', 'c++*']):
-    if not ctx.env.NO_COVERAGE:
-        diropts  = ''
-        for i in dirs:
-            diropts += ' -d ' + i
-        coverage_log           = open('lcov-coverage.log', 'w')
-        coverage_lcov          = open('coverage.lcov', 'w')
-        coverage_stripped_lcov = open('coverage-stripped.lcov', 'w')
+        return time.time()
+
+class TestOutput:
+    """Test output that is truthy if result is as expected"""
+    def __init__(self, expected, result=None):
+        self.stdout = self.stderr = None
+        self.expected = expected
+        self.result = result
+
+    def __bool__(self):
+        return self.expected is None or self.result == self.expected
+
+    __nonzero__ = __bool__
+
+def is_string(s):
+    if sys.version_info[0] < 3:
+        return isinstance(s, basestring)
+    return isinstance(s, str)
+
+class TestScope:
+    """Scope for running tests that maintains pass/fail statistics"""
+    def __init__(self, tst, name, defaults):
+        self.tst = tst
+        self.name = name
+        self.defaults = defaults
+        self.n_failed = 0
+        self.n_total = 0
+
+    def run(self, test, **kwargs):
+        if type(test) == list and 'name' not in kwargs:
+            import pipes
+            kwargs['name'] = ' '.join(map(pipes.quote, test))
+
+        if Options.options.test_filter and 'name' in kwargs:
+            import re
+            found = False
+            for scope in self.tst.stack:
+                if re.search(Options.options.test_filter, scope.name):
+                    found = True
+                    break
+
+            if (not found and
+                not re.search(Options.options.test_filter, self.name) and
+                not re.search(Options.options.test_filter, kwargs['name'])):
+                return True
+
+        if callable(test):
+            output = self._run_callable(test, **kwargs)
+        elif type(test) == list:
+
+            output = self._run_command(test, **kwargs)
+        else:
+            raise Exception("Unknown test type")
+
+        if not output:
+            self.tst.log_bad('FAILED', kwargs['name'])
+
+        return self.tst.test_result(output)
+
+    def _run_callable(self, test, **kwargs):
+        expected = kwargs['expected'] if 'expected' in kwargs else True
+        return TestOutput(expected, test())
+
+    def _run_command(self, test, **kwargs):
+        if 'stderr' in kwargs and kwargs['stderr'] == NONEMPTY:
+            # Run with a temp file for stderr and check that it is non-empty
+            import tempfile
+            with tempfile.TemporaryFile() as stderr:
+                kwargs['stderr'] = stderr
+                output = self.run(test, **kwargs)
+                stderr.seek(0, 2) # Seek to end
+                return (output if not output else
+                        self.run(
+                            lambda: stderr.tell() > 0,
+                            name=kwargs['name'] + ' error message'))
+
         try:
-            try:
-                base = '.'
-                if g_is_child:
-                    base = '..'
-
-                # Generate coverage data
-                lcov_cmd = 'lcov -c %s -b %s' % (diropts, base)
-                if ctx.env.LLVM_COV:
-                    lcov_cmd += ' --gcov-tool %s' % ctx.env.LLVM_COV[0]
-                subprocess.call(lcov_cmd.split(),
-                                stdout=coverage_lcov, stderr=coverage_log)
-
-                # Strip unwanted stuff
-                subprocess.call(
-                    ['lcov', '--remove', 'coverage.lcov'] + remove,
-                    stdout=coverage_stripped_lcov, stderr=coverage_log)
-
-                # Generate HTML coverage output
-                if not os.path.isdir('coverage'):
-                    os.makedirs('coverage')
-                subprocess.call(
-                    'genhtml -o coverage coverage-stripped.lcov'.split(),
-                    stdout=coverage_log, stderr=coverage_log)
-
-            except Exception:
-                Logs.warn('Failed to run lcov, no coverage report generated')
+            # Run with stdout and stderr set to the appropriate streams
+            out_stream = self._stream('stdout', kwargs)
+            err_stream = self._stream('stderr', kwargs)
+            return self._exec(test, **kwargs)
         finally:
-            coverage_stripped_lcov.close()
-            coverage_lcov.close()
-            coverage_log.close()
-
-    if ctx.autowaf_tests[appname]['failed'] > 0:
-        Logs.pprint('RED', '\nSummary:  %d / %d %s tests failed' % (
-            ctx.autowaf_tests[appname]['failed'],
-            ctx.autowaf_tests[appname]['total'],
-            appname))
-    else:
-        Logs.pprint('GREEN', '\nSummary:  All %d %s tests passed' % (
-            ctx.autowaf_tests[appname]['total'], appname))
-
-    if not ctx.env.NO_COVERAGE:
-        Logs.pprint('GREEN', 'Coverage: <file://%s>\n'
-                    % os.path.abspath('coverage/index.html'))
-
-    Logs.pprint('GREEN', ("Waf: Leaving directory `%s'" %
-                          os.path.abspath(os.getcwd())))
-    top_level = (len(ctx.stack_path) > 1)
-    if top_level:
-        cd_to_orig_dir(ctx, top_level)
-
-def run_test(ctx,
-             appname,
-             test,
-             desired_status=0,
-             dirs=['src'],
-             name='',
-             header=False,
-             quiet=False):
-    """Run an individual test.
-
-    `test` is either a shell command string, or a list of [name, return status]
-    for displaying tests implemented in the calling Python code.
-    """
-
-    ctx.autowaf_tests_total += 1
-    ctx.autowaf_local_tests_total += 1
-    ctx.autowaf_tests[appname]['total'] += 1
+            out_stream = out_stream.close() if out_stream else None
+            err_stream = err_stream.close() if err_stream else None
+
+    def _stream(self, stream_name, kwargs):
+        s = kwargs[stream_name] if stream_name in kwargs else None
+        if is_string(s):
+            kwargs[stream_name] = open(s, 'wb')
+            return kwargs[stream_name]
+        return None
+
+    def _exec(self,
+              test,
+              expected=0,
+              name='',
+              stdin=None,
+              stdout=None,
+              stderr=None,
+              verbosity=1):
+        def stream(s):
+            return open(s, 'wb') if type(s) == str else s
+
+        if verbosity > 1:
+            self.tst.log_good('RUN     ', name)
 
-    out = (None, None)
-    if type(test) == list:
-        name       = test[0]
-        returncode = test[1]
-    elif callable(test):
-        returncode = test()
-    else:
-        s = test
-        if isinstance(test, type([])):
-            s = ' '.join(test)
-        if header and not quiet:
-            Logs.pprint('Green', '\n** Test %s' % s)
-        cmd = test
         if Options.options.test_wrapper:
-            cmd = Options.options.test_wrapper + ' ' + test
-        if name == '':
-            name = test
-
-        proc = subprocess.Popen(cmd, shell=True,
-                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out = proc.communicate()
-        returncode = proc.returncode
-
-    success = desired_status is None or returncode == desired_status
-    if success:
-        if not quiet:
-            Logs.pprint('GREEN', '** Pass %s' % name)
-    else:
-        Logs.pprint('RED', '** FAIL %s' % name)
-        ctx.autowaf_tests_failed += 1
-        ctx.autowaf_tests[appname]['failed'] += 1
-        if type(test) != list and not callable(test):
-            Logs.pprint('RED', test)
+            test = [Options.options.test_wrapper] + test
 
-    if Options.options.verbose_tests and type(test) != list and not callable(test):
-        sys.stdout.write(out[0])
-        sys.stderr.write(out[1])
+        output = TestOutput(expected)
+        with open(os.devnull, 'wb') as null:
+            out = null if verbosity < 3 and not stdout else stdout
+            err = null if verbosity < 2 and not stderr else stderr
+            proc = subprocess.Popen(test, stdin=stdin, stdout=out, stderr=err)
+            output.stdout, output.stderr = proc.communicate()
+            output.result = proc.returncode
 
-    return (success, out)
+        if output and verbosity > 0:
+            self.tst.log_good('      OK', name)
 
-def tests_name(ctx, appname, name='*'):
-    if name == '*':
-        return appname
-    else:
-        return '%s.%s' % (appname, name)
+        return output
 
-def begin_tests(ctx, appname, name='*'):
-    ctx.autowaf_local_tests_failed = 0
-    ctx.autowaf_local_tests_total  = 0
-    Logs.pprint('GREEN', '\n** Begin %s tests' % (
-        tests_name(ctx, appname, name)))
+class TestContext(Build.BuildContext):
+    "runs test suite"
+    fun = cmd = 'test'
 
-    class Handle:
-        def __enter__(self):
-            pass
+    def __init__(self, **kwargs):
+        super(TestContext, self).__init__(**kwargs)
+        self.start_time = bench_time()
+        self.max_depth = 1
 
-        def __exit__(self, type, value, traceback):
-            end_tests(ctx, appname, name)
+        defaults = {'verbosity': Options.options.verbose}
+        self.stack = [TestScope(self, Context.g_module.APPNAME, defaults)]
 
-    return Handle()
+    def defaults(self):
+        return self.stack[-1].defaults
 
-def end_tests(ctx, appname, name='*'):
-    failures = ctx.autowaf_local_tests_failed
-    if failures == 0:
-        Logs.pprint('GREEN', '** Passed all %d %s tests' % (
-            ctx.autowaf_local_tests_total, tests_name(ctx, appname, name)))
-    else:
-        Logs.pprint('RED', '** Failed %d / %d %s tests' % (
-            failures, ctx.autowaf_local_tests_total, tests_name(ctx, appname, name)))
-
-def run_tests(ctx,
-              appname,
-              tests,
-              desired_status=0,
-              dirs=['src'],
-              name='*',
-              headers=False):
-    begin_tests(ctx, appname, name)
-
-    diropts  = ''
-    for i in dirs:
-        diropts += ' -d ' + i
-
-    for i in tests:
-        run_test(ctx, appname, i, desired_status, dirs, i, headers)
-
-    end_tests(ctx, appname, name)
+    def finalize(self):
+        if self.stack[-1].n_failed > 0:
+            sys.exit(1)
+
+        super(TestContext, self).finalize()
+
+    def __call__(self, test, **kwargs):
+        return self.stack[-1].run(test, **self.args(**kwargs))
+
+    def file_equals(self, from_path, to_path, **kwargs):
+        kwargs.update({'expected': True,
+                       'name': '%s == %s' % (from_path, to_path)})
+        return self(lambda: test_file_equals(from_path, to_path), **kwargs)
+
+    def log_good(self, title, fmt, *args):
+        Logs.pprint('GREEN', '[%s] %s' % (title.center(10), fmt % args))
+
+    def log_bad(self, title, fmt, *args):
+        Logs.pprint('RED', '[%s] %s' % (title.center(10), fmt % args))
+
+    def pre_recurse(self, node):
+        wscript_module = Context.load_module(node.abspath())
+        group_name = wscript_module.APPNAME
+        self.stack.append(TestScope(self, group_name, self.defaults()))
+        self.max_depth = max(self.max_depth, len(self.stack) - 1)
+
+        bld_dir = node.get_bld().parent
+        if bld_dir != self.path.get_bld():
+            Logs.info('')
+
+        self.original_dir = os.getcwd()
+        Logs.info("Waf: Entering directory `%s'\n", bld_dir)
+        os.chdir(str(bld_dir))
+
+        if not self.env.NO_COVERAGE and str(node.parent) == Context.top_dir:
+            self.clear_coverage()
+
+        self.log_good('=' * 10, 'Running %s tests', group_name)
+        super(TestContext, self).pre_recurse(node)
+
+    def test_result(self, success):
+        self.stack[-1].n_total += 1
+        self.stack[-1].n_failed += 1 if not success else 0
+        return success
+
+    def pop(self):
+        scope = self.stack.pop()
+        self.stack[-1].n_total += scope.n_total
+        self.stack[-1].n_failed += scope.n_failed
+        return scope
+
+    def post_recurse(self, node):
+        super(TestContext, self).post_recurse(node)
+
+        scope = self.pop()
+        duration = (bench_time() - self.start_time) * 1000.0
+        is_top = str(node.parent) == str(Context.top_dir)
+
+        if is_top and self.max_depth > 1:
+            Logs.info('')
+
+        self.log_good('=' * 10, '%d tests from %s ran (%d ms total)',
+                      scope.n_total, scope.name, duration)
+
+        if not self.env.NO_COVERAGE:
+            if is_top:
+                self.gen_coverage()
+
+            if os.path.exists('coverage/index.html'):
+                self.log_good('REPORT', '<file://%s>',
+                              os.path.abspath('coverage/index.html'))
+
+        successes = scope.n_total - scope.n_failed
+        Logs.pprint('GREEN', '[  PASSED  ] %d tests' % successes)
+        if scope.n_failed > 0:
+            Logs.pprint('RED', '[  FAILED  ] %d tests' % scope.n_failed)
+        if is_top:
+            Logs.info("\nWaf: Leaving directory `%s'" % os.getcwd())
+
+        os.chdir(self.original_dir)
+
+    def execute(self):
+        self.restore()
+        if not self.all_envs:
+            self.load_envs()
+
+        if not self.env.BUILD_TESTS:
+            self.fatal('Configuration does not include tests')
+
+        with ExecutionEnvironment(self.env.AUTOWAF_RUN_ENV) as env:
+            if self.defaults()['verbosity'] > 0:
+                Logs.pprint('GREEN', str(env) + '\n')
+            self.recurse([self.run_dir])
+
+    def src_path(self, path):
+        return os.path.relpath(os.path.join(str(self.path), path))
+
+    def args(self, **kwargs):
+        all_kwargs = self.defaults().copy()
+        all_kwargs.update(kwargs)
+        return all_kwargs
+
+    def group(self, name, **kwargs):
+        return TestGroup(
+            self, self.stack[-1].name, name, **self.args(**kwargs))
+
+    def set_test_defaults(self, **kwargs):
+        """Set default arguments to be passed to all tests"""
+        self.stack[-1].defaults.update(kwargs)
+
+    def clear_coverage(self):
+        """Zero old coverage data"""
+        try:
+            with open('cov-clear.log', 'w') as log:
+                subprocess.call(['lcov', '-z', '-d', str(self.path)],
+                                stdout=log, stderr=log)
+
+        except Exception:
+            Logs.warn('Failed to run lcov to clear old coverage data')
+
+    def gen_coverage(self):
+        """Generate coverage data and report"""
+        try:
+            with open('cov.lcov', 'w') as out:
+                with open('cov.log', 'w') as err:
+                    subprocess.call(['lcov', '-c', '--no-external',
+                                     '--rc', 'lcov_branch_coverage=1',
+                                     '-b', '.',
+                                     '-d', str(self.path)],
+                                    stdout=out, stderr=err)
+
+            if not os.path.isdir('coverage'):
+                os.makedirs('coverage')
+
+            with open('genhtml.log', 'w') as log:
+                subprocess.call(['genhtml',
+                                 '-o', 'coverage',
+                                 '--rc', 'genhtml_branch_coverage=1',
+                                 'cov.lcov'],
+                                stdout=log, stderr=log)
+
+            summary = subprocess.check_output(
+                ['lcov', '--summary',
+                 '--rc', 'lcov_branch_coverage=1',
+                 'cov.lcov'],
+                stderr=subprocess.STDOUT).decode('ascii')
+
+            import re
+            lines = re.search('lines\.*: (.*)%.*', summary).group(1)
+            functions = re.search('functions\.*: (.*)%.*', summary).group(1)
+            branches = re.search('branches\.*: (.*)%.*', summary).group(1)
+            self.log_good('COVERAGE', '%s%% lines, %s%% functions, %s%% branches',
+                          lines, functions, branches)
+
+        except Exception:
+            Logs.warn('Failed to run lcov to generate coverage report')
+
+class TestGroup:
+    def __init__(self, tst, suitename, name, **kwargs):
+        self.tst = tst
+        self.suitename = suitename
+        self.name = name
+        self.kwargs = kwargs
+        self.start_time = bench_time()
+        tst.stack.append(TestScope(tst, name, tst.defaults()))
+
+    def label(self):
+        return self.suitename + '.%s' % self.name if self.name else ''
+
+    def args(self, **kwargs):
+        all_kwargs = self.tst.args(**self.kwargs)
+        all_kwargs.update(kwargs)
+        return all_kwargs
+
+    def __enter__(self):
+        if 'verbosity' in self.kwargs and self.kwargs['verbosity'] > 0:
+            self.tst.log_good('-' * 10, self.label())
+        return self
+
+    def __call__(self, test, **kwargs):
+        return self.tst(test, **self.args(**kwargs))
+
+    def file_equals(self, from_path, to_path, **kwargs):
+        return self.tst.file_equals(from_path, to_path, **kwargs)
+
+    def __exit__(self, type, value, traceback):
+        duration = (bench_time() - self.start_time) * 1000.0
+        scope = self.tst.pop()
+        n_passed = scope.n_total - scope.n_failed
+        if scope.n_failed == 0:
+            self.tst.log_good('-' * 10, '%d tests from %s (%d ms total)',
+                              scope.n_total, self.label(), duration)
+        else:
+            self.tst.log_bad('-' * 10, '%d/%d tests from %s (%d ms total)',
+                             n_passed, scope.n_total, self.label(), duration)
 
 def run_ldconfig(ctx):
     should_run = (ctx.cmd == 'install' and
diff --git a/extras/buildcopy.py b/extras/buildcopy.py
index a6d9ac8..eaff7e6 100644
--- a/extras/buildcopy.py
+++ b/extras/buildcopy.py
@@ -22,7 +22,7 @@ Examples::
 
 """
 import os, shutil
-from waflib import Errors, Task, TaskGen, Utils, Node
+from waflib import Errors, Task, TaskGen, Utils, Node, Logs
 
 @TaskGen.before_method('process_source')
 @TaskGen.feature('buildcopy')
@@ -58,10 +58,13 @@ def make_buildcopy(self):
 		raise Errors.WafError('buildcopy: File not found in src: %s'%os.path.join(*lst))
 
 	nodes = [ to_src_nodes(n) for n in getattr(self, 'buildcopy_source', getattr(self, 'source', [])) ]
+	if not nodes:
+		Logs.warn('buildcopy: No source files provided to buildcopy in %s (set `buildcopy_source` or `source`)',
+			self)
+		return
 	node_pairs = [(n, n.get_bld()) for n in nodes]
 	self.create_task('buildcopy', [n[0] for n in node_pairs], [n[1] for n in node_pairs], node_pairs=node_pairs)
 
-
 class buildcopy(Task.Task):
 	"""
 	Copy for each pair `n` in `node_pairs`: n[0] -> n[1].
diff --git a/extras/cpplint.py b/extras/cpplint.py
index e3302e5..8cdd6dd 100644
--- a/extras/cpplint.py
+++ b/extras/cpplint.py
@@ -43,12 +43,12 @@ from waflib import Errors, Task, TaskGen, Logs, Options, Node, Utils
 
 critical_errors = 0
 CPPLINT_FORMAT = '[CPPLINT] %(filename)s:\nline %(linenum)s, severity %(confidence)s, category: %(category)s\n%(message)s\n'
-RE_EMACS = re.compile('(?P<filename>.*):(?P<linenum>\d+):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]')
+RE_EMACS = re.compile(r'(?P<filename>.*):(?P<linenum>\d+):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]')
 CPPLINT_RE = {
     'waf': RE_EMACS,
     'emacs': RE_EMACS,
-    'vs7': re.compile('(?P<filename>.*)\((?P<linenum>\d+)\):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
-    'eclipse': re.compile('(?P<filename>.*):(?P<linenum>\d+): warning: (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
+    'vs7': re.compile(r'(?P<filename>.*)\((?P<linenum>\d+)\):  (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
+    'eclipse': re.compile(r'(?P<filename>.*):(?P<linenum>\d+): warning: (?P<message>.*)  \[(?P<category>.*)\] \[(?P<confidence>\d+)\]'),
 }
 CPPLINT_STR = ('${CPPLINT} '
                '--verbose=${CPPLINT_LEVEL} '
diff --git a/extras/cython.py b/extras/cython.py
index 481d6f4..591c274 100644
--- a/extras/cython.py
+++ b/extras/cython.py
@@ -8,8 +8,9 @@ from waflib.TaskGen import extension
 
 cy_api_pat = re.compile(r'\s*?cdef\s*?(public|api)\w*')
 re_cyt = re.compile(r"""
-	(?:from\s+(\w+)\s+)?   # optionally match "from foo" and capture foo
-	c?import\s(\w+|[*])    # require "import bar" and capture bar
+	^\s*                           # must begin with some whitespace characters
+	(?:from\s+(\w+)(?:\.\w+)*\s+)? # optionally match "from foo(.baz)" and capture foo
+	c?import\s(\w+|[*])            # require "import bar" and capture bar
 	""", re.M | re.VERBOSE)
 
 @extension('.pyx')
@@ -85,12 +86,12 @@ class cython(Task.Task):
 		node = self.inputs[0]
 		txt = node.read()
 
-		mods = []
+		mods = set()
 		for m in re_cyt.finditer(txt):
 			if m.group(1):  # matches "from foo import bar"
-				mods.append(m.group(1))
+				mods.add(m.group(1))
 			else:
-				mods.append(m.group(2))
+				mods.add(m.group(2))
 
 		Logs.debug('cython: mods %r', mods)
 		incs = getattr(self.generator, 'cython_includes', [])
@@ -99,7 +100,7 @@ class cython(Task.Task):
 
 		found = []
 		missing = []
-		for x in mods:
+		for x in sorted(mods):
 			for y in incs:
 				k = y.find_resource(x + '.pxd')
 				if k:
diff --git a/extras/distnet.py b/extras/distnet.py
index 09a31a6..ff3ed8e 100644
--- a/extras/distnet.py
+++ b/extras/distnet.py
@@ -44,7 +44,7 @@ TARFORMAT = 'w:bz2'
 TIMEOUT = 60
 REQUIRES = 'requires.txt'
 
-re_com = re.compile('\s*#.*', re.M)
+re_com = re.compile(r'\s*#.*', re.M)
 
 def total_version_order(num):
 	lst = num.split('.')
diff --git a/extras/erlang.py b/extras/erlang.py
index 49f6d5b..0b93d9a 100644
--- a/extras/erlang.py
+++ b/extras/erlang.py
@@ -51,7 +51,7 @@ class erl(Task.Task):
 			if n.abspath() in scanned:
 				continue
 
-			for i in re.findall('-include\("(.*)"\)\.', n.read()):
+			for i in re.findall(r'-include\("(.*)"\)\.', n.read()):
 				for d in task.erlc_incnodes:
 					r = d.find_node(i)
 					if r:
diff --git a/extras/fast_partial.py b/extras/fast_partial.py
index b3af513..d5b6144 100644
--- a/extras/fast_partial.py
+++ b/extras/fast_partial.py
@@ -17,7 +17,7 @@ Usage::
 	def options(opt):
 		opt.load('fast_partial')
 
-Assuptions:
+Assumptions:
 * Mostly for C/C++/Fortran targets with link tasks (object-only targets are not handled)
 * For full project builds: no --targets and no pruning from subfolders
 * The installation phase is ignored
diff --git a/extras/fc_nfort.py b/extras/fc_nfort.py
new file mode 100644
index 0000000..c25886b
--- /dev/null
+++ b/extras/fc_nfort.py
@@ -0,0 +1,52 @@
+#! /usr/bin/env python
+# encoding: utf-8
+# Detection of the NEC Fortran compiler for Aurora Tsubasa
+
+import re
+from waflib.Tools import fc,fc_config,fc_scan
+from waflib.Configure import conf
+from waflib.Tools.compiler_fc import fc_compiler
+fc_compiler['linux'].append('fc_nfort')
+
+@conf
+def find_nfort(conf):
+	fc=conf.find_program(['nfort'],var='FC')
+	conf.get_nfort_version(fc)
+	conf.env.FC_NAME='NFORT'
+	conf.env.FC_MOD_CAPITALIZATION='lower'
+
+@conf
+def nfort_flags(conf):
+	v=conf.env
+	v['_FCMODOUTFLAGS']=[]
+	v['FCFLAGS_DEBUG']=[]
+	v['FCFLAGS_fcshlib']=[]
+	v['LINKFLAGS_fcshlib']=[]
+	v['FCSTLIB_MARKER']=''
+	v['FCSHLIB_MARKER']=''
+
+@conf
+def get_nfort_version(conf,fc):
+	version_re=re.compile(r"nfort\s*\(NFORT\)\s*(?P<major>\d+)\.(?P<minor>\d+)\.",re.I).search
+	cmd=fc+['--version']
+	out,err=fc_config.getoutput(conf,cmd,stdin=False)
+	if out:
+		match=version_re(out)
+	else:
+		match=version_re(err)
+	if not match:
+		return(False)
+		conf.fatal('Could not determine the NEC NFORT Fortran compiler version.')
+	else:
+		k=match.groupdict()
+		conf.env['FC_VERSION']=(k['major'],k['minor'])
+
+def configure(conf):
+	conf.find_nfort()
+	conf.find_program('nar',var='AR')
+	conf.add_os_flags('ARFLAGS')
+	if not conf.env.ARFLAGS:
+		conf.env.ARFLAGS=['rcs']
+	conf.fc_flags()
+	conf.fc_add_flags()
+	conf.nfort_flags()
diff --git a/extras/gccdeps.py b/extras/gccdeps.py
index d9758ab..bfabe72 100644
--- a/extras/gccdeps.py
+++ b/extras/gccdeps.py
@@ -36,7 +36,7 @@ def scan(self):
 	names = []
 	return (nodes, names)
 
-re_o = re.compile("\.o$")
+re_o = re.compile(r"\.o$")
 re_splitter = re.compile(r'(?<!\\)\s+') # split by space, except when spaces are escaped
 
 def remove_makefile_rule_lhs(line):
@@ -197,7 +197,7 @@ def configure(conf):
 		except Errors.ConfigurationError:
 			pass
 		else:
-			conf.env.append_value('CFLAGS', gccdeps_flags)
+			conf.env.append_value('CFLAGS', flags)
 			conf.env.append_unique('ENABLE_GCCDEPS', 'c')
 
 	if conf.env.CXX_NAME in supported_compilers:
@@ -206,7 +206,7 @@ def configure(conf):
 		except Errors.ConfigurationError:
 			pass
 		else:
-			conf.env.append_value('CXXFLAGS', gccdeps_flags)
+			conf.env.append_value('CXXFLAGS', flags)
 			conf.env.append_unique('ENABLE_GCCDEPS', 'cxx')
 
 def options(opt):
diff --git a/extras/kde4.py b/extras/kde4.py
index e49a9ec..aed9bfb 100644
--- a/extras/kde4.py
+++ b/extras/kde4.py
@@ -71,7 +71,7 @@ def configure(self):
 	fu = re.compile('#(.*)\n')
 	txt = fu.sub('', txt)
 
-	setregexp = re.compile('([sS][eE][tT]\s*\()\s*([^\s]+)\s+\"([^"]+)\"\)')
+	setregexp = re.compile(r'([sS][eE][tT]\s*\()\s*([^\s]+)\s+\"([^"]+)\"\)')
 	found = setregexp.findall(txt)
 
 	for (_, key, val) in found:
diff --git a/extras/lv2.py b/extras/lv2.py
index 815987f..ffcb2e7 100644
--- a/extras/lv2.py
+++ b/extras/lv2.py
@@ -10,6 +10,33 @@ def options(opt):
                          help='install LV2 bundles to user location')
     conf_opts.add_option('--lv2dir', type='string',
                          help='LV2 bundles [Default: LIBDIR/lv2]')
+
+def register_lv2_path(conf, path):
+    """Return the default LV2_PATH to use for this system"""
+    if 'LV2_PATH' not in conf.run_env and 'LV2_PATH' not in os.environ:
+        conf.run_env['LV2_PATH'] = [conf.env['LV2DIR']]
+
+    conf.run_env.append_unique('LV2_PATH', path)
+
+def default_lv2_path(conf):
+    """Return the default LV2_PATH for the build target as a list"""
+    if conf.env.DEST_OS == 'darwin':
+        return ['~/Library/Audio/Plug-Ins/LV2',
+                '~/.lv2',
+                '/usr/local/lib/lv2',
+                '/usr/lib/lv2',
+                '/Library/Audio/Plug-Ins/LV2']
+    elif conf.env.DEST_OS == 'haiku':
+        return ['~/.lv2',
+                '/boot/common/add-ons/lv2']
+    elif conf.env.DEST_OS == 'win32':
+        return ['%APPDATA%\\\\LV2',
+                '%COMMONPROGRAMFILES%\\\\LV2']
+    else:
+        libdirname = os.path.basename(conf.env.LIBDIR)
+        return ['~/.lv2',
+                '/usr/%s/lv2' % libdirname,
+                '/usr/local/%s/lv2' % libdirname]
     
 def configure(conf):
     def env_path(parent_dir_var, name):
@@ -43,5 +70,6 @@ def configure(conf):
         else:
             conf.env['LV2DIR'] = os.path.join(conf.env['LIBDIR'], 'lv2')
 
-    conf.env['LV2DIR'] = normpath(conf.env['LV2DIR'])
-
+    # Add default LV2_PATH to runtime environment for tests that use plugins
+    if 'LV2_PATH' not in os.environ:
+        conf.run_env['LV2_PATH'] = default_lv2_path(conf)
diff --git a/extras/ocaml.py b/extras/ocaml.py
index afe73c0..7d785c6 100644
--- a/extras/ocaml.py
+++ b/extras/ocaml.py
@@ -15,7 +15,7 @@ EXT_MLI = ['.mli']
 EXT_MLC = ['.c']
 EXT_ML  = ['.ml']
 
-open_re = re.compile('^\s*open\s+([a-zA-Z]+)(;;){0,1}$', re.M)
+open_re = re.compile(r'^\s*open\s+([a-zA-Z]+)(;;){0,1}$', re.M)
 foo = re.compile(r"""(\(\*)|(\*\))|("(\\.|[^"\\])*"|'(\\.|[^'\\])*'|.[^()*"'\\]*)""", re.M)
 def filter_comments(txt):
 	meh = [0]
diff --git a/extras/parallel_debug.py b/extras/parallel_debug.py
index 35883a3..4ffec5e 100644
--- a/extras/parallel_debug.py
+++ b/extras/parallel_debug.py
@@ -3,13 +3,16 @@
 # Thomas Nagy, 2007-2010 (ita)
 
 """
-Debugging helper for parallel compilation, outputs
-a file named pdebug.svg in the source directory::
+Debugging helper for parallel compilation.
+
+Copy it to your project and load it with::
 
 	def options(opt):
-		opt.load('parallel_debug')
+		opt.load('parallel_debug', tooldir='.')
 	def build(bld):
 		...
+
+The build will then output a file named pdebug.svg in the source directory.
 """
 
 import re, sys, threading, time, traceback
diff --git a/extras/pgicc.py b/extras/pgicc.py
index 9790b9c..f8068d5 100644
--- a/extras/pgicc.py
+++ b/extras/pgicc.py
@@ -60,7 +60,7 @@ def get_pgi_version(conf, cc):
 	except Errors.WafError:
 		conf.fatal('Could not find pgi compiler %r' % cmd)
 
-	version = re.findall('^COMPVER\s*=(.*)', out, re.M)
+	version = re.findall(r'^COMPVER\s*=(.*)', out, re.M)
 	if len(version) != 1:
 		conf.fatal('Could not determine the compiler version')
 	return version[0]
diff --git a/extras/protoc.py b/extras/protoc.py
index f3cb4d8..839c510 100644
--- a/extras/protoc.py
+++ b/extras/protoc.py
@@ -6,7 +6,7 @@
 import re, os
 from waflib.Task import Task
 from waflib.TaskGen import extension
-from waflib import Errors, Context
+from waflib import Errors, Context, Logs
 
 """
 A simple tool to integrate protocol buffers into your build system.
@@ -67,6 +67,13 @@ Example for Java:
                 protoc_includes = ['inc']) # for protoc to search dependencies
 
 
+Protoc includes passed via protoc_includes are either relative to the taskgen
+or to the project and are searched in this order.
+
+Include directories external to the waf project can also be passed to the
+extra by using protoc_extincludes
+
+                protoc_extincludes = ['/usr/include/pblib']
 
 
 Notes when using this tool:
@@ -82,7 +89,7 @@ Notes when using this tool:
 """
 
 class protoc(Task):
-	run_str = '${PROTOC} ${PROTOC_FL:PROTOC_FLAGS} ${PROTOC_ST:INCPATHS} ${PROTOC_ST:PROTOC_INCPATHS} ${SRC[0].bldpath()}'
+	run_str = '${PROTOC} ${PROTOC_FL:PROTOC_FLAGS} ${PROTOC_ST:INCPATHS} ${PROTOC_ST:PROTOC_INCPATHS} ${PROTOC_ST:PROTOC_EXTINCPATHS} ${SRC[0].bldpath()}'
 	color   = 'BLUE'
 	ext_out = ['.h', 'pb.cc', '.py', '.java']
 	def scan(self):
@@ -104,7 +111,17 @@ class protoc(Task):
 
 		if 'py' in self.generator.features or 'javac' in self.generator.features:
 			for incpath in getattr(self.generator, 'protoc_includes', []):
-				search_nodes.append(self.generator.bld.path.find_node(incpath))
+				incpath_node = self.generator.path.find_node(incpath)
+				if incpath_node:
+					search_nodes.append(incpath_node)
+				else:
+					# Check if relative to top-level for extra tg dependencies
+					incpath_node = self.generator.bld.path.find_node(incpath)
+					if incpath_node:
+						search_nodes.append(incpath_node)
+					else:
+						raise Errors.WafError('protoc: include path %r does not exist' % incpath)
+
 
 		def parse_node(node):
 			if node in seen:
@@ -126,7 +143,7 @@ class protoc(Task):
 		parse_node(node)
 		# Add also dependencies path to INCPATHS so protoc will find the included file
 		for deppath in nodes:
-			self.env.append_value('INCPATHS', deppath.parent.bldpath())
+			self.env.append_unique('INCPATHS', deppath.parent.bldpath())
 		return (nodes, names)
 
 @extension('.proto')
@@ -153,61 +170,11 @@ def process_protoc(self, node):
 		protoc_flags.append('--python_out=%s' % node.parent.get_bld().bldpath())
 
 	if 'javac' in self.features:
-		pkgname, javapkg, javacn, nodename = None, None, None, None
-		messages = []
-
-		# .java file name is done with some rules depending on .proto file content:
-		#   -) package is either derived from option java_package if present
-		#      or from package directive
-		#   -) file name is either derived from option java_outer_classname if present
-		#      or the .proto file is converted to camelcase. If a message
-		#      is named the same then the behaviour depends on protoc version
-		#
-		# See also: https://developers.google.com/protocol-buffers/docs/reference/java-generated#invocation
-
-		code = node.read().splitlines()
-		for line in code:
-			m = re.search(r'^package\s+(.*);', line)
-			if m:
-				pkgname = m.groups()[0]
-			m = re.search(r'^option\s+(\S*)\s*=\s*"(\S*)";', line)
-			if m:
-				optname = m.groups()[0]
-				if optname == 'java_package':
-					javapkg = m.groups()[1]
-				elif optname == 'java_outer_classname':
-					javacn = m.groups()[1]
-			if self.env.PROTOC_MAJOR > '2':
-				m = re.search(r'^message\s+(\w*)\s*{*', line)
-				if m:
-					messages.append(m.groups()[0])
-
-		if javapkg:
-			nodename = javapkg
-		elif pkgname:
-			nodename = pkgname
-		else:
-			raise Errors.WafError('Cannot derive java name from protoc file')
-
-		nodename = nodename.replace('.',os.sep) + os.sep
-		if javacn:
-			nodename += javacn + '.java'
-		else:
-			if self.env.PROTOC_MAJOR > '2' and node.abspath()[node.abspath().rfind(os.sep)+1:node.abspath().rfind('.')].title() in messages:
-				nodename += node.abspath()[node.abspath().rfind(os.sep)+1:node.abspath().rfind('.')].title().replace('_','') + 'OuterClass.java'
-			else:
-				nodename += node.abspath()[node.abspath().rfind(os.sep)+1:node.abspath().rfind('.')].title().replace('_','') + '.java'
-
-		java_node = node.parent.find_or_declare(nodename)
-		out_nodes.append(java_node)
-		protoc_flags.append('--java_out=%s' % node.parent.get_bld().bldpath())
-
 		# Make javac get also pick java code generated in build
 		if not node.parent.get_bld() in self.javac_task.srcdir:
 			self.javac_task.srcdir.append(node.parent.get_bld())
 
-	if not out_nodes:
-		raise Errors.WafError('Feature %r not supported by protoc extra' % self.features)
+		protoc_flags.append('--java_out=%s' % node.parent.get_bld().bldpath())
 
 	tsk = self.create_task('protoc', node, out_nodes)
 	tsk.env.append_value('PROTOC_FLAGS', protoc_flags)
@@ -219,9 +186,22 @@ def process_protoc(self, node):
 	# For C++ standard include files dirs are used,
 	# but this doesn't apply to Python for example
 	for incpath in getattr(self, 'protoc_includes', []):
-		incdirs.append(self.path.find_node(incpath).bldpath())
+		incpath_node = self.path.find_node(incpath)
+		if incpath_node:
+			incdirs.append(incpath_node.bldpath())
+		else:
+			# Check if relative to top-level for extra tg dependencies
+			incpath_node = self.bld.path.find_node(incpath)
+			if incpath_node:
+				incdirs.append(incpath_node.bldpath())
+			else:
+				raise Errors.WafError('protoc: include path %r does not exist' % incpath)
+
 	tsk.env.PROTOC_INCPATHS = incdirs
 
+	# Include paths external to the waf project (ie. shared pb repositories)
+	tsk.env.PROTOC_EXTINCPATHS = getattr(self, 'protoc_extincludes', [])
+
 	# PR2115: protoc generates output of .proto files in nested
 	# directories  by canonicalizing paths. To avoid this we have to pass
 	# as first include the full directory file of the .proto file
diff --git a/extras/pyqt5.py b/extras/pyqt5.py
index c21dfa7..80f43b8 100644
--- a/extras/pyqt5.py
+++ b/extras/pyqt5.py
@@ -111,9 +111,9 @@ def apply_pyqt5(self):
 	"""
 	The additional parameters are:
 
-	:param lang: list of translation files (\*.ts) to process
+	:param lang: list of translation files (\\*.ts) to process
 	:type lang: list of :py:class:`waflib.Node.Node` or string without the .ts extension
-	:param langname: if given, transform the \*.ts files into a .qrc files to include in the binary file
+	:param langname: if given, transform the \\*.ts files into a .qrc files to include in the binary file
 	:type langname: :py:class:`waflib.Node.Node` or string without the .qrc extension
 	"""
 	if getattr(self, 'lang', None):
diff --git a/extras/qt4.py b/extras/qt4.py
index 90cae7e..d19a4dd 100644
--- a/extras/qt4.py
+++ b/extras/qt4.py
@@ -290,11 +290,11 @@ def apply_qt4(self):
 
 	The additional parameters are:
 
-	:param lang: list of translation files (\*.ts) to process
+	:param lang: list of translation files (\\*.ts) to process
 	:type lang: list of :py:class:`waflib.Node.Node` or string without the .ts extension
-	:param update: whether to process the C++ files to update the \*.ts files (use **waf --translate**)
+	:param update: whether to process the C++ files to update the \\*.ts files (use **waf --translate**)
 	:type update: bool
-	:param langname: if given, transform the \*.ts files into a .qrc files to include in the binary file
+	:param langname: if given, transform the \\*.ts files into a .qrc files to include in the binary file
 	:type langname: :py:class:`waflib.Node.Node` or string without the .qrc extension
 	"""
 	if getattr(self, 'lang', None):
diff --git a/extras/remote.py b/extras/remote.py
index 3b038f7..f43b600 100644
--- a/extras/remote.py
+++ b/extras/remote.py
@@ -203,7 +203,7 @@ class remote(BuildContext):
 					Options.commands.remove(k)
 
 	def login_to_host(self, login):
-		return re.sub('(\w+@)', '', login)
+		return re.sub(r'(\w+@)', '', login)
 
 	def variant_to_login(self, variant):
 		"""linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
diff --git a/extras/run_do_script.py b/extras/run_do_script.py
index f3c5812..07e3aa2 100644
--- a/extras/run_do_script.py
+++ b/extras/run_do_script.py
@@ -101,7 +101,7 @@ class run_do_script(run_do_script_base):
 		with open(**kwargs) as log:
 			log_tail = log.readlines()[-10:]
 			for line in log_tail:
-				error_found = re.match("r\(([0-9]+)\)", line)
+				error_found = re.match(r"r\(([0-9]+)\)", line)
 				if error_found:
 					return error_found.group(1), ''.join(log_tail)
 				else:
diff --git a/extras/swig.py b/extras/swig.py
index fd3d6d2..740ab46 100644
--- a/extras/swig.py
+++ b/extras/swig.py
@@ -17,10 +17,10 @@ tasks have to be added dynamically:
 
 SWIG_EXTS = ['.swig', '.i']
 
-re_module = re.compile('%module(?:\s*\(.*\))?\s+(.+)', re.M)
+re_module = re.compile(r'%module(?:\s*\(.*\))?\s+(.+)', re.M)
 
 re_1 = re.compile(r'^%module.*?\s+([\w]+)\s*?$', re.M)
-re_2 = re.compile('[#%]include [<"](.*)[">]', re.M)
+re_2 = re.compile(r'[#%](?:include|import(?:\(module=".*"\))+|python(?:begin|code)) [<"](.*)[">]', re.M)
 
 class swig(Task.Task):
 	color   = 'BLUE'