summaryrefslogtreecommitdiffstats
path: root/extras/batched_cc.py
diff options
context:
space:
mode:
authorDavid Robillard <d@drobilla.net>2019-03-17 17:31:05 +0100
committerDavid Robillard <d@drobilla.net>2019-03-17 17:31:05 +0100
commit406f89271452fdb573c7e28113b1ed08ff2b4eda (patch)
treed2dcbaf61f3749f73dc7a5e10d3fc6cd5e6e129a /extras/batched_cc.py
parent7983a5aae615290d04fd43cbc2752f8cf4a46d10 (diff)
downloadsuil-406f89271452fdb573c7e28113b1ed08ff2b4eda.tar.gz
suil-406f89271452fdb573c7e28113b1ed08ff2b4eda.tar.bz2
suil-406f89271452fdb573c7e28113b1ed08ff2b4eda.zip
Squashed 'waflib/' changes from 915dcb1..e7a29b6
e7a29b6 Upgrade to waf 2.0.15 8280f9d Add command for running executables from the build directory 8073c1a Make make_simple_dox() safe in case of exception 70d03b8 Avoid use of global counter hacks for configuration display b7d689a Rewrite test framework 94deadf Automatically add options and move add_flags() to options context f4259ee Reduce system include path noise 927b608 Automatically display configuration header c44b8f3 Set line justification from a constant in the wscript a48e26f Automatically detect if wscript has a test hook ef66724 Save runtime variables in the environment 63bcbcd Clean up TestContext b1d9505 Add ExecutionContext for setting runtime environment 387c1df Add show_diff() and test_file_equals() utilities 29d4d29 Fix in-tree library paths 9fde01f Add custom configuration context 6d3612f Add lib_path_name constant git-subtree-dir: waflib git-subtree-split: e7a29b6b9b2f842314244c23c14d8f8f560904e1
Diffstat (limited to 'extras/batched_cc.py')
-rw-r--r--extras/batched_cc.py173
1 files changed, 173 insertions, 0 deletions
diff --git a/extras/batched_cc.py b/extras/batched_cc.py
new file mode 100644
index 0000000..aad2872
--- /dev/null
+++ b/extras/batched_cc.py
@@ -0,0 +1,173 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Thomas Nagy, 2006-2015 (ita)
+
+"""
+Instead of compiling object files one by one, c/c++ compilers are often able to compile at once:
+cc -c ../file1.c ../file2.c ../file3.c
+
+Files are output on the directory where the compiler is called, and dependencies are more difficult
+to track (do not run the command on all source files if only one file changes)
+As such, we do as if the files were compiled one by one, but no command is actually run:
+replace each cc/cpp Task by a TaskSlave. A new task called TaskMaster collects the
+signatures from each slave and finds out the command-line to run.
+
+Just import this module to start using it:
+def build(bld):
+ bld.load('batched_cc')
+
+Note that this is provided as an example, unity builds are recommended
+for best performance results (fewer tasks and fewer jobs to execute).
+See waflib/extras/unity.py.
+"""
+
+from waflib import Task, Utils
+from waflib.TaskGen import extension, feature, after_method
+from waflib.Tools import c, cxx
+
+MAX_BATCH = 50
+
+c_str = '${CC} ${ARCH_ST:ARCH} ${CFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
+c_fun, _ = Task.compile_fun_noshell(c_str)
+
+cxx_str = '${CXX} ${ARCH_ST:ARCH} ${CXXFLAGS} ${FRAMEWORKPATH_ST:FRAMEWORKPATH} ${tsk.batch_incpaths()} ${DEFINES_ST:DEFINES} -c ${SRCLST} ${CXX_TGT_F_BATCHED} ${CPPFLAGS}'
+cxx_fun, _ = Task.compile_fun_noshell(cxx_str)
+
+count = 70000
+class batch(Task.Task):
+ color = 'PINK'
+
+ after = ['c', 'cxx']
+ before = ['cprogram', 'cshlib', 'cstlib', 'cxxprogram', 'cxxshlib', 'cxxstlib']
+
+ def uid(self):
+ return Utils.h_list([Task.Task.uid(self), self.generator.idx, self.generator.path.abspath(), self.generator.target])
+
+ def __str__(self):
+ return 'Batch compilation for %d slaves' % len(self.slaves)
+
+ def __init__(self, *k, **kw):
+ Task.Task.__init__(self, *k, **kw)
+ self.slaves = []
+ self.inputs = []
+ self.hasrun = 0
+
+ global count
+ count += 1
+ self.idx = count
+
+ def add_slave(self, slave):
+ self.slaves.append(slave)
+ self.set_run_after(slave)
+
+ def runnable_status(self):
+ for t in self.run_after:
+ if not t.hasrun:
+ return Task.ASK_LATER
+
+ for t in self.slaves:
+ #if t.executed:
+ if t.hasrun != Task.SKIPPED:
+ return Task.RUN_ME
+
+ return Task.SKIP_ME
+
+ def get_cwd(self):
+ return self.slaves[0].outputs[0].parent
+
+ def batch_incpaths(self):
+ st = self.env.CPPPATH_ST
+ return [st % node.abspath() for node in self.generator.includes_nodes]
+
+ def run(self):
+ self.outputs = []
+
+ srclst = []
+ slaves = []
+ for t in self.slaves:
+ if t.hasrun != Task.SKIPPED:
+ slaves.append(t)
+ srclst.append(t.inputs[0].abspath())
+
+ self.env.SRCLST = srclst
+
+ if self.slaves[0].__class__.__name__ == 'c':
+ ret = c_fun(self)
+ else:
+ ret = cxx_fun(self)
+
+ if ret:
+ return ret
+
+ for t in slaves:
+ t.old_post_run()
+
+def hook(cls_type):
+ def n_hook(self, node):
+
+ ext = '.obj' if self.env.CC_NAME == 'msvc' else '.o'
+ name = node.name
+ k = name.rfind('.')
+ if k >= 0:
+ basename = name[:k] + ext
+ else:
+ basename = name + ext
+
+ outdir = node.parent.get_bld().make_node('%d' % self.idx)
+ outdir.mkdir()
+ out = outdir.find_or_declare(basename)
+
+ task = self.create_task(cls_type, node, out)
+
+ try:
+ self.compiled_tasks.append(task)
+ except AttributeError:
+ self.compiled_tasks = [task]
+
+ if not getattr(self, 'masters', None):
+ self.masters = {}
+ self.allmasters = []
+
+ def fix_path(tsk):
+ if self.env.CC_NAME == 'msvc':
+ tsk.env.append_unique('CXX_TGT_F_BATCHED', '/Fo%s\\' % outdir.abspath())
+
+ if not node.parent in self.masters:
+ m = self.masters[node.parent] = self.master = self.create_task('batch')
+ fix_path(m)
+ self.allmasters.append(m)
+ else:
+ m = self.masters[node.parent]
+ if len(m.slaves) > MAX_BATCH:
+ m = self.masters[node.parent] = self.master = self.create_task('batch')
+ fix_path(m)
+ self.allmasters.append(m)
+ m.add_slave(task)
+ return task
+ return n_hook
+
+extension('.c')(hook('c'))
+extension('.cpp','.cc','.cxx','.C','.c++')(hook('cxx'))
+
+@feature('cprogram', 'cshlib', 'cstaticlib', 'cxxprogram', 'cxxshlib', 'cxxstlib')
+@after_method('apply_link')
+def link_after_masters(self):
+ if getattr(self, 'allmasters', None):
+ for m in self.allmasters:
+ self.link_task.set_run_after(m)
+
+# Modify the c and cxx task classes - in theory it would be best to
+# create subclasses and to re-map the c/c++ extensions
+for x in ('c', 'cxx'):
+ t = Task.classes[x]
+ def run(self):
+ pass
+
+ def post_run(self):
+ pass
+
+ setattr(t, 'oldrun', getattr(t, 'run', None))
+ setattr(t, 'run', run)
+ setattr(t, 'old_post_run', t.post_run)
+ setattr(t, 'post_run', post_run)
+