diff options
Diffstat (limited to 'waflib/extras/waf_xattr.py')
| -rw-r--r-- | waflib/extras/waf_xattr.py | 150 |
1 files changed, 150 insertions, 0 deletions
diff --git a/waflib/extras/waf_xattr.py b/waflib/extras/waf_xattr.py new file mode 100644 index 0000000..351dd63 --- /dev/null +++ b/waflib/extras/waf_xattr.py @@ -0,0 +1,150 @@ +#! /usr/bin/env python +# encoding: utf-8 + +""" +Use extended attributes instead of database files + +1. Input files will be made writable +2. This is only for systems providing extended filesystem attributes +3. By default, hashes are calculated only if timestamp/size change (HASH_CACHE below) +4. The module enables "deep_inputs" on all tasks by propagating task signatures +5. This module also skips task signature comparisons for task code changes due to point 4. +6. This module is for Python3/Linux only, but it could be extended to Python2/other systems + using the xattr library +7. For projects in which tasks always declare output files, it should be possible to + store the rest of build context attributes on output files (imp_sigs, raw_deps and node_deps) + but this is not done here + +On a simple C++ project benchmark, the variations before and after adding waf_xattr.py were observed: +total build time: 20s -> 22s +no-op build time: 2.4s -> 1.8s +pickle file size: 2.9MB -> 2.6MB +""" + +import os +from waflib import Logs, Node, Task, Utils, Errors +from waflib.Task import SKIP_ME, RUN_ME, CANCEL_ME, ASK_LATER, SKIPPED, MISSING + +HASH_CACHE = True +SIG_VAR = 'user.waf.sig' +SEP = ','.encode() +TEMPLATE = '%b%d,%d'.encode() + +try: + PermissionError +except NameError: + PermissionError = IOError + +def getxattr(self): + return os.getxattr(self.abspath(), SIG_VAR) + +def setxattr(self, val): + os.setxattr(self.abspath(), SIG_VAR, val) + +def h_file(self): + try: + ret = getxattr(self) + except OSError: + if HASH_CACHE: + st = os.stat(self.abspath()) + mtime = st.st_mtime + size = st.st_size + else: + if len(ret) == 16: + # for build directory files + return ret + + if HASH_CACHE: + # check if timestamp and mtime match to avoid re-hashing + st = os.stat(self.abspath()) + mtime, size = ret[16:].split(SEP) + if int(1000 * st.st_mtime) == int(mtime) and st.st_size == int(size): + return ret[:16] + + ret = Utils.h_file(self.abspath()) + if HASH_CACHE: + val = TEMPLATE % (ret, int(1000 * st.st_mtime), int(st.st_size)) + try: + setxattr(self, val) + except PermissionError: + os.chmod(self.abspath(), st.st_mode | 128) + setxattr(self, val) + return ret + +def runnable_status(self): + bld = self.generator.bld + if bld.is_install < 0: + return SKIP_ME + + for t in self.run_after: + if not t.hasrun: + return ASK_LATER + elif t.hasrun < SKIPPED: + # a dependency has an error + return CANCEL_ME + + # first compute the signature + try: + new_sig = self.signature() + except Errors.TaskNotReady: + return ASK_LATER + + if not self.outputs: + # compare the signature to a signature computed previously + # this part is only for tasks with no output files + key = self.uid() + try: + prev_sig = bld.task_sigs[key] + except KeyError: + Logs.debug('task: task %r must run: it was never run before or the task code changed', self) + return RUN_ME + if new_sig != prev_sig: + Logs.debug('task: task %r must run: the task signature changed', self) + return RUN_ME + + # compare the signatures of the outputs to make a decision + for node in self.outputs: + try: + sig = node.h_file() + except EnvironmentError: + Logs.debug('task: task %r must run: an output node does not exist', self) + return RUN_ME + if sig != new_sig: + Logs.debug('task: task %r must run: an output node is stale', self) + return RUN_ME + + return (self.always_run and RUN_ME) or SKIP_ME + +def post_run(self): + bld = self.generator.bld + sig = self.signature() + for node in self.outputs: + if not node.exists(): + self.hasrun = MISSING + self.err_msg = '-> missing file: %r' % node.abspath() + raise Errors.WafError(self.err_msg) + os.setxattr(node.abspath(), 'user.waf.sig', sig) + if not self.outputs: + # only for task with no outputs + bld.task_sigs[self.uid()] = sig + if not self.keep_last_cmd: + try: + del self.last_cmd + except AttributeError: + pass + +try: + os.getxattr +except AttributeError: + pass +else: + h_file.__doc__ = Node.Node.h_file.__doc__ + + # keep file hashes as file attributes + Node.Node.h_file = h_file + + # enable "deep_inputs" on all tasks + Task.Task.runnable_status = runnable_status + Task.Task.post_run = post_run + Task.Task.sig_deep_inputs = Utils.nada + |