diff options
Diffstat (limited to 'extras/fast_partial.py')
| -rw-r--r-- | extras/fast_partial.py | 518 |
1 files changed, 0 insertions, 518 deletions
diff --git a/extras/fast_partial.py b/extras/fast_partial.py deleted file mode 100644 index d5b6144..0000000 --- a/extras/fast_partial.py +++ /dev/null @@ -1,518 +0,0 @@ -#! /usr/bin/env python -# encoding: utf-8 -# Thomas Nagy, 2017-2018 (ita) - -""" -A system for fast partial rebuilds - -Creating a large amount of task objects up front can take some time. -By making a few assumptions, it is possible to avoid posting creating -task objects for targets that are already up-to-date. - -On a silly benchmark the gain observed for 1M tasks can be 5m->10s -for a single file change. - -Usage:: - - def options(opt): - opt.load('fast_partial') - -Assumptions: -* Mostly for C/C++/Fortran targets with link tasks (object-only targets are not handled) -* For full project builds: no --targets and no pruning from subfolders -* The installation phase is ignored -* `use=` dependencies are specified up front even across build groups -* Task generator source files are not obtained from globs - -Implementation details: -* The first layer obtains file timestamps to recalculate file hashes only - when necessary (similar to md5_tstamp); the timestamps are then stored - in a dedicated pickle file -* A second layer associates each task generator to a file set to help - detecting changes. Task generators are to create their tasks only when - the related files have been modified. A specific db file is created - to store such data (5m -> 1m10) -* A third layer binds build context proxies onto task generators, replacing - the default context. While loading data for the full build uses more memory - (4GB -> 9GB), partial builds are then much faster (1m10 -> 13s) -* A fourth layer enables a 2-level cache on file signatures to - reduce the size of the main pickle file (13s -> 10s) -""" - -import os -from waflib import Build, Context, Errors, Logs, Task, TaskGen, Utils -from waflib.TaskGen import feature, after_method, taskgen_method -import waflib.Node - -DONE = 0 -DIRTY = 1 -NEEDED = 2 - -SKIPPABLE = ['cshlib', 'cxxshlib', 'cstlib', 'cxxstlib', 'cprogram', 'cxxprogram'] - -TSTAMP_DB = '.wafpickle_tstamp_db_file' - -SAVED_ATTRS = 'root node_sigs task_sigs imp_sigs raw_deps node_deps'.split() - -class bld_proxy(object): - def __init__(self, bld): - object.__setattr__(self, 'bld', bld) - - object.__setattr__(self, 'node_class', type('Nod3', (waflib.Node.Node,), {})) - self.node_class.__module__ = 'waflib.Node' - self.node_class.ctx = self - - object.__setattr__(self, 'root', self.node_class('', None)) - for x in SAVED_ATTRS: - if x != 'root': - object.__setattr__(self, x, {}) - - self.fix_nodes() - - def __setattr__(self, name, value): - bld = object.__getattribute__(self, 'bld') - setattr(bld, name, value) - - def __delattr__(self, name): - bld = object.__getattribute__(self, 'bld') - delattr(bld, name) - - def __getattribute__(self, name): - try: - return object.__getattribute__(self, name) - except AttributeError: - bld = object.__getattribute__(self, 'bld') - return getattr(bld, name) - - def __call__(self, *k, **kw): - return self.bld(*k, **kw) - - def fix_nodes(self): - for x in ('srcnode', 'path', 'bldnode'): - node = self.root.find_dir(getattr(self.bld, x).abspath()) - object.__setattr__(self, x, node) - - def set_key(self, store_key): - object.__setattr__(self, 'store_key', store_key) - - def fix_tg_path(self, *tgs): - # changing Node objects on task generators is possible - # yet, all Node objects must belong to the same parent - for tg in tgs: - tg.path = self.root.make_node(tg.path.abspath()) - - def restore(self): - dbfn = os.path.join(self.variant_dir, Context.DBFILE + self.store_key) - Logs.debug('rev_use: reading %s', dbfn) - try: - data = Utils.readf(dbfn, 'rb') - except (EnvironmentError, EOFError): - # handle missing file/empty file - Logs.debug('rev_use: Could not load the build cache %s (missing)', dbfn) - else: - try: - waflib.Node.pickle_lock.acquire() - waflib.Node.Nod3 = self.node_class - try: - data = Build.cPickle.loads(data) - except Exception as e: - Logs.debug('rev_use: Could not pickle the build cache %s: %r', dbfn, e) - else: - for x in SAVED_ATTRS: - object.__setattr__(self, x, data.get(x, {})) - finally: - waflib.Node.pickle_lock.release() - self.fix_nodes() - - def store(self): - data = {} - for x in Build.SAVED_ATTRS: - data[x] = getattr(self, x) - db = os.path.join(self.variant_dir, Context.DBFILE + self.store_key) - - try: - waflib.Node.pickle_lock.acquire() - waflib.Node.Nod3 = self.node_class - x = Build.cPickle.dumps(data, Build.PROTOCOL) - finally: - waflib.Node.pickle_lock.release() - - Logs.debug('rev_use: storing %s', db) - Utils.writef(db + '.tmp', x, m='wb') - try: - st = os.stat(db) - os.remove(db) - if not Utils.is_win32: - os.chown(db + '.tmp', st.st_uid, st.st_gid) - except (AttributeError, OSError): - pass - os.rename(db + '.tmp', db) - -class bld(Build.BuildContext): - def __init__(self, **kw): - super(bld, self).__init__(**kw) - self.hashes_md5_tstamp = {} - - def __call__(self, *k, **kw): - # this is one way of doing it, one could use a task generator method too - bld = kw['bld'] = bld_proxy(self) - ret = TaskGen.task_gen(*k, **kw) - self.task_gen_cache_names = {} - self.add_to_group(ret, group=kw.get('group')) - ret.bld = bld - bld.set_key(ret.path.abspath().replace(os.sep, '') + str(ret.idx)) - return ret - - def is_dirty(self): - return True - - def store_tstamps(self): - # Called after a build is finished - # For each task generator, record all files involved in task objects - # optimization: done only if there was something built - do_store = False - try: - f_deps = self.f_deps - except AttributeError: - f_deps = self.f_deps = {} - self.f_tstamps = {} - - allfiles = set() - for g in self.groups: - for tg in g: - try: - staleness = tg.staleness - except AttributeError: - staleness = DIRTY - - if staleness != DIRTY: - # DONE case: there was nothing built - # NEEDED case: the tg was brought in because of 'use' propagation - # but nothing really changed for them, there may be incomplete - # tasks (object files) and in this case it is best to let the next build - # figure out if an input/output file changed - continue - - do_cache = False - for tsk in tg.tasks: - if tsk.hasrun == Task.SUCCESS: - do_cache = True - pass - elif tsk.hasrun == Task.SKIPPED: - pass - else: - # one failed task, clear the cache for this tg - try: - del f_deps[(tg.path.abspath(), tg.idx)] - except KeyError: - pass - else: - # just store the new state because there is a change - do_store = True - - # skip the rest because there is no valid cache possible - break - else: - if not do_cache: - # all skipped, but is there anything in cache? - try: - f_deps[(tg.path.abspath(), tg.idx)] - except KeyError: - # probably cleared because a wscript file changed - # store it - do_cache = True - - if do_cache: - - # there was a rebuild, store the data structure too - tg.bld.store() - - # all tasks skipped but no cache - # or a successful task build - do_store = True - st = set() - for tsk in tg.tasks: - st.update(tsk.inputs) - st.update(self.node_deps.get(tsk.uid(), [])) - - # TODO do last/when loading the tgs? - lst = [] - for k in ('wscript', 'wscript_build'): - n = tg.path.find_node(k) - if n: - n.get_bld_sig() - lst.append(n.abspath()) - - lst.extend(sorted(x.abspath() for x in st)) - allfiles.update(lst) - f_deps[(tg.path.abspath(), tg.idx)] = lst - - for x in allfiles: - # f_tstamps has everything, while md5_tstamp can be relatively empty on partial builds - self.f_tstamps[x] = self.hashes_md5_tstamp[x][0] - - if do_store: - dbfn = os.path.join(self.variant_dir, TSTAMP_DB) - Logs.debug('rev_use: storing %s', dbfn) - dbfn_tmp = dbfn + '.tmp' - x = Build.cPickle.dumps([self.f_tstamps, f_deps], Build.PROTOCOL) - Utils.writef(dbfn_tmp, x, m='wb') - os.rename(dbfn_tmp, dbfn) - Logs.debug('rev_use: stored %s', dbfn) - - def store(self): - self.store_tstamps() - if self.producer.dirty: - Build.BuildContext.store(self) - - def compute_needed_tgs(self): - # assume the 'use' keys are not modified during the build phase - - dbfn = os.path.join(self.variant_dir, TSTAMP_DB) - Logs.debug('rev_use: Loading %s', dbfn) - try: - data = Utils.readf(dbfn, 'rb') - except (EnvironmentError, EOFError): - Logs.debug('rev_use: Could not load the build cache %s (missing)', dbfn) - self.f_deps = {} - self.f_tstamps = {} - else: - try: - self.f_tstamps, self.f_deps = Build.cPickle.loads(data) - except Exception as e: - Logs.debug('rev_use: Could not pickle the build cache %s: %r', dbfn, e) - self.f_deps = {} - self.f_tstamps = {} - else: - Logs.debug('rev_use: Loaded %s', dbfn) - - - # 1. obtain task generators that contain rebuilds - # 2. obtain the 'use' graph and its dual - stales = set() - reverse_use_map = Utils.defaultdict(list) - use_map = Utils.defaultdict(list) - - for g in self.groups: - for tg in g: - if tg.is_stale(): - stales.add(tg) - - try: - lst = tg.use = Utils.to_list(tg.use) - except AttributeError: - pass - else: - for x in lst: - try: - xtg = self.get_tgen_by_name(x) - except Errors.WafError: - pass - else: - use_map[tg].append(xtg) - reverse_use_map[xtg].append(tg) - - Logs.debug('rev_use: found %r stale tgs', len(stales)) - - # 3. dfs to post downstream tg as stale - visited = set() - def mark_down(tg): - if tg in visited: - return - visited.add(tg) - Logs.debug('rev_use: marking down %r as stale', tg.name) - tg.staleness = DIRTY - for x in reverse_use_map[tg]: - mark_down(x) - for tg in stales: - mark_down(tg) - - # 4. dfs to find ancestors tg to mark as needed - self.needed_tgs = needed_tgs = set() - def mark_needed(tg): - if tg in needed_tgs: - return - needed_tgs.add(tg) - if tg.staleness == DONE: - Logs.debug('rev_use: marking up %r as needed', tg.name) - tg.staleness = NEEDED - for x in use_map[tg]: - mark_needed(x) - for xx in visited: - mark_needed(xx) - - # so we have the whole tg trees to post in the set "needed" - # load their build trees - for tg in needed_tgs: - tg.bld.restore() - tg.bld.fix_tg_path(tg) - - # the stale ones should be fully build, while the needed ones - # may skip a few tasks, see create_compiled_task and apply_link_after below - Logs.debug('rev_use: amount of needed task gens: %r', len(needed_tgs)) - - def post_group(self): - # assumption: we can ignore the folder/subfolders cuts - def tgpost(tg): - try: - f = tg.post - except AttributeError: - pass - else: - f() - - if not self.targets or self.targets == '*': - for tg in self.groups[self.current_group]: - # this can cut quite a lot of tg objects - if tg in self.needed_tgs: - tgpost(tg) - else: - # default implementation - return Build.BuildContext.post_group() - - def get_build_iterator(self): - if not self.targets or self.targets == '*': - self.compute_needed_tgs() - return Build.BuildContext.get_build_iterator(self) - -@taskgen_method -def is_stale(self): - # assume no globs - self.staleness = DIRTY - - # 1. the case of always stale targets - if getattr(self, 'always_stale', False): - return True - - # 2. check if the db file exists - db = os.path.join(self.bld.variant_dir, Context.DBFILE) - try: - dbstat = os.stat(db).st_mtime - except OSError: - Logs.debug('rev_use: must post %r because this is a clean build') - return True - - # 3. check if the configuration changed - if os.stat(self.bld.bldnode.find_node('c4che/build.config.py').abspath()).st_mtime > dbstat: - Logs.debug('rev_use: must post %r because the configuration has changed', self.name) - return True - - # 3.a any tstamp data? - try: - f_deps = self.bld.f_deps - except AttributeError: - Logs.debug('rev_use: must post %r because there is no f_deps', self.name) - return True - - # 4. check if this is the first build (no cache) - try: - lst = f_deps[(self.path.abspath(), self.idx)] - except KeyError: - Logs.debug('rev_use: must post %r because there it has no cached data', self.name) - return True - - try: - cache = self.bld.cache_tstamp_rev_use - except AttributeError: - cache = self.bld.cache_tstamp_rev_use = {} - - # 5. check the timestamp of each dependency files listed is unchanged - f_tstamps = self.bld.f_tstamps - for x in lst: - try: - old_ts = f_tstamps[x] - except KeyError: - Logs.debug('rev_use: must post %r because %r is not in cache', self.name, x) - return True - - try: - try: - ts = cache[x] - except KeyError: - ts = cache[x] = os.stat(x).st_mtime - except OSError: - del f_deps[(self.path.abspath(), self.idx)] - Logs.debug('rev_use: must post %r because %r does not exist anymore', self.name, x) - return True - else: - if ts != old_ts: - Logs.debug('rev_use: must post %r because the timestamp on %r changed %r %r', self.name, x, old_ts, ts) - return True - - self.staleness = DONE - return False - -@taskgen_method -def create_compiled_task(self, name, node): - # skip the creation of object files - # assumption: object-only targets are not skippable - if self.staleness == NEEDED: - # only libraries/programs can skip object files - for x in SKIPPABLE: - if x in self.features: - return None - - out = '%s.%d.o' % (node.name, self.idx) - task = self.create_task(name, node, node.parent.find_or_declare(out)) - try: - self.compiled_tasks.append(task) - except AttributeError: - self.compiled_tasks = [task] - return task - -@feature(*SKIPPABLE) -@after_method('apply_link') -def apply_link_after(self): - # cprogram/cxxprogram might be unnecessary - if self.staleness != NEEDED: - return - for tsk in self.tasks: - tsk.hasrun = Task.SKIPPED - -def path_from(self, node): - # handle nodes of distinct types - if node.ctx is not self.ctx: - node = self.ctx.root.make_node(node.abspath()) - return self.default_path_from(node) -waflib.Node.Node.default_path_from = waflib.Node.Node.path_from -waflib.Node.Node.path_from = path_from - -def h_file(self): - # similar to md5_tstamp.py, but with 2-layer cache - # global_cache for the build context common for all task generators - # local_cache for the build context proxy (one by task generator) - # - # the global cache is not persistent - # the local cache is persistent and meant for partial builds - # - # assume all calls are made from a single thread - # - filename = self.abspath() - st = os.stat(filename) - - global_cache = self.ctx.bld.hashes_md5_tstamp - local_cache = self.ctx.hashes_md5_tstamp - - if filename in global_cache: - # value already calculated in this build - cval = global_cache[filename] - - # the value in global cache is assumed to be calculated once - # reverifying it could cause task generators - # to get distinct tstamp values, thus missing rebuilds - local_cache[filename] = cval - return cval[1] - - if filename in local_cache: - cval = local_cache[filename] - if cval[0] == st.st_mtime: - # correct value from a previous build - # put it in the global cache - global_cache[filename] = cval - return cval[1] - - ret = Utils.h_file(filename) - local_cache[filename] = global_cache[filename] = (st.st_mtime, ret) - return ret -waflib.Node.Node.h_file = h_file - |