From 8c96b17a5393bffee0c521c4872a9fa999048032 Mon Sep 17 00:00:00 2001 From: David Robillard Date: Sat, 15 Sep 2018 14:51:02 +0200 Subject: Squashed 'waflib/' content from commit 982416b git-subtree-dir: waflib git-subtree-split: 982416b8a6c6728e200243e1be3ab60435c08830 --- extras/parallel_debug.py | 459 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 extras/parallel_debug.py (limited to 'extras/parallel_debug.py') diff --git a/extras/parallel_debug.py b/extras/parallel_debug.py new file mode 100644 index 0000000..35883a3 --- /dev/null +++ b/extras/parallel_debug.py @@ -0,0 +1,459 @@ +#! /usr/bin/env python +# encoding: utf-8 +# Thomas Nagy, 2007-2010 (ita) + +""" +Debugging helper for parallel compilation, outputs +a file named pdebug.svg in the source directory:: + + def options(opt): + opt.load('parallel_debug') + def build(bld): + ... +""" + +import re, sys, threading, time, traceback +try: + from Queue import Queue +except: + from queue import Queue +from waflib import Runner, Options, Task, Logs, Errors + +SVG_TEMPLATE = """ + + + + + + + + + + +${if project.title} + ${project.title} +${endif} + + +${for cls in project.groups} + + ${for rect in cls.rects} + + ${endfor} + +${endfor} + +${for info in project.infos} + + + ${info.text} + +${endfor} + +${if project.tooltip} + + + + +${endif} + + +""" + +COMPILE_TEMPLATE = '''def f(project): + lst = [] + def xml_escape(value): + return value.replace("&", "&").replace('"', """).replace("'", "'").replace("<", "<").replace(">", ">") + + %s + return ''.join(lst) +''' +reg_act = re.compile(r"(?P\\)|(?P\$\$)|(?P\$\{(?P[^}]*?)\})", re.M) +def compile_template(line): + + extr = [] + def repl(match): + g = match.group + if g('dollar'): + return "$" + elif g('backslash'): + return "\\" + elif g('subst'): + extr.append(g('code')) + return "<<|@|>>" + return None + + line2 = reg_act.sub(repl, line) + params = line2.split('<<|@|>>') + assert(extr) + + + indent = 0 + buf = [] + app = buf.append + + def app(txt): + buf.append(indent * '\t' + txt) + + for x in range(len(extr)): + if params[x]: + app("lst.append(%r)" % params[x]) + + f = extr[x] + if f.startswith(('if', 'for')): + app(f + ':') + indent += 1 + elif f.startswith('py:'): + app(f[3:]) + elif f.startswith(('endif', 'endfor')): + indent -= 1 + elif f.startswith(('else', 'elif')): + indent -= 1 + app(f + ':') + indent += 1 + elif f.startswith('xml:'): + app('lst.append(xml_escape(%s))' % f[4:]) + else: + #app('lst.append((%s) or "cannot find %s")' % (f, f)) + app('lst.append(str(%s))' % f) + + if extr: + if params[-1]: + app("lst.append(%r)" % params[-1]) + + fun = COMPILE_TEMPLATE % "\n\t".join(buf) + # uncomment the following to debug the template + #for i, x in enumerate(fun.splitlines()): + # print i, x + return Task.funex(fun) + +# red #ff4d4d +# green #4da74d +# lila #a751ff + +color2code = { + 'GREEN' : '#4da74d', + 'YELLOW' : '#fefe44', + 'PINK' : '#a751ff', + 'RED' : '#cc1d1d', + 'BLUE' : '#6687bb', + 'CYAN' : '#34e2e2', +} + +mp = {} +info = [] # list of (text,color) + +def map_to_color(name): + if name in mp: + return mp[name] + try: + cls = Task.classes[name] + except KeyError: + return color2code['RED'] + if cls.color in mp: + return mp[cls.color] + if cls.color in color2code: + return color2code[cls.color] + return color2code['RED'] + +def process(self): + m = self.generator.bld.producer + try: + # TODO another place for this? + del self.generator.bld.task_sigs[self.uid()] + except KeyError: + pass + + self.generator.bld.producer.set_running(1, self) + + try: + ret = self.run() + except Exception: + self.err_msg = traceback.format_exc() + self.hasrun = Task.EXCEPTION + + # TODO cleanup + m.error_handler(self) + return + + if ret: + self.err_code = ret + self.hasrun = Task.CRASHED + else: + try: + self.post_run() + except Errors.WafError: + pass + except Exception: + self.err_msg = traceback.format_exc() + self.hasrun = Task.EXCEPTION + else: + self.hasrun = Task.SUCCESS + if self.hasrun != Task.SUCCESS: + m.error_handler(self) + + self.generator.bld.producer.set_running(-1, self) + +Task.Task.process_back = Task.Task.process +Task.Task.process = process + +old_start = Runner.Parallel.start +def do_start(self): + try: + Options.options.dband + except AttributeError: + self.bld.fatal('use def options(opt): opt.load("parallel_debug")!') + + self.taskinfo = Queue() + old_start(self) + if self.dirty: + make_picture(self) +Runner.Parallel.start = do_start + +lock_running = threading.Lock() +def set_running(self, by, tsk): + with lock_running: + try: + cache = self.lock_cache + except AttributeError: + cache = self.lock_cache = {} + + i = 0 + if by > 0: + vals = cache.values() + for i in range(self.numjobs): + if i not in vals: + cache[tsk] = i + break + else: + i = cache[tsk] + del cache[tsk] + + self.taskinfo.put( (i, id(tsk), time.time(), tsk.__class__.__name__, self.processed, self.count, by, ",".join(map(str, tsk.outputs))) ) +Runner.Parallel.set_running = set_running + +def name2class(name): + return name.replace(' ', '_').replace('.', '_') + +def make_picture(producer): + # first, cast the parameters + if not hasattr(producer.bld, 'path'): + return + + tmp = [] + try: + while True: + tup = producer.taskinfo.get(False) + tmp.append(list(tup)) + except: + pass + + try: + ini = float(tmp[0][2]) + except: + return + + if not info: + seen = [] + for x in tmp: + name = x[3] + if not name in seen: + seen.append(name) + else: + continue + + info.append((name, map_to_color(name))) + info.sort(key=lambda x: x[0]) + + thread_count = 0 + acc = [] + for x in tmp: + thread_count += x[6] + acc.append("%d %d %f %r %d %d %d %s" % (x[0], x[1], x[2] - ini, x[3], x[4], x[5], thread_count, x[7])) + + data_node = producer.bld.path.make_node('pdebug.dat') + data_node.write('\n'.join(acc)) + + tmp = [lst[:2] + [float(lst[2]) - ini] + lst[3:] for lst in tmp] + + st = {} + for l in tmp: + if not l[0] in st: + st[l[0]] = len(st.keys()) + tmp = [ [st[lst[0]]] + lst[1:] for lst in tmp ] + THREAD_AMOUNT = len(st.keys()) + + st = {} + for l in tmp: + if not l[1] in st: + st[l[1]] = len(st.keys()) + tmp = [ [lst[0]] + [st[lst[1]]] + lst[2:] for lst in tmp ] + + + BAND = Options.options.dband + + seen = {} + acc = [] + for x in range(len(tmp)): + line = tmp[x] + id = line[1] + + if id in seen: + continue + seen[id] = True + + begin = line[2] + thread_id = line[0] + for y in range(x + 1, len(tmp)): + line = tmp[y] + if line[1] == id: + end = line[2] + #print id, thread_id, begin, end + #acc.append( ( 10*thread_id, 10*(thread_id+1), 10*begin, 10*end ) ) + acc.append( (BAND * begin, BAND*thread_id, BAND*end - BAND*begin, BAND, line[3], line[7]) ) + break + + if Options.options.dmaxtime < 0.1: + gwidth = 1 + for x in tmp: + m = BAND * x[2] + if m > gwidth: + gwidth = m + else: + gwidth = BAND * Options.options.dmaxtime + + ratio = float(Options.options.dwidth) / gwidth + gwidth = Options.options.dwidth + gheight = BAND * (THREAD_AMOUNT + len(info) + 1.5) + + + # simple data model for our template + class tobject(object): + pass + + model = tobject() + model.x = 0 + model.y = 0 + model.width = gwidth + 4 + model.height = gheight + 4 + + model.tooltip = not Options.options.dnotooltip + + model.title = Options.options.dtitle + model.title_x = gwidth / 2 + model.title_y = gheight + - 5 + + groups = {} + for (x, y, w, h, clsname, name) in acc: + try: + groups[clsname].append((x, y, w, h, name)) + except: + groups[clsname] = [(x, y, w, h, name)] + + # groups of rectangles (else js highlighting is slow) + model.groups = [] + for cls in groups: + g = tobject() + model.groups.append(g) + g.classname = name2class(cls) + g.rects = [] + for (x, y, w, h, name) in groups[cls]: + r = tobject() + g.rects.append(r) + r.x = 2 + x * ratio + r.y = 2 + y + r.width = w * ratio + r.height = h + r.name = name + r.color = map_to_color(cls) + + cnt = THREAD_AMOUNT + + # caption + model.infos = [] + for (text, color) in info: + inf = tobject() + model.infos.append(inf) + inf.classname = name2class(text) + inf.x = 2 + BAND + inf.y = 5 + (cnt + 0.5) * BAND + inf.width = BAND/2 + inf.height = BAND/2 + inf.color = color + + inf.text = text + inf.text_x = 2 + 2 * BAND + inf.text_y = 5 + (cnt + 0.5) * BAND + 10 + + cnt += 1 + + # write the file... + template1 = compile_template(SVG_TEMPLATE) + txt = template1(model) + + node = producer.bld.path.make_node('pdebug.svg') + node.write(txt) + Logs.warn('Created the diagram %r', node) + +def options(opt): + opt.add_option('--dtitle', action='store', default='Parallel build representation for %r' % ' '.join(sys.argv), + help='title for the svg diagram', dest='dtitle') + opt.add_option('--dwidth', action='store', type='int', help='diagram width', default=800, dest='dwidth') + opt.add_option('--dtime', action='store', type='float', help='recording interval in seconds', default=0.009, dest='dtime') + opt.add_option('--dband', action='store', type='int', help='band width', default=22, dest='dband') + opt.add_option('--dmaxtime', action='store', type='float', help='maximum time, for drawing fair comparisons', default=0, dest='dmaxtime') + opt.add_option('--dnotooltip', action='store_true', help='disable tooltips', default=False, dest='dnotooltip') + -- cgit v1.2.1