diff options
Diffstat (limited to 'Node.py')
| -rw-r--r-- | Node.py | 970 |
1 files changed, 970 insertions, 0 deletions
@@ -0,0 +1,970 @@ +#!/usr/bin/env python +# encoding: utf-8 +# Thomas Nagy, 2005-2018 (ita) + +""" +Node: filesystem structure + +#. Each file/folder is represented by exactly one node. + +#. Some potential class properties are stored on :py:class:`waflib.Build.BuildContext` : nodes to depend on, etc. + Unused class members can increase the `.wafpickle` file size sensibly. + +#. Node objects should never be created directly, use + the methods :py:func:`Node.make_node` or :py:func:`Node.find_node` for the low-level operations + +#. The methods :py:func:`Node.find_resource`, :py:func:`Node.find_dir` :py:func:`Node.find_or_declare` must be + used when a build context is present + +#. Each instance of :py:class:`waflib.Context.Context` has a unique :py:class:`Node` subclass required for serialization. + (:py:class:`waflib.Node.Nod3`, see the :py:class:`waflib.Context.Context` initializer). A reference to the context + owning a node is held as *self.ctx* +""" + +import os, re, sys, shutil +from waflib import Utils, Errors + +exclude_regs = ''' +**/*~ +**/#*# +**/.#* +**/%*% +**/._* +**/*.swp +**/CVS +**/CVS/** +**/.cvsignore +**/SCCS +**/SCCS/** +**/vssver.scc +**/.svn +**/.svn/** +**/BitKeeper +**/.git +**/.git/** +**/.gitignore +**/.bzr +**/.bzrignore +**/.bzr/** +**/.hg +**/.hg/** +**/_MTN +**/_MTN/** +**/.arch-ids +**/{arch} +**/_darcs +**/_darcs/** +**/.intlcache +**/.DS_Store''' +""" +Ant patterns for files and folders to exclude while doing the +recursive traversal in :py:meth:`waflib.Node.Node.ant_glob` +""" + +def ant_matcher(s, ignorecase): + reflags = re.I if ignorecase else 0 + ret = [] + for x in Utils.to_list(s): + x = x.replace('\\', '/').replace('//', '/') + if x.endswith('/'): + x += '**' + accu = [] + for k in x.split('/'): + if k == '**': + accu.append(k) + else: + k = k.replace('.', '[.]').replace('*','.*').replace('?', '.').replace('+', '\\+') + k = '^%s$' % k + try: + exp = re.compile(k, flags=reflags) + except Exception as e: + raise Errors.WafError('Invalid pattern: %s' % k, e) + else: + accu.append(exp) + ret.append(accu) + return ret + +def ant_sub_filter(name, nn): + ret = [] + for lst in nn: + if not lst: + pass + elif lst[0] == '**': + ret.append(lst) + if len(lst) > 1: + if lst[1].match(name): + ret.append(lst[2:]) + else: + ret.append([]) + elif lst[0].match(name): + ret.append(lst[1:]) + return ret + +def ant_sub_matcher(name, pats): + nacc = ant_sub_filter(name, pats[0]) + nrej = ant_sub_filter(name, pats[1]) + if [] in nrej: + nacc = [] + return [nacc, nrej] + +class Node(object): + """ + This class is organized in two parts: + + * The basic methods meant for filesystem access (compute paths, create folders, etc) + * The methods bound to a :py:class:`waflib.Build.BuildContext` (require ``bld.srcnode`` and ``bld.bldnode``) + """ + + dict_class = dict + """ + Subclasses can provide a dict class to enable case insensitivity for example. + """ + + __slots__ = ('name', 'parent', 'children', 'cache_abspath', 'cache_isdir') + def __init__(self, name, parent): + """ + .. note:: Use :py:func:`Node.make_node` or :py:func:`Node.find_node` instead of calling this constructor + """ + self.name = name + self.parent = parent + if parent: + if name in parent.children: + raise Errors.WafError('node %s exists in the parent files %r already' % (name, parent)) + parent.children[name] = self + + def __setstate__(self, data): + "Deserializes node information, used for persistence" + self.name = data[0] + self.parent = data[1] + if data[2] is not None: + # Issue 1480 + self.children = self.dict_class(data[2]) + + def __getstate__(self): + "Serializes node information, used for persistence" + return (self.name, self.parent, getattr(self, 'children', None)) + + def __str__(self): + """ + String representation (abspath), for debugging purposes + + :rtype: string + """ + return self.abspath() + + def __repr__(self): + """ + String representation (abspath), for debugging purposes + + :rtype: string + """ + return self.abspath() + + def __copy__(self): + """ + Provided to prevent nodes from being copied + + :raises: :py:class:`waflib.Errors.WafError` + """ + raise Errors.WafError('nodes are not supposed to be copied') + + def read(self, flags='r', encoding='latin-1'): + """ + Reads and returns the contents of the file represented by this node, see :py:func:`waflib.Utils.readf`:: + + def build(bld): + bld.path.find_node('wscript').read() + + :param flags: Open mode + :type flags: string + :param encoding: encoding value for Python3 + :type encoding: string + :rtype: string or bytes + :return: File contents + """ + return Utils.readf(self.abspath(), flags, encoding) + + def write(self, data, flags='w', encoding='latin-1'): + """ + Writes data to the file represented by this node, see :py:func:`waflib.Utils.writef`:: + + def build(bld): + bld.path.make_node('foo.txt').write('Hello, world!') + + :param data: data to write + :type data: string + :param flags: Write mode + :type flags: string + :param encoding: encoding value for Python3 + :type encoding: string + """ + Utils.writef(self.abspath(), data, flags, encoding) + + def read_json(self, convert=True, encoding='utf-8'): + """ + Reads and parses the contents of this node as JSON (Python ≥ 2.6):: + + def build(bld): + bld.path.find_node('abc.json').read_json() + + Note that this by default automatically decodes unicode strings on Python2, unlike what the Python JSON module does. + + :type convert: boolean + :param convert: Prevents decoding of unicode strings on Python2 + :type encoding: string + :param encoding: The encoding of the file to read. This default to UTF8 as per the JSON standard + :rtype: object + :return: Parsed file contents + """ + import json # Python 2.6 and up + object_pairs_hook = None + if convert and sys.hexversion < 0x3000000: + try: + _type = unicode + except NameError: + _type = str + + def convert(value): + if isinstance(value, list): + return [convert(element) for element in value] + elif isinstance(value, _type): + return str(value) + else: + return value + + def object_pairs(pairs): + return dict((str(pair[0]), convert(pair[1])) for pair in pairs) + + object_pairs_hook = object_pairs + + return json.loads(self.read(encoding=encoding), object_pairs_hook=object_pairs_hook) + + def write_json(self, data, pretty=True): + """ + Writes a python object as JSON to disk (Python ≥ 2.6) as UTF-8 data (JSON standard):: + + def build(bld): + bld.path.find_node('xyz.json').write_json(199) + + :type data: object + :param data: The data to write to disk + :type pretty: boolean + :param pretty: Determines if the JSON will be nicely space separated + """ + import json # Python 2.6 and up + indent = 2 + separators = (',', ': ') + sort_keys = pretty + newline = os.linesep + if not pretty: + indent = None + separators = (',', ':') + newline = '' + output = json.dumps(data, indent=indent, separators=separators, sort_keys=sort_keys) + newline + self.write(output, encoding='utf-8') + + def exists(self): + """ + Returns whether the Node is present on the filesystem + + :rtype: bool + """ + return os.path.exists(self.abspath()) + + def isdir(self): + """ + Returns whether the Node represents a folder + + :rtype: bool + """ + return os.path.isdir(self.abspath()) + + def chmod(self, val): + """ + Changes the file/dir permissions:: + + def build(bld): + bld.path.chmod(493) # 0755 + """ + os.chmod(self.abspath(), val) + + def delete(self, evict=True): + """ + Removes the file/folder from the filesystem (equivalent to `rm -rf`), and remove this object from the Node tree. + Do not use this object after calling this method. + """ + try: + try: + if os.path.isdir(self.abspath()): + shutil.rmtree(self.abspath()) + else: + os.remove(self.abspath()) + except OSError: + if os.path.exists(self.abspath()): + raise + finally: + if evict: + self.evict() + + def evict(self): + """ + Removes this node from the Node tree + """ + del self.parent.children[self.name] + + def suffix(self): + """ + Returns the file rightmost extension, for example `a.b.c.d → .d` + + :rtype: string + """ + k = max(0, self.name.rfind('.')) + return self.name[k:] + + def height(self): + """ + Returns the depth in the folder hierarchy from the filesystem root or from all the file drives + + :returns: filesystem depth + :rtype: integer + """ + d = self + val = -1 + while d: + d = d.parent + val += 1 + return val + + def listdir(self): + """ + Lists the folder contents + + :returns: list of file/folder names ordered alphabetically + :rtype: list of string + """ + lst = Utils.listdir(self.abspath()) + lst.sort() + return lst + + def mkdir(self): + """ + Creates a folder represented by this node. Intermediate folders are created as needed. + + :raises: :py:class:`waflib.Errors.WafError` when the folder is missing + """ + if self.isdir(): + return + + try: + self.parent.mkdir() + except OSError: + pass + + if self.name: + try: + os.makedirs(self.abspath()) + except OSError: + pass + + if not self.isdir(): + raise Errors.WafError('Could not create the directory %r' % self) + + try: + self.children + except AttributeError: + self.children = self.dict_class() + + def find_node(self, lst): + """ + Finds a node on the file system (files or folders), and creates the corresponding Node objects if it exists + + :param lst: relative path + :type lst: string or list of string + :returns: The corresponding Node object or None if no entry was found on the filesystem + :rtype: :py:class:´waflib.Node.Node´ + """ + + if isinstance(lst, str): + lst = [x for x in Utils.split_path(lst) if x and x != '.'] + + if lst and lst[0].startswith('\\\\') and not self.parent: + node = self.ctx.root.make_node(lst[0]) + node.cache_isdir = True + return node.find_node(lst[1:]) + + cur = self + for x in lst: + if x == '..': + cur = cur.parent or cur + continue + + try: + ch = cur.children + except AttributeError: + cur.children = self.dict_class() + else: + try: + cur = ch[x] + continue + except KeyError: + pass + + # optimistic: create the node first then look if it was correct to do so + cur = self.__class__(x, cur) + if not cur.exists(): + cur.evict() + return None + + if not cur.exists(): + cur.evict() + return None + + return cur + + def make_node(self, lst): + """ + Returns or creates a Node object corresponding to the input path without considering the filesystem. + + :param lst: relative path + :type lst: string or list of string + :rtype: :py:class:´waflib.Node.Node´ + """ + if isinstance(lst, str): + lst = [x for x in Utils.split_path(lst) if x and x != '.'] + + cur = self + for x in lst: + if x == '..': + cur = cur.parent or cur + continue + + try: + cur = cur.children[x] + except AttributeError: + cur.children = self.dict_class() + except KeyError: + pass + else: + continue + cur = self.__class__(x, cur) + return cur + + def search_node(self, lst): + """ + Returns a Node previously defined in the data structure. The filesystem is not considered. + + :param lst: relative path + :type lst: string or list of string + :rtype: :py:class:´waflib.Node.Node´ or None if there is no entry in the Node datastructure + """ + if isinstance(lst, str): + lst = [x for x in Utils.split_path(lst) if x and x != '.'] + + cur = self + for x in lst: + if x == '..': + cur = cur.parent or cur + else: + try: + cur = cur.children[x] + except (AttributeError, KeyError): + return None + return cur + + def path_from(self, node): + """ + Path of this node seen from the other:: + + def build(bld): + n1 = bld.path.find_node('foo/bar/xyz.txt') + n2 = bld.path.find_node('foo/stuff/') + n1.path_from(n2) # '../bar/xyz.txt' + + :param node: path to use as a reference + :type node: :py:class:`waflib.Node.Node` + :returns: a relative path or an absolute one if that is better + :rtype: string + """ + c1 = self + c2 = node + + c1h = c1.height() + c2h = c2.height() + + lst = [] + up = 0 + + while c1h > c2h: + lst.append(c1.name) + c1 = c1.parent + c1h -= 1 + + while c2h > c1h: + up += 1 + c2 = c2.parent + c2h -= 1 + + while not c1 is c2: + lst.append(c1.name) + up += 1 + + c1 = c1.parent + c2 = c2.parent + + if c1.parent: + lst.extend(['..'] * up) + lst.reverse() + return os.sep.join(lst) or '.' + else: + return self.abspath() + + def abspath(self): + """ + Returns the absolute path. A cache is kept in the context as ``cache_node_abspath`` + + :rtype: string + """ + try: + return self.cache_abspath + except AttributeError: + pass + # think twice before touching this (performance + complexity + correctness) + + if not self.parent: + val = os.sep + elif not self.parent.name: + val = os.sep + self.name + else: + val = self.parent.abspath() + os.sep + self.name + self.cache_abspath = val + return val + + if Utils.is_win32: + def abspath(self): + try: + return self.cache_abspath + except AttributeError: + pass + if not self.parent: + val = '' + elif not self.parent.name: + val = self.name + os.sep + else: + val = self.parent.abspath().rstrip(os.sep) + os.sep + self.name + self.cache_abspath = val + return val + + def is_child_of(self, node): + """ + Returns whether the object belongs to a subtree of the input node:: + + def build(bld): + node = bld.path.find_node('wscript') + node.is_child_of(bld.path) # True + + :param node: path to use as a reference + :type node: :py:class:`waflib.Node.Node` + :rtype: bool + """ + p = self + diff = self.height() - node.height() + while diff > 0: + diff -= 1 + p = p.parent + return p is node + + def ant_iter(self, accept=None, maxdepth=25, pats=[], dir=False, src=True, remove=True, quiet=False): + """ + Recursive method used by :py:meth:`waflib.Node.ant_glob`. + + :param accept: function used for accepting/rejecting a node, returns the patterns that can be still accepted in recursion + :type accept: function + :param maxdepth: maximum depth in the filesystem (25) + :type maxdepth: int + :param pats: list of patterns to accept and list of patterns to exclude + :type pats: tuple + :param dir: return folders too (False by default) + :type dir: bool + :param src: return files (True by default) + :type src: bool + :param remove: remove files/folders that do not exist (True by default) + :type remove: bool + :param quiet: disable build directory traversal warnings (verbose mode) + :type quiet: bool + :returns: A generator object to iterate from + :rtype: iterator + """ + dircont = self.listdir() + dircont.sort() + + try: + lst = set(self.children.keys()) + except AttributeError: + self.children = self.dict_class() + else: + if remove: + for x in lst - set(dircont): + self.children[x].evict() + + for name in dircont: + npats = accept(name, pats) + if npats and npats[0]: + accepted = [] in npats[0] + + node = self.make_node([name]) + + isdir = node.isdir() + if accepted: + if isdir: + if dir: + yield node + elif src: + yield node + + if isdir: + node.cache_isdir = True + if maxdepth: + for k in node.ant_iter(accept=accept, maxdepth=maxdepth - 1, pats=npats, dir=dir, src=src, remove=remove, quiet=quiet): + yield k + + def ant_glob(self, *k, **kw): + """ + Finds files across folders and returns Node objects: + + * ``**/*`` find all files recursively + * ``**/*.class`` find all files ending by .class + * ``..`` find files having two dot characters + + For example:: + + def configure(cfg): + # find all .cpp files + cfg.path.ant_glob('**/*.cpp') + # find particular files from the root filesystem (can be slow) + cfg.root.ant_glob('etc/*.txt') + # simple exclusion rule example + cfg.path.ant_glob('*.c*', excl=['*.c'], src=True, dir=False) + + For more information about the patterns, consult http://ant.apache.org/manual/dirtasks.html + Please remember that the '..' sequence does not represent the parent directory:: + + def configure(cfg): + cfg.path.ant_glob('../*.h') # incorrect + cfg.path.parent.ant_glob('*.h') # correct + + The Node structure is itself a filesystem cache, so certain precautions must + be taken while matching files in the build or installation phases. + Nodes objects that do have a corresponding file or folder are garbage-collected by default. + This garbage collection is usually required to prevent returning files that do not + exist anymore. Yet, this may also remove Node objects of files that are yet-to-be built. + + This typically happens when trying to match files in the build directory, + but there are also cases when files are created in the source directory. + Run ``waf -v`` to display any warnings, and try consider passing ``remove=False`` + when matching files in the build directory. + + Since ant_glob can traverse both source and build folders, it is a best practice + to call this method only from the most specific build node:: + + def build(bld): + # traverses the build directory, may need ``remove=False``: + bld.path.ant_glob('project/dir/**/*.h') + # better, no accidental build directory traversal: + bld.path.find_node('project/dir').ant_glob('**/*.h') # best + + In addition, files and folders are listed immediately. When matching files in the + build folders, consider passing ``generator=True`` so that the generator object + returned can defer computation to a later stage. For example:: + + def build(bld): + bld(rule='tar xvf ${SRC}', source='arch.tar') + bld.add_group() + gen = bld.bldnode.ant_glob("*.h", generator=True, remove=True) + # files will be listed only after the arch.tar is unpacked + bld(rule='ls ${SRC}', source=gen, name='XYZ') + + + :param incl: ant patterns or list of patterns to include + :type incl: string or list of strings + :param excl: ant patterns or list of patterns to exclude + :type excl: string or list of strings + :param dir: return folders too (False by default) + :type dir: bool + :param src: return files (True by default) + :type src: bool + :param maxdepth: maximum depth of recursion + :type maxdepth: int + :param ignorecase: ignore case while matching (False by default) + :type ignorecase: bool + :param generator: Whether to evaluate the Nodes lazily + :type generator: bool + :param remove: remove files/folders that do not exist (True by default) + :type remove: bool + :param quiet: disable build directory traversal warnings (verbose mode) + :type quiet: bool + :returns: The corresponding Node objects as a list or as a generator object (generator=True) + :rtype: by default, list of :py:class:`waflib.Node.Node` instances + """ + src = kw.get('src', True) + dir = kw.get('dir') + excl = kw.get('excl', exclude_regs) + incl = k and k[0] or kw.get('incl', '**') + remove = kw.get('remove', True) + maxdepth = kw.get('maxdepth', 25) + ignorecase = kw.get('ignorecase', False) + quiet = kw.get('quiet', False) + pats = (ant_matcher(incl, ignorecase), ant_matcher(excl, ignorecase)) + + if kw.get('generator'): + return Utils.lazy_generator(self.ant_iter, (ant_sub_matcher, maxdepth, pats, dir, src, remove, quiet)) + + it = self.ant_iter(ant_sub_matcher, maxdepth, pats, dir, src, remove, quiet) + if kw.get('flat'): + # returns relative paths as a space-delimited string + # prefer Node objects whenever possible + return ' '.join(x.path_from(self) for x in it) + return list(it) + + # ---------------------------------------------------------------------------- + # the methods below require the source/build folders (bld.srcnode/bld.bldnode) + + def is_src(self): + """ + Returns True if the node is below the source directory. Note that ``!is_src() ≠ is_bld()`` + + :rtype: bool + """ + cur = self + x = self.ctx.srcnode + y = self.ctx.bldnode + while cur.parent: + if cur is y: + return False + if cur is x: + return True + cur = cur.parent + return False + + def is_bld(self): + """ + Returns True if the node is below the build directory. Note that ``!is_bld() ≠ is_src()`` + + :rtype: bool + """ + cur = self + y = self.ctx.bldnode + while cur.parent: + if cur is y: + return True + cur = cur.parent + return False + + def get_src(self): + """ + Returns the corresponding Node object in the source directory (or self if already + under the source directory). Use this method only if the purpose is to create + a Node object (this is common with folders but not with files, see ticket 1937) + + :rtype: :py:class:`waflib.Node.Node` + """ + cur = self + x = self.ctx.srcnode + y = self.ctx.bldnode + lst = [] + while cur.parent: + if cur is y: + lst.reverse() + return x.make_node(lst) + if cur is x: + return self + lst.append(cur.name) + cur = cur.parent + return self + + def get_bld(self): + """ + Return the corresponding Node object in the build directory (or self if already + under the build directory). Use this method only if the purpose is to create + a Node object (this is common with folders but not with files, see ticket 1937) + + :rtype: :py:class:`waflib.Node.Node` + """ + cur = self + x = self.ctx.srcnode + y = self.ctx.bldnode + lst = [] + while cur.parent: + if cur is y: + return self + if cur is x: + lst.reverse() + return self.ctx.bldnode.make_node(lst) + lst.append(cur.name) + cur = cur.parent + # the file is external to the current project, make a fake root in the current build directory + lst.reverse() + if lst and Utils.is_win32 and len(lst[0]) == 2 and lst[0].endswith(':'): + lst[0] = lst[0][0] + return self.ctx.bldnode.make_node(['__root__'] + lst) + + def find_resource(self, lst): + """ + Use this method in the build phase to find source files corresponding to the relative path given. + + First it looks up the Node data structure to find any declared Node object in the build directory. + If None is found, it then considers the filesystem in the source directory. + + :param lst: relative path + :type lst: string or list of string + :returns: the corresponding Node object or None + :rtype: :py:class:`waflib.Node.Node` + """ + if isinstance(lst, str): + lst = [x for x in Utils.split_path(lst) if x and x != '.'] + + node = self.get_bld().search_node(lst) + if not node: + node = self.get_src().find_node(lst) + if node and node.isdir(): + return None + return node + + def find_or_declare(self, lst): + """ + Use this method in the build phase to declare output files which + are meant to be written in the build directory. + + This method creates the Node object and its parent folder + as needed. + + :param lst: relative path + :type lst: string or list of string + """ + if isinstance(lst, str) and os.path.isabs(lst): + node = self.ctx.root.make_node(lst) + else: + node = self.get_bld().make_node(lst) + node.parent.mkdir() + return node + + def find_dir(self, lst): + """ + Searches for a folder on the filesystem (see :py:meth:`waflib.Node.Node.find_node`) + + :param lst: relative path + :type lst: string or list of string + :returns: The corresponding Node object or None if there is no such folder + :rtype: :py:class:`waflib.Node.Node` + """ + if isinstance(lst, str): + lst = [x for x in Utils.split_path(lst) if x and x != '.'] + + node = self.find_node(lst) + if node and not node.isdir(): + return None + return node + + # helpers for building things + def change_ext(self, ext, ext_in=None): + """ + Declares a build node with a distinct extension; this is uses :py:meth:`waflib.Node.Node.find_or_declare` + + :return: A build node of the same path, but with a different extension + :rtype: :py:class:`waflib.Node.Node` + """ + name = self.name + if ext_in is None: + k = name.rfind('.') + if k >= 0: + name = name[:k] + ext + else: + name = name + ext + else: + name = name[:- len(ext_in)] + ext + + return self.parent.find_or_declare([name]) + + def bldpath(self): + """ + Returns the relative path seen from the build directory ``src/foo.cpp`` + + :rtype: string + """ + return self.path_from(self.ctx.bldnode) + + def srcpath(self): + """ + Returns the relative path seen from the source directory ``../src/foo.cpp`` + + :rtype: string + """ + return self.path_from(self.ctx.srcnode) + + def relpath(self): + """ + If a file in the build directory, returns :py:meth:`waflib.Node.Node.bldpath`, + else returns :py:meth:`waflib.Node.Node.srcpath` + + :rtype: string + """ + cur = self + x = self.ctx.bldnode + while cur.parent: + if cur is x: + return self.bldpath() + cur = cur.parent + return self.srcpath() + + def bld_dir(self): + """ + Equivalent to self.parent.bldpath() + + :rtype: string + """ + return self.parent.bldpath() + + def h_file(self): + """ + See :py:func:`waflib.Utils.h_file` + + :return: a hash representing the file contents + :rtype: string or bytes + """ + return Utils.h_file(self.abspath()) + + def get_bld_sig(self): + """ + Returns a signature (see :py:meth:`waflib.Node.Node.h_file`) for the purpose + of build dependency calculation. This method uses a per-context cache. + + :return: a hash representing the object contents + :rtype: string or bytes + """ + # previous behaviour can be set by returning self.ctx.node_sigs[self] when a build node + try: + cache = self.ctx.cache_sig + except AttributeError: + cache = self.ctx.cache_sig = {} + try: + ret = cache[self] + except KeyError: + p = self.abspath() + try: + ret = cache[self] = self.h_file() + except EnvironmentError: + if self.isdir(): + # allow folders as build nodes, do not use the creation time + st = os.stat(p) + ret = cache[self] = Utils.h_list([p, st.st_ino, st.st_mode]) + return ret + raise + return ret + +pickle_lock = Utils.threading.Lock() +"""Lock mandatory for thread-safe node serialization""" + +class Nod3(Node): + """Mandatory subclass for thread-safe node serialization""" + pass # do not remove + + |