diff options
Diffstat (limited to 'extras/win32_opts.py')
| -rw-r--r-- | extras/win32_opts.py | 170 |
1 files changed, 170 insertions, 0 deletions
diff --git a/extras/win32_opts.py b/extras/win32_opts.py new file mode 100644 index 0000000..9f7443c --- /dev/null +++ b/extras/win32_opts.py @@ -0,0 +1,170 @@ +#! /usr/bin/env python +# encoding: utf-8 + +""" +Windows-specific optimizations + +This module can help reducing the overhead of listing files on windows +(more than 10000 files). Python 3.5 already provides the listdir +optimization though. +""" + +import os +from waflib import Utils, Build, Node, Logs + +try: + TP = '%s\\*'.decode('ascii') +except AttributeError: + TP = '%s\\*' + +if Utils.is_win32: + from waflib.Tools import md5_tstamp + import ctypes, ctypes.wintypes + + FindFirstFile = ctypes.windll.kernel32.FindFirstFileW + FindNextFile = ctypes.windll.kernel32.FindNextFileW + FindClose = ctypes.windll.kernel32.FindClose + FILE_ATTRIBUTE_DIRECTORY = 0x10 + INVALID_HANDLE_VALUE = -1 + UPPER_FOLDERS = ('.', '..') + try: + UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS] + except NameError: + pass + + def cached_hash_file(self): + try: + cache = self.ctx.cache_listdir_cache_hash_file + except AttributeError: + cache = self.ctx.cache_listdir_cache_hash_file = {} + + if id(self.parent) in cache: + try: + t = cache[id(self.parent)][self.name] + except KeyError: + raise IOError('Not a file') + else: + # an opportunity to list the files and the timestamps at once + findData = ctypes.wintypes.WIN32_FIND_DATAW() + find = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData)) + + if find == INVALID_HANDLE_VALUE: + cache[id(self.parent)] = {} + raise IOError('Not a file') + + cache[id(self.parent)] = lst_files = {} + try: + while True: + if findData.cFileName not in UPPER_FOLDERS: + thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + if not thatsadir: + ts = findData.ftLastWriteTime + d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime + lst_files[str(findData.cFileName)] = d + if not FindNextFile(find, ctypes.byref(findData)): + break + except Exception: + cache[id(self.parent)] = {} + raise IOError('Not a file') + finally: + FindClose(find) + t = lst_files[self.name] + + fname = self.abspath() + if fname in Build.hashes_md5_tstamp: + if Build.hashes_md5_tstamp[fname][0] == t: + return Build.hashes_md5_tstamp[fname][1] + + try: + fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT) + except OSError: + raise IOError('Cannot read from %r' % fname) + f = os.fdopen(fd, 'rb') + m = Utils.md5() + rb = 1 + try: + while rb: + rb = f.read(200000) + m.update(rb) + finally: + f.close() + + # ensure that the cache is overwritten + Build.hashes_md5_tstamp[fname] = (t, m.digest()) + return m.digest() + Node.Node.cached_hash_file = cached_hash_file + + def get_bld_sig_win32(self): + try: + return self.ctx.hash_cache[id(self)] + except KeyError: + pass + except AttributeError: + self.ctx.hash_cache = {} + self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath()) + return ret + Node.Node.get_bld_sig = get_bld_sig_win32 + + def isfile_cached(self): + # optimize for nt.stat calls, assuming there are many files for few folders + try: + cache = self.__class__.cache_isfile_cache + except AttributeError: + cache = self.__class__.cache_isfile_cache = {} + + try: + c1 = cache[id(self.parent)] + except KeyError: + c1 = cache[id(self.parent)] = [] + + curpath = self.parent.abspath() + findData = ctypes.wintypes.WIN32_FIND_DATAW() + find = FindFirstFile(TP % curpath, ctypes.byref(findData)) + + if find == INVALID_HANDLE_VALUE: + Logs.error("invalid win32 handle isfile_cached %r", self.abspath()) + return os.path.isfile(self.abspath()) + + try: + while True: + if findData.cFileName not in UPPER_FOLDERS: + thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY + if not thatsadir: + c1.append(str(findData.cFileName)) + if not FindNextFile(find, ctypes.byref(findData)): + break + except Exception as e: + Logs.error('exception while listing a folder %r %r', self.abspath(), e) + return os.path.isfile(self.abspath()) + finally: + FindClose(find) + return self.name in c1 + Node.Node.isfile_cached = isfile_cached + + def find_or_declare_win32(self, lst): + # assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile + if isinstance(lst, str): + lst = [x for x in Utils.split_path(lst) if x and x != '.'] + + node = self.get_bld().search_node(lst) + if node: + if not node.isfile_cached(): + try: + node.parent.mkdir() + except OSError: + pass + return node + self = self.get_src() + node = self.find_node(lst) + if node: + if not node.isfile_cached(): + try: + node.parent.mkdir() + except OSError: + pass + return node + node = self.get_bld().make_node(lst) + node.parent.mkdir() + return node + Node.Node.find_or_declare = find_or_declare_win32 + |