aboutsummaryrefslogtreecommitdiffstats
path: root/extras/win32_opts.py
diff options
context:
space:
mode:
Diffstat (limited to 'extras/win32_opts.py')
-rw-r--r--extras/win32_opts.py170
1 files changed, 170 insertions, 0 deletions
diff --git a/extras/win32_opts.py b/extras/win32_opts.py
new file mode 100644
index 0000000..9f7443c
--- /dev/null
+++ b/extras/win32_opts.py
@@ -0,0 +1,170 @@
+#! /usr/bin/env python
+# encoding: utf-8
+
+"""
+Windows-specific optimizations
+
+This module can help reducing the overhead of listing files on windows
+(more than 10000 files). Python 3.5 already provides the listdir
+optimization though.
+"""
+
+import os
+from waflib import Utils, Build, Node, Logs
+
+try:
+ TP = '%s\\*'.decode('ascii')
+except AttributeError:
+ TP = '%s\\*'
+
+if Utils.is_win32:
+ from waflib.Tools import md5_tstamp
+ import ctypes, ctypes.wintypes
+
+ FindFirstFile = ctypes.windll.kernel32.FindFirstFileW
+ FindNextFile = ctypes.windll.kernel32.FindNextFileW
+ FindClose = ctypes.windll.kernel32.FindClose
+ FILE_ATTRIBUTE_DIRECTORY = 0x10
+ INVALID_HANDLE_VALUE = -1
+ UPPER_FOLDERS = ('.', '..')
+ try:
+ UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS]
+ except NameError:
+ pass
+
+ def cached_hash_file(self):
+ try:
+ cache = self.ctx.cache_listdir_cache_hash_file
+ except AttributeError:
+ cache = self.ctx.cache_listdir_cache_hash_file = {}
+
+ if id(self.parent) in cache:
+ try:
+ t = cache[id(self.parent)][self.name]
+ except KeyError:
+ raise IOError('Not a file')
+ else:
+ # an opportunity to list the files and the timestamps at once
+ findData = ctypes.wintypes.WIN32_FIND_DATAW()
+ find = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData))
+
+ if find == INVALID_HANDLE_VALUE:
+ cache[id(self.parent)] = {}
+ raise IOError('Not a file')
+
+ cache[id(self.parent)] = lst_files = {}
+ try:
+ while True:
+ if findData.cFileName not in UPPER_FOLDERS:
+ thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
+ if not thatsadir:
+ ts = findData.ftLastWriteTime
+ d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime
+ lst_files[str(findData.cFileName)] = d
+ if not FindNextFile(find, ctypes.byref(findData)):
+ break
+ except Exception:
+ cache[id(self.parent)] = {}
+ raise IOError('Not a file')
+ finally:
+ FindClose(find)
+ t = lst_files[self.name]
+
+ fname = self.abspath()
+ if fname in Build.hashes_md5_tstamp:
+ if Build.hashes_md5_tstamp[fname][0] == t:
+ return Build.hashes_md5_tstamp[fname][1]
+
+ try:
+ fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT)
+ except OSError:
+ raise IOError('Cannot read from %r' % fname)
+ f = os.fdopen(fd, 'rb')
+ m = Utils.md5()
+ rb = 1
+ try:
+ while rb:
+ rb = f.read(200000)
+ m.update(rb)
+ finally:
+ f.close()
+
+ # ensure that the cache is overwritten
+ Build.hashes_md5_tstamp[fname] = (t, m.digest())
+ return m.digest()
+ Node.Node.cached_hash_file = cached_hash_file
+
+ def get_bld_sig_win32(self):
+ try:
+ return self.ctx.hash_cache[id(self)]
+ except KeyError:
+ pass
+ except AttributeError:
+ self.ctx.hash_cache = {}
+ self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath())
+ return ret
+ Node.Node.get_bld_sig = get_bld_sig_win32
+
+ def isfile_cached(self):
+ # optimize for nt.stat calls, assuming there are many files for few folders
+ try:
+ cache = self.__class__.cache_isfile_cache
+ except AttributeError:
+ cache = self.__class__.cache_isfile_cache = {}
+
+ try:
+ c1 = cache[id(self.parent)]
+ except KeyError:
+ c1 = cache[id(self.parent)] = []
+
+ curpath = self.parent.abspath()
+ findData = ctypes.wintypes.WIN32_FIND_DATAW()
+ find = FindFirstFile(TP % curpath, ctypes.byref(findData))
+
+ if find == INVALID_HANDLE_VALUE:
+ Logs.error("invalid win32 handle isfile_cached %r", self.abspath())
+ return os.path.isfile(self.abspath())
+
+ try:
+ while True:
+ if findData.cFileName not in UPPER_FOLDERS:
+ thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
+ if not thatsadir:
+ c1.append(str(findData.cFileName))
+ if not FindNextFile(find, ctypes.byref(findData)):
+ break
+ except Exception as e:
+ Logs.error('exception while listing a folder %r %r', self.abspath(), e)
+ return os.path.isfile(self.abspath())
+ finally:
+ FindClose(find)
+ return self.name in c1
+ Node.Node.isfile_cached = isfile_cached
+
+ def find_or_declare_win32(self, lst):
+ # assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile
+ if isinstance(lst, str):
+ lst = [x for x in Utils.split_path(lst) if x and x != '.']
+
+ node = self.get_bld().search_node(lst)
+ if node:
+ if not node.isfile_cached():
+ try:
+ node.parent.mkdir()
+ except OSError:
+ pass
+ return node
+ self = self.get_src()
+ node = self.find_node(lst)
+ if node:
+ if not node.isfile_cached():
+ try:
+ node.parent.mkdir()
+ except OSError:
+ pass
+ return node
+ node = self.get_bld().make_node(lst)
+ node.parent.mkdir()
+ return node
+ Node.Node.find_or_declare = find_or_declare_win32
+