Merge commit 'c2b0355b0de23f6ec83b5fa5be038bcce208e26f' as 'waflib'

1 files changed, 170 insertions, 0 deletions

diff --git a/waflib/extras/win32_opts.py b/waflib/extras/win32_opts.py
new file mode 100644
index 00000000..9f7443c3
--- /dev/null
+++ b/waflib/extras/win32_opts.py
@@ -0,0 +1,170 @@
+#! /usr/bin/env python
+# encoding: utf-8
+
+"""
+Windows-specific optimizations
+
+This module can help reducing the overhead of listing files on windows
+(more than 10000 files). Python 3.5 already provides the listdir
+optimization though.
+"""
+
+import os
+from waflib import Utils, Build, Node, Logs
+
+try:
+	TP = '%s\\*'.decode('ascii')
+except AttributeError:
+	TP = '%s\\*'
+
+if Utils.is_win32:
+	from waflib.Tools import md5_tstamp
+	import ctypes, ctypes.wintypes
+
+	FindFirstFile        = ctypes.windll.kernel32.FindFirstFileW
+	FindNextFile         = ctypes.windll.kernel32.FindNextFileW
+	FindClose            = ctypes.windll.kernel32.FindClose
+	FILE_ATTRIBUTE_DIRECTORY = 0x10
+	INVALID_HANDLE_VALUE = -1
+	UPPER_FOLDERS = ('.', '..')
+	try:
+		UPPER_FOLDERS = [unicode(x) for x in UPPER_FOLDERS]
+	except NameError:
+		pass
+
+	def cached_hash_file(self):
+		try:
+			cache = self.ctx.cache_listdir_cache_hash_file
+		except AttributeError:
+			cache = self.ctx.cache_listdir_cache_hash_file = {}
+
+		if id(self.parent) in cache:
+			try:
+				t = cache[id(self.parent)][self.name]
+			except KeyError:
+				raise IOError('Not a file')
+		else:
+			# an opportunity to list the files and the timestamps at once
+			findData = ctypes.wintypes.WIN32_FIND_DATAW()
+			find     = FindFirstFile(TP % self.parent.abspath(), ctypes.byref(findData))
+
+			if find == INVALID_HANDLE_VALUE:
+				cache[id(self.parent)] = {}
+				raise IOError('Not a file')
+
+			cache[id(self.parent)] = lst_files = {}
+			try:
+				while True:
+					if findData.cFileName not in UPPER_FOLDERS:
+						thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
+						if not thatsadir:
+							ts = findData.ftLastWriteTime
+							d = (ts.dwLowDateTime << 32) | ts.dwHighDateTime
+							lst_files[str(findData.cFileName)] = d
+					if not FindNextFile(find, ctypes.byref(findData)):
+						break
+			except Exception:
+				cache[id(self.parent)] = {}
+				raise IOError('Not a file')
+			finally:
+				FindClose(find)
+			t = lst_files[self.name]
+
+		fname = self.abspath()
+		if fname in Build.hashes_md5_tstamp:
+			if Build.hashes_md5_tstamp[fname][0] == t:
+				return Build.hashes_md5_tstamp[fname][1]
+
+		try:
+			fd = os.open(fname, os.O_BINARY | os.O_RDONLY | os.O_NOINHERIT)
+		except OSError:
+			raise IOError('Cannot read from %r' % fname)
+		f = os.fdopen(fd, 'rb')
+		m = Utils.md5()
+		rb = 1
+		try:
+			while rb:
+				rb = f.read(200000)
+				m.update(rb)
+		finally:
+			f.close()
+
+		# ensure that the cache is overwritten
+		Build.hashes_md5_tstamp[fname] = (t, m.digest())
+		return m.digest()
+	Node.Node.cached_hash_file = cached_hash_file
+
+	def get_bld_sig_win32(self):
+		try:
+			return self.ctx.hash_cache[id(self)]
+		except KeyError:
+			pass
+		except AttributeError:
+			self.ctx.hash_cache = {}
+		self.ctx.hash_cache[id(self)] = ret = Utils.h_file(self.abspath())
+		return ret
+	Node.Node.get_bld_sig = get_bld_sig_win32
+
+	def isfile_cached(self):
+		# optimize for nt.stat calls, assuming there are many files for few folders
+		try:
+			cache = self.__class__.cache_isfile_cache
+		except AttributeError:
+			cache = self.__class__.cache_isfile_cache = {}
+
+		try:
+			c1 = cache[id(self.parent)]
+		except KeyError:
+			c1 = cache[id(self.parent)] = []
+
+			curpath = self.parent.abspath()
+			findData = ctypes.wintypes.WIN32_FIND_DATAW()
+			find     = FindFirstFile(TP % curpath, ctypes.byref(findData))
+
+			if find == INVALID_HANDLE_VALUE:
+				Logs.error("invalid win32 handle isfile_cached %r", self.abspath())
+				return os.path.isfile(self.abspath())
+
+			try:
+				while True:
+					if findData.cFileName not in UPPER_FOLDERS:
+						thatsadir = findData.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY
+						if not thatsadir:
+							c1.append(str(findData.cFileName))
+					if not FindNextFile(find, ctypes.byref(findData)):
+						break
+			except Exception as e:
+				Logs.error('exception while listing a folder %r %r', self.abspath(), e)
+				return os.path.isfile(self.abspath())
+			finally:
+				FindClose(find)
+		return self.name in c1
+	Node.Node.isfile_cached = isfile_cached
+
+	def find_or_declare_win32(self, lst):
+		# assuming that "find_or_declare" is called before the build starts, remove the calls to os.path.isfile
+		if isinstance(lst, str):
+			lst = [x for x in Utils.split_path(lst) if x and x != '.']
+
+		node = self.get_bld().search_node(lst)
+		if node:
+			if not node.isfile_cached():
+				try:
+					node.parent.mkdir()
+				except OSError:
+					pass
+			return node
+		self = self.get_src()
+		node = self.find_node(lst)
+		if node:
+			if not node.isfile_cached():
+				try:
+					node.parent.mkdir()
+				except OSError:
+					pass
+			return node
+		node = self.get_bld().make_node(lst)
+		node.parent.mkdir()
+		return node
+	Node.Node.find_or_declare = find_or_declare_win32
+