Merge commit 'c2b0355b0de23f6ec83b5fa5be038bcce208e26f' as 'waflib'

1 files changed, 327 insertions, 0 deletions

diff --git a/waflib/extras/remote.py b/waflib/extras/remote.py
new file mode 100644
index 0000000..3b038f7
--- /dev/null
+++ b/waflib/extras/remote.py
@@ -0,0 +1,327 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Remote Builds tool using rsync+ssh
+
+__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
+__copyright__ = "Jérôme Carretero, 2013"
+
+"""
+Simple Remote Builds
+********************
+
+This tool is an *experimental* tool (meaning, do not even try to pollute
+the waf bug tracker with bugs in here, contact me directly) providing simple
+remote builds.
+
+It uses rsync and ssh to perform the remote builds.
+It is intended for performing cross-compilation on platforms where
+a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product
+does not exist (eg. Windows builds using Visual Studio) or simply not installed.
+This tool sends the sources and the waf script to the remote host,
+and commands the usual waf execution.
+
+There are alternatives to using this tool, such as setting up shared folders,
+logging on to remote machines, and building on the shared folders.
+Electing one method or another depends on the size of the program.
+
+
+Usage
+=====
+
+1. Set your wscript file so it includes a list of variants,
+   e.g.::
+
+     from waflib import Utils
+     top = '.'
+     out = 'build'
+
+     variants = [
+      'linux_64_debug',
+      'linux_64_release',
+      'linux_32_debug',
+      'linux_32_release',
+      ]
+
+     from waflib.extras import remote
+
+     def options(opt):
+         # normal stuff from here on
+         opt.load('compiler_c')
+
+     def configure(conf):
+         if not conf.variant:
+             return
+         # normal stuff from here on
+         conf.load('compiler_c')
+
+     def build(bld):
+         if not bld.variant:
+             return
+         # normal stuff from here on
+         bld(features='c cprogram', target='app', source='main.c')
+
+
+2. Build the waf file, so it includes this tool, and put it in the current
+   directory
+
+   .. code:: bash
+
+      ./waf-light --tools=remote
+
+3. Set the host names to access the hosts:
+
+   .. code:: bash
+
+      export REMOTE_QNX=user@kiunix
+
+4. Setup the ssh server and ssh keys
+
+   The ssh key should not be protected by a password, or it will prompt for it every time.
+   Create the key on the client:
+
+   .. code:: bash
+
+      ssh-keygen -t rsa -f foo.rsa
+
+   Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys),
+   and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys)
+
+   A separate key for the build processes can be set in the environment variable WAF_SSH_KEY.
+   The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so
+   be warned to use this feature on internal networks only (MITM).
+
+   .. code:: bash
+
+      export WAF_SSH_KEY=~/foo.rsa
+
+5. Perform the build:
+
+   .. code:: bash
+
+      waf configure_all build_all --remote
+
+"""
+
+
+import getpass, os, re, sys
+from collections import OrderedDict
+from waflib import Context, Options, Utils, ConfigSet
+
+from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext
+from waflib.Configure import ConfigurationContext
+
+
+is_remote = False
+if '--remote' in sys.argv:
+	is_remote = True
+	sys.argv.remove('--remote')
+
+class init(Context.Context):
+	"""
+	Generates the *_all commands
+	"""
+	cmd = 'init'
+	fun = 'init'
+	def execute(self):
+		for x in list(Context.g_module.variants):
+			self.make_variant(x)
+		lst = ['remote']
+		for k in Options.commands:
+			if k.endswith('_all'):
+				name = k.replace('_all', '')
+				for x in Context.g_module.variants:
+					lst.append('%s_%s' % (name, x))
+			else:
+				lst.append(k)
+		del Options.commands[:]
+		Options.commands += lst
+
+	def make_variant(self, x):
+		for y in (BuildContext, CleanContext, InstallContext, UninstallContext):
+			name = y.__name__.replace('Context','').lower()
+			class tmp(y):
+				cmd = name + '_' + x
+				fun = 'build'
+				variant = x
+		class tmp(ConfigurationContext):
+			cmd = 'configure_' + x
+			fun = 'configure'
+			variant = x
+			def __init__(self, **kw):
+				ConfigurationContext.__init__(self, **kw)
+				self.setenv(x)
+
+class remote(BuildContext):
+	cmd = 'remote'
+	fun = 'build'
+
+	def get_ssh_hosts(self):
+		lst = []
+		for v in Context.g_module.variants:
+			self.env.HOST = self.login_to_host(self.variant_to_login(v))
+			cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env)
+			out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH)
+			lst.append(out.strip())
+		return lst
+
+	def setup_private_ssh_key(self):
+		"""
+		When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory
+		Make sure that the ssh key does not prompt for a password
+		"""
+		key = os.environ.get('WAF_SSH_KEY', '')
+		if not key:
+			return
+		if not os.path.isfile(key):
+			self.fatal('Key in WAF_SSH_KEY must point to a valid file')
+		self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh')
+		self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts')
+		self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key))
+		self.ssh_config = os.path.join(self.ssh_dir, 'config')
+		for x in self.ssh_hosts, self.ssh_key, self.ssh_config:
+			if not os.path.isfile(x):
+				if not os.path.isdir(self.ssh_dir):
+					os.makedirs(self.ssh_dir)
+				Utils.writef(self.ssh_key, Utils.readf(key), 'wb')
+				os.chmod(self.ssh_key, 448)
+
+				Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts()))
+				os.chmod(self.ssh_key, 448)
+
+				Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb')
+				os.chmod(self.ssh_config, 448)
+		self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key]
+		self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh')
+
+	def skip_unbuildable_variant(self):
+		# skip variants that cannot be built on this OS
+		for k in Options.commands:
+			a, _, b = k.partition('_')
+			if b in Context.g_module.variants:
+				c, _, _ = b.partition('_')
+				if c != Utils.unversioned_sys_platform():
+					Options.commands.remove(k)
+
+	def login_to_host(self, login):
+		return re.sub('(\w+@)', '', login)
+
+	def variant_to_login(self, variant):
+		"""linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
+		x = variant[:variant.rfind('_')]
+		ret = os.environ.get('REMOTE_' + x.upper(), '')
+		if not ret:
+			x = x[:x.find('_')]
+			ret = os.environ.get('REMOTE_' + x.upper(), '')
+		if not ret:
+			ret = '%s@localhost' % getpass.getuser()
+		return ret
+
+	def execute(self):
+		global is_remote
+		if not is_remote:
+			self.skip_unbuildable_variant()
+		else:
+			BuildContext.execute(self)
+
+	def restore(self):
+		self.top_dir = os.path.abspath(Context.g_module.top)
+		self.srcnode = self.root.find_node(self.top_dir)
+		self.path = self.srcnode
+
+		self.out_dir = os.path.join(self.top_dir, Context.g_module.out)
+		self.bldnode = self.root.make_node(self.out_dir)
+		self.bldnode.mkdir()
+
+		self.env = ConfigSet.ConfigSet()
+
+	def extract_groups_of_builds(self):
+		"""Return a dict mapping each variants to the commands to build"""
+		self.vgroups = {}
+		for x in reversed(Options.commands):
+			_, _, variant = x.partition('_')
+			if variant in Context.g_module.variants:
+				try:
+					dct = self.vgroups[variant]
+				except KeyError:
+					dct = self.vgroups[variant] = OrderedDict()
+				try:
+					dct[variant].append(x)
+				except KeyError:
+					dct[variant] = [x]
+				Options.commands.remove(x)
+
+	def custom_options(self, login):
+		try:
+			return Context.g_module.host_options[login]
+		except (AttributeError, KeyError):
+			return {}
+
+	def recurse(self, *k, **kw):
+		self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx')
+		self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh')
+		self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan')
+		try:
+			self.env.WAF = getattr(Context.g_module, 'waf')
+		except AttributeError:
+			try:
+				os.stat('waf')
+			except KeyError:
+				self.fatal('Put a waf file in the directory (./waf-light --tools=remote)')
+			else:
+				self.env.WAF = './waf'
+
+		self.extract_groups_of_builds()
+		self.setup_private_ssh_key()
+		for k, v in self.vgroups.items():
+			task = self(rule=rsync_and_ssh, always=True)
+			task.env.login = self.variant_to_login(k)
+
+			task.env.commands = []
+			for opt, value in v.items():
+				task.env.commands += value
+			task.env.variant = task.env.commands[0].partition('_')[2]
+			for opt, value in self.custom_options(k):
+				task.env[opt] = value
+		self.jobs = len(self.vgroups)
+
+	def make_mkdir_command(self, task):
+		return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env)
+
+	def make_send_command(self, task):
+		return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env)
+
+	def make_exec_command(self, task):
+		txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"'''
+		return Utils.subst_vars(txt, task.env)
+
+	def make_save_command(self, task):
+		return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env)
+
+def rsync_and_ssh(task):
+
+	# remove a warning
+	task.uid_ = id(task)
+
+	bld = task.generator.bld
+
+	task.env.user, _, _ = task.env.login.partition('@')
+	task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant)))
+	task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir)
+	task.env.local_dir = bld.srcnode.abspath() + '/'
+
+	task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant)
+	task.env.build_dir = bld.bldnode.abspath()
+
+	ret = task.exec_command(bld.make_mkdir_command(task))
+	if ret:
+		return ret
+	ret = task.exec_command(bld.make_send_command(task))
+	if ret:
+		return ret
+	ret = task.exec_command(bld.make_exec_command(task))
+	if ret:
+		return ret
+	ret = task.exec_command(bld.make_save_command(task))
+	if ret:
+		return ret
+