summaryrefslogtreecommitdiffstats
path: root/waflib/extras/remote.py
diff options
context:
space:
mode:
Diffstat (limited to 'waflib/extras/remote.py')
-rw-r--r--waflib/extras/remote.py327
1 files changed, 327 insertions, 0 deletions
diff --git a/waflib/extras/remote.py b/waflib/extras/remote.py
new file mode 100644
index 0000000..f43b600
--- /dev/null
+++ b/waflib/extras/remote.py
@@ -0,0 +1,327 @@
+#!/usr/bin/env python
+# encoding: utf-8
+# Remote Builds tool using rsync+ssh
+
+__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>"
+__copyright__ = "Jérôme Carretero, 2013"
+
+"""
+Simple Remote Builds
+********************
+
+This tool is an *experimental* tool (meaning, do not even try to pollute
+the waf bug tracker with bugs in here, contact me directly) providing simple
+remote builds.
+
+It uses rsync and ssh to perform the remote builds.
+It is intended for performing cross-compilation on platforms where
+a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product
+does not exist (eg. Windows builds using Visual Studio) or simply not installed.
+This tool sends the sources and the waf script to the remote host,
+and commands the usual waf execution.
+
+There are alternatives to using this tool, such as setting up shared folders,
+logging on to remote machines, and building on the shared folders.
+Electing one method or another depends on the size of the program.
+
+
+Usage
+=====
+
+1. Set your wscript file so it includes a list of variants,
+ e.g.::
+
+ from waflib import Utils
+ top = '.'
+ out = 'build'
+
+ variants = [
+ 'linux_64_debug',
+ 'linux_64_release',
+ 'linux_32_debug',
+ 'linux_32_release',
+ ]
+
+ from waflib.extras import remote
+
+ def options(opt):
+ # normal stuff from here on
+ opt.load('compiler_c')
+
+ def configure(conf):
+ if not conf.variant:
+ return
+ # normal stuff from here on
+ conf.load('compiler_c')
+
+ def build(bld):
+ if not bld.variant:
+ return
+ # normal stuff from here on
+ bld(features='c cprogram', target='app', source='main.c')
+
+
+2. Build the waf file, so it includes this tool, and put it in the current
+ directory
+
+ .. code:: bash
+
+ ./waf-light --tools=remote
+
+3. Set the host names to access the hosts:
+
+ .. code:: bash
+
+ export REMOTE_QNX=user@kiunix
+
+4. Setup the ssh server and ssh keys
+
+ The ssh key should not be protected by a password, or it will prompt for it every time.
+ Create the key on the client:
+
+ .. code:: bash
+
+ ssh-keygen -t rsa -f foo.rsa
+
+ Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys),
+ and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys)
+
+ A separate key for the build processes can be set in the environment variable WAF_SSH_KEY.
+ The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so
+ be warned to use this feature on internal networks only (MITM).
+
+ .. code:: bash
+
+ export WAF_SSH_KEY=~/foo.rsa
+
+5. Perform the build:
+
+ .. code:: bash
+
+ waf configure_all build_all --remote
+
+"""
+
+
+import getpass, os, re, sys
+from collections import OrderedDict
+from waflib import Context, Options, Utils, ConfigSet
+
+from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext
+from waflib.Configure import ConfigurationContext
+
+
+is_remote = False
+if '--remote' in sys.argv:
+ is_remote = True
+ sys.argv.remove('--remote')
+
+class init(Context.Context):
+ """
+ Generates the *_all commands
+ """
+ cmd = 'init'
+ fun = 'init'
+ def execute(self):
+ for x in list(Context.g_module.variants):
+ self.make_variant(x)
+ lst = ['remote']
+ for k in Options.commands:
+ if k.endswith('_all'):
+ name = k.replace('_all', '')
+ for x in Context.g_module.variants:
+ lst.append('%s_%s' % (name, x))
+ else:
+ lst.append(k)
+ del Options.commands[:]
+ Options.commands += lst
+
+ def make_variant(self, x):
+ for y in (BuildContext, CleanContext, InstallContext, UninstallContext):
+ name = y.__name__.replace('Context','').lower()
+ class tmp(y):
+ cmd = name + '_' + x
+ fun = 'build'
+ variant = x
+ class tmp(ConfigurationContext):
+ cmd = 'configure_' + x
+ fun = 'configure'
+ variant = x
+ def __init__(self, **kw):
+ ConfigurationContext.__init__(self, **kw)
+ self.setenv(x)
+
+class remote(BuildContext):
+ cmd = 'remote'
+ fun = 'build'
+
+ def get_ssh_hosts(self):
+ lst = []
+ for v in Context.g_module.variants:
+ self.env.HOST = self.login_to_host(self.variant_to_login(v))
+ cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env)
+ out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH)
+ lst.append(out.strip())
+ return lst
+
+ def setup_private_ssh_key(self):
+ """
+ When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory
+ Make sure that the ssh key does not prompt for a password
+ """
+ key = os.environ.get('WAF_SSH_KEY', '')
+ if not key:
+ return
+ if not os.path.isfile(key):
+ self.fatal('Key in WAF_SSH_KEY must point to a valid file')
+ self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh')
+ self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts')
+ self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key))
+ self.ssh_config = os.path.join(self.ssh_dir, 'config')
+ for x in self.ssh_hosts, self.ssh_key, self.ssh_config:
+ if not os.path.isfile(x):
+ if not os.path.isdir(self.ssh_dir):
+ os.makedirs(self.ssh_dir)
+ Utils.writef(self.ssh_key, Utils.readf(key), 'wb')
+ os.chmod(self.ssh_key, 448)
+
+ Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts()))
+ os.chmod(self.ssh_key, 448)
+
+ Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb')
+ os.chmod(self.ssh_config, 448)
+ self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key]
+ self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh')
+
+ def skip_unbuildable_variant(self):
+ # skip variants that cannot be built on this OS
+ for k in Options.commands:
+ a, _, b = k.partition('_')
+ if b in Context.g_module.variants:
+ c, _, _ = b.partition('_')
+ if c != Utils.unversioned_sys_platform():
+ Options.commands.remove(k)
+
+ def login_to_host(self, login):
+ return re.sub(r'(\w+@)', '', login)
+
+ def variant_to_login(self, variant):
+ """linux_32_debug -> search env.LINUX_32 and then env.LINUX"""
+ x = variant[:variant.rfind('_')]
+ ret = os.environ.get('REMOTE_' + x.upper(), '')
+ if not ret:
+ x = x[:x.find('_')]
+ ret = os.environ.get('REMOTE_' + x.upper(), '')
+ if not ret:
+ ret = '%s@localhost' % getpass.getuser()
+ return ret
+
+ def execute(self):
+ global is_remote
+ if not is_remote:
+ self.skip_unbuildable_variant()
+ else:
+ BuildContext.execute(self)
+
+ def restore(self):
+ self.top_dir = os.path.abspath(Context.g_module.top)
+ self.srcnode = self.root.find_node(self.top_dir)
+ self.path = self.srcnode
+
+ self.out_dir = os.path.join(self.top_dir, Context.g_module.out)
+ self.bldnode = self.root.make_node(self.out_dir)
+ self.bldnode.mkdir()
+
+ self.env = ConfigSet.ConfigSet()
+
+ def extract_groups_of_builds(self):
+ """Return a dict mapping each variants to the commands to build"""
+ self.vgroups = {}
+ for x in reversed(Options.commands):
+ _, _, variant = x.partition('_')
+ if variant in Context.g_module.variants:
+ try:
+ dct = self.vgroups[variant]
+ except KeyError:
+ dct = self.vgroups[variant] = OrderedDict()
+ try:
+ dct[variant].append(x)
+ except KeyError:
+ dct[variant] = [x]
+ Options.commands.remove(x)
+
+ def custom_options(self, login):
+ try:
+ return Context.g_module.host_options[login]
+ except (AttributeError, KeyError):
+ return {}
+
+ def recurse(self, *k, **kw):
+ self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx')
+ self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh')
+ self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan')
+ try:
+ self.env.WAF = getattr(Context.g_module, 'waf')
+ except AttributeError:
+ try:
+ os.stat('waf')
+ except KeyError:
+ self.fatal('Put a waf file in the directory (./waf-light --tools=remote)')
+ else:
+ self.env.WAF = './waf'
+
+ self.extract_groups_of_builds()
+ self.setup_private_ssh_key()
+ for k, v in self.vgroups.items():
+ task = self(rule=rsync_and_ssh, always=True)
+ task.env.login = self.variant_to_login(k)
+
+ task.env.commands = []
+ for opt, value in v.items():
+ task.env.commands += value
+ task.env.variant = task.env.commands[0].partition('_')[2]
+ for opt, value in self.custom_options(k):
+ task.env[opt] = value
+ self.jobs = len(self.vgroups)
+
+ def make_mkdir_command(self, task):
+ return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env)
+
+ def make_send_command(self, task):
+ return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env)
+
+ def make_exec_command(self, task):
+ txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"'''
+ return Utils.subst_vars(txt, task.env)
+
+ def make_save_command(self, task):
+ return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env)
+
+def rsync_and_ssh(task):
+
+ # remove a warning
+ task.uid_ = id(task)
+
+ bld = task.generator.bld
+
+ task.env.user, _, _ = task.env.login.partition('@')
+ task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant)))
+ task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir)
+ task.env.local_dir = bld.srcnode.abspath() + '/'
+
+ task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant)
+ task.env.build_dir = bld.bldnode.abspath()
+
+ ret = task.exec_command(bld.make_mkdir_command(task))
+ if ret:
+ return ret
+ ret = task.exec_command(bld.make_send_command(task))
+ if ret:
+ return ret
+ ret = task.exec_command(bld.make_exec_command(task))
+ if ret:
+ return ret
+ ret = task.exec_command(bld.make_save_command(task))
+ if ret:
+ return ret
+
generated by cgit v1.2.1 (git 2.18.0) at 2024-11-23 01:57:45 +0000