diff options
Diffstat (limited to 'waflib/extras/remote.py')
m--------- | waflib | 0 | ||||
-rw-r--r-- | waflib/extras/remote.py | 327 |
2 files changed, 0 insertions, 327 deletions
diff --git a/waflib b/waflib new file mode 160000 +Subproject 2314e236ca6e7d94a26c3c17091da0f25f5867f diff --git a/waflib/extras/remote.py b/waflib/extras/remote.py deleted file mode 100644 index f43b600f..00000000 --- a/waflib/extras/remote.py +++ /dev/null @@ -1,327 +0,0 @@ -#!/usr/bin/env python -# encoding: utf-8 -# Remote Builds tool using rsync+ssh - -__author__ = "Jérôme Carretero <cJ-waf@zougloub.eu>" -__copyright__ = "Jérôme Carretero, 2013" - -""" -Simple Remote Builds -******************** - -This tool is an *experimental* tool (meaning, do not even try to pollute -the waf bug tracker with bugs in here, contact me directly) providing simple -remote builds. - -It uses rsync and ssh to perform the remote builds. -It is intended for performing cross-compilation on platforms where -a cross-compiler is either unavailable (eg. MacOS, QNX) a specific product -does not exist (eg. Windows builds using Visual Studio) or simply not installed. -This tool sends the sources and the waf script to the remote host, -and commands the usual waf execution. - -There are alternatives to using this tool, such as setting up shared folders, -logging on to remote machines, and building on the shared folders. -Electing one method or another depends on the size of the program. - - -Usage -===== - -1. Set your wscript file so it includes a list of variants, - e.g.:: - - from waflib import Utils - top = '.' - out = 'build' - - variants = [ - 'linux_64_debug', - 'linux_64_release', - 'linux_32_debug', - 'linux_32_release', - ] - - from waflib.extras import remote - - def options(opt): - # normal stuff from here on - opt.load('compiler_c') - - def configure(conf): - if not conf.variant: - return - # normal stuff from here on - conf.load('compiler_c') - - def build(bld): - if not bld.variant: - return - # normal stuff from here on - bld(features='c cprogram', target='app', source='main.c') - - -2. Build the waf file, so it includes this tool, and put it in the current - directory - - .. code:: bash - - ./waf-light --tools=remote - -3. Set the host names to access the hosts: - - .. code:: bash - - export REMOTE_QNX=user@kiunix - -4. Setup the ssh server and ssh keys - - The ssh key should not be protected by a password, or it will prompt for it every time. - Create the key on the client: - - .. code:: bash - - ssh-keygen -t rsa -f foo.rsa - - Then copy foo.rsa.pub to the remote machine (user@kiunix:/home/user/.ssh/authorized_keys), - and make sure the permissions are correct (chmod go-w ~ ~/.ssh ~/.ssh/authorized_keys) - - A separate key for the build processes can be set in the environment variable WAF_SSH_KEY. - The tool will then use 'ssh-keyscan' to avoid prompting for remote hosts, so - be warned to use this feature on internal networks only (MITM). - - .. code:: bash - - export WAF_SSH_KEY=~/foo.rsa - -5. Perform the build: - - .. code:: bash - - waf configure_all build_all --remote - -""" - - -import getpass, os, re, sys -from collections import OrderedDict -from waflib import Context, Options, Utils, ConfigSet - -from waflib.Build import BuildContext, CleanContext, InstallContext, UninstallContext -from waflib.Configure import ConfigurationContext - - -is_remote = False -if '--remote' in sys.argv: - is_remote = True - sys.argv.remove('--remote') - -class init(Context.Context): - """ - Generates the *_all commands - """ - cmd = 'init' - fun = 'init' - def execute(self): - for x in list(Context.g_module.variants): - self.make_variant(x) - lst = ['remote'] - for k in Options.commands: - if k.endswith('_all'): - name = k.replace('_all', '') - for x in Context.g_module.variants: - lst.append('%s_%s' % (name, x)) - else: - lst.append(k) - del Options.commands[:] - Options.commands += lst - - def make_variant(self, x): - for y in (BuildContext, CleanContext, InstallContext, UninstallContext): - name = y.__name__.replace('Context','').lower() - class tmp(y): - cmd = name + '_' + x - fun = 'build' - variant = x - class tmp(ConfigurationContext): - cmd = 'configure_' + x - fun = 'configure' - variant = x - def __init__(self, **kw): - ConfigurationContext.__init__(self, **kw) - self.setenv(x) - -class remote(BuildContext): - cmd = 'remote' - fun = 'build' - - def get_ssh_hosts(self): - lst = [] - for v in Context.g_module.variants: - self.env.HOST = self.login_to_host(self.variant_to_login(v)) - cmd = Utils.subst_vars('${SSH_KEYSCAN} -t rsa,ecdsa ${HOST}', self.env) - out, err = self.cmd_and_log(cmd, output=Context.BOTH, quiet=Context.BOTH) - lst.append(out.strip()) - return lst - - def setup_private_ssh_key(self): - """ - When WAF_SSH_KEY points to a private key, a .ssh directory will be created in the build directory - Make sure that the ssh key does not prompt for a password - """ - key = os.environ.get('WAF_SSH_KEY', '') - if not key: - return - if not os.path.isfile(key): - self.fatal('Key in WAF_SSH_KEY must point to a valid file') - self.ssh_dir = os.path.join(self.path.abspath(), 'build', '.ssh') - self.ssh_hosts = os.path.join(self.ssh_dir, 'known_hosts') - self.ssh_key = os.path.join(self.ssh_dir, os.path.basename(key)) - self.ssh_config = os.path.join(self.ssh_dir, 'config') - for x in self.ssh_hosts, self.ssh_key, self.ssh_config: - if not os.path.isfile(x): - if not os.path.isdir(self.ssh_dir): - os.makedirs(self.ssh_dir) - Utils.writef(self.ssh_key, Utils.readf(key), 'wb') - os.chmod(self.ssh_key, 448) - - Utils.writef(self.ssh_hosts, '\n'.join(self.get_ssh_hosts())) - os.chmod(self.ssh_key, 448) - - Utils.writef(self.ssh_config, 'UserKnownHostsFile %s' % self.ssh_hosts, 'wb') - os.chmod(self.ssh_config, 448) - self.env.SSH_OPTS = ['-F', self.ssh_config, '-i', self.ssh_key] - self.env.append_value('RSYNC_SEND_OPTS', '--exclude=build/.ssh') - - def skip_unbuildable_variant(self): - # skip variants that cannot be built on this OS - for k in Options.commands: - a, _, b = k.partition('_') - if b in Context.g_module.variants: - c, _, _ = b.partition('_') - if c != Utils.unversioned_sys_platform(): - Options.commands.remove(k) - - def login_to_host(self, login): - return re.sub(r'(\w+@)', '', login) - - def variant_to_login(self, variant): - """linux_32_debug -> search env.LINUX_32 and then env.LINUX""" - x = variant[:variant.rfind('_')] - ret = os.environ.get('REMOTE_' + x.upper(), '') - if not ret: - x = x[:x.find('_')] - ret = os.environ.get('REMOTE_' + x.upper(), '') - if not ret: - ret = '%s@localhost' % getpass.getuser() - return ret - - def execute(self): - global is_remote - if not is_remote: - self.skip_unbuildable_variant() - else: - BuildContext.execute(self) - - def restore(self): - self.top_dir = os.path.abspath(Context.g_module.top) - self.srcnode = self.root.find_node(self.top_dir) - self.path = self.srcnode - - self.out_dir = os.path.join(self.top_dir, Context.g_module.out) - self.bldnode = self.root.make_node(self.out_dir) - self.bldnode.mkdir() - - self.env = ConfigSet.ConfigSet() - - def extract_groups_of_builds(self): - """Return a dict mapping each variants to the commands to build""" - self.vgroups = {} - for x in reversed(Options.commands): - _, _, variant = x.partition('_') - if variant in Context.g_module.variants: - try: - dct = self.vgroups[variant] - except KeyError: - dct = self.vgroups[variant] = OrderedDict() - try: - dct[variant].append(x) - except KeyError: - dct[variant] = [x] - Options.commands.remove(x) - - def custom_options(self, login): - try: - return Context.g_module.host_options[login] - except (AttributeError, KeyError): - return {} - - def recurse(self, *k, **kw): - self.env.RSYNC = getattr(Context.g_module, 'rsync', 'rsync -a --chmod=u+rwx') - self.env.SSH = getattr(Context.g_module, 'ssh', 'ssh') - self.env.SSH_KEYSCAN = getattr(Context.g_module, 'ssh_keyscan', 'ssh-keyscan') - try: - self.env.WAF = getattr(Context.g_module, 'waf') - except AttributeError: - try: - os.stat('waf') - except KeyError: - self.fatal('Put a waf file in the directory (./waf-light --tools=remote)') - else: - self.env.WAF = './waf' - - self.extract_groups_of_builds() - self.setup_private_ssh_key() - for k, v in self.vgroups.items(): - task = self(rule=rsync_and_ssh, always=True) - task.env.login = self.variant_to_login(k) - - task.env.commands = [] - for opt, value in v.items(): - task.env.commands += value - task.env.variant = task.env.commands[0].partition('_')[2] - for opt, value in self.custom_options(k): - task.env[opt] = value - self.jobs = len(self.vgroups) - - def make_mkdir_command(self, task): - return Utils.subst_vars('${SSH} ${SSH_OPTS} ${login} "rm -fr ${remote_dir} && mkdir -p ${remote_dir}"', task.env) - - def make_send_command(self, task): - return Utils.subst_vars('${RSYNC} ${RSYNC_SEND_OPTS} -e "${SSH} ${SSH_OPTS}" ${local_dir} ${login}:${remote_dir}', task.env) - - def make_exec_command(self, task): - txt = '''${SSH} ${SSH_OPTS} ${login} "cd ${remote_dir} && ${WAF} ${commands}"''' - return Utils.subst_vars(txt, task.env) - - def make_save_command(self, task): - return Utils.subst_vars('${RSYNC} ${RSYNC_SAVE_OPTS} -e "${SSH} ${SSH_OPTS}" ${login}:${remote_dir_variant} ${build_dir}', task.env) - -def rsync_and_ssh(task): - - # remove a warning - task.uid_ = id(task) - - bld = task.generator.bld - - task.env.user, _, _ = task.env.login.partition('@') - task.env.hdir = Utils.to_hex(Utils.h_list((task.generator.path.abspath(), task.env.variant))) - task.env.remote_dir = '~%s/wafremote/%s' % (task.env.user, task.env.hdir) - task.env.local_dir = bld.srcnode.abspath() + '/' - - task.env.remote_dir_variant = '%s/%s/%s' % (task.env.remote_dir, Context.g_module.out, task.env.variant) - task.env.build_dir = bld.bldnode.abspath() - - ret = task.exec_command(bld.make_mkdir_command(task)) - if ret: - return ret - ret = task.exec_command(bld.make_send_command(task)) - if ret: - return ret - ret = task.exec_command(bld.make_exec_command(task)) - if ret: - return ret - ret = task.exec_command(bld.make_save_command(task)) - if ret: - return ret - |