Planetlab utilities

git-svn-id: https://scm.gforge.inria.fr/svn/pacemaker@50 30fcff6e-8de6-41c7-acce-77ff6d1dd07b

6 files changed, 581 insertions, 0 deletions

diff --git a/planetlab/pssh/bin/pnuke b/planetlab/pssh/bin/pnuke
new file mode 100755
index 0000000..2b4feb5
--- /dev/null
+++ b/planetlab/pssh/bin/pnuke
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Nukes all processes that match pattern running as user on the set of nodes
+in hosts.txt.
+"""
+
+import os
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+_DEFAULT_TIMEOUT = 60
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] -h hosts.txt pattern"
+    parser.epilog = "Example: pnuke -h hosts.txt -l irb2 java"
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults(timeout=_DEFAULT_TIMEOUT)
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Pattern not specified.')
+
+    if len(args) > 1:
+        parser.error('Extra arguments given after the pattern.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pnuke(hosts, pattern, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        cmd = ['ssh', host, '-o', 'NumberOfPasswordPrompts=1']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if user:
+            cmd += ['-l', user]
+        if port:
+            cmd += ['-p', port]
+        if opts.extra:
+            cmd.extend(opts.extra)
+        cmd.append('pkill -9 %s' % pattern)
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status != 0:
+            sys.exit(4)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    pattern = args[0]
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pnuke(hosts, pattern, opts)
diff --git a/planetlab/pssh/bin/prsync b/planetlab/pssh/bin/prsync
new file mode 100755
index 0000000..b66443b
--- /dev/null
+++ b/planetlab/pssh/bin/prsync
@@ -0,0 +1,125 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel rsync to the set of nodes in hosts.txt.
+
+For each node, we essentially do a rsync -rv -e ssh local user@host:remote.
+Note that remote must be an absolute path.
+"""
+
+import os
+import re
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] -h hosts.txt local remote"
+    parser.epilog = ("Example: prsync -r -h hosts.txt -l irb2 foo " +
+          "/home/irb2/foo")
+
+    parser.add_option('-r', '--recursive', dest='recursive',
+            action='store_true', help='recusively copy directories (OPTIONAL)')
+    parser.add_option('-a', '--archive', dest='archive', action='store_true',
+            help='use rsync -a (archive mode) (OPTIONAL)')
+    parser.add_option('-z', '--compress', dest='compress', action='store_true',
+            help='use rsync compression (OPTIONAL)')
+    parser.add_option('-S', '--ssh-args', metavar="ARGS", dest='ssh_args',
+            action='store', help='extra arguments for ssh')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults()
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Paths not specified.')
+
+    if len(args) < 2:
+        parser.error('Remote path not specified.')
+
+    if len(args) > 2:
+        parser.error('Extra arguments given after the remote path.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_prsync(hosts, local, remote, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        ssh = ['ssh']
+        if opts.options:
+            ssh += ['-o', opts.options]
+        if port:
+            ssh += ['-p', port]
+        if opts.ssh_args:
+            ssh += [opts.ssh_args]
+
+        cmd = ['rsync', '-e', ' '.join(ssh)]
+        if opts.verbose:
+            cmd.append('-v')
+        if opts.recursive:
+            cmd.append('-r')
+        if opts.archive:
+            cmd.append('-a')
+        if opts.compress:
+            cmd.append('-z')
+        if opts.extra:
+            cmd.extend(opts.extra)
+        cmd.append(local)
+        if user:
+            cmd.append('%s@%s:%s' % (user, host, remote))
+        else:
+            cmd.append('%s:%s' % (host, remote))
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status != 0:
+            sys.exit(4)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    local = args[0]
+    remote = args[1]
+    if not re.match("^/", remote):
+        print("Remote path %s must be an absolute path" % remote)
+        sys.exit(3)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_prsync(hosts, local, remote, opts)
diff --git a/planetlab/pssh/bin/pscp b/planetlab/pssh/bin/pscp
new file mode 100755
index 0000000..3caf455
--- /dev/null
+++ b/planetlab/pssh/bin/pscp
@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel scp to the set of nodes in hosts.txt.
+
+For each node, we essentially do a scp [-r] local user@host:remote.  This
+program also uses the -q (quiet) and -C (compression) options.  Note that
+remote must be an absolute path.
+"""
+
+import os
+import re
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] -h hosts.txt local remote"
+    parser.epilog = ("Example: pscp -h hosts.txt -l irb2 foo.txt " +
+            "/home/irb2/foo.txt")
+
+    parser.add_option('-r', '--recursive', dest='recursive',
+            action='store_true', help='recusively copy directories (OPTIONAL)')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults()
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Paths not specified.')
+
+    if len(args) < 2:
+        parser.error('Remote path not specified.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pscp(hosts, localargs, remote, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        cmd = ['scp', '-qC']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if port:
+            cmd += ['-P', port]
+        if opts.recursive:
+            cmd.append('-r')
+        if opts.extra:
+            cmd.extend(opts.extra)
+        cmd.extend(localargs)
+        if user:
+            cmd.append('%s@%s:%s' % (user, host, remote))
+        else:
+            cmd.append('%s:%s' % (host, remote))
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status != 0:
+            sys.exit(4)
+
+if __name__ == "__main__":
+    opts, args = parse_args()
+    localargs = args[0:-1]
+    remote = args[-1]
+    if not re.match("^/", remote):
+        print("Remote path %s must be an absolute path" % remote)
+        sys.exit(3)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pscp(hosts, localargs, remote, opts)
diff --git a/planetlab/pssh/bin/pslurp b/planetlab/pssh/bin/pslurp
new file mode 100755
index 0000000..5797e7c
--- /dev/null
+++ b/planetlab/pssh/bin/pslurp
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel scp from the set of nodes in hosts.txt.
+
+For each node, we essentially do a scp [-r] user@host:remote
+outdir/<node>/local.  This program also uses the -q (quiet) and -C
+(compression) options.  Note that remote must be an absolute path.
+"""
+
+import os
+import re
+import sys
+    
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+    
+from psshlib import psshutil
+from psshlib.task import Task
+from psshlib.manager import Manager, FatalError
+from psshlib.cli import common_parser, common_defaults
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] -h hosts.txt remote local"
+    parser.epilog = ("Example: pslurp -h hosts.txt -L /tmp/outdir -l irb2 " +
+            "         /home/irb2/foo.txt foo.txt")
+
+    parser.add_option('-r', '--recursive', dest='recursive',
+            action='store_true', help='recusively copy directories (OPTIONAL)')
+    parser.add_option('-L', '--localdir', dest='localdir',
+            help='output directory for remote file copies')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults()
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) < 1:
+        parser.error('Paths not specified.')
+
+    if len(args) < 2:
+        parser.error('Local path not specified.')
+
+    if len(args) > 2:
+        parser.error('Extra arguments given after the local path.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pslurp(hosts, remote, local, opts):
+    if opts.localdir and not os.path.exists(opts.localdir):
+        os.makedirs(opts.localdir)
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    for host, port, user in hosts:
+        if opts.localdir:
+            dirname = "%s/%s" % (opts.localdir, host)
+        else:
+            dirname = host
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        if opts.localdir:
+            localpath = "%s/%s/%s" % (opts.localdir, host, local)
+        else:
+            localpath = "%s/%s" % (host, local)
+        cmd = ['scp', '-qC']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if port:
+            cmd += ['-P', port]
+        if opts.recursive:
+            cmd.append('-r')
+        if opts.extra:
+            cmd.extend(opts.extra)
+        if user:
+            cmd.append('%s@%s:%s' % (user, host, remote))
+        else:
+            cmd.append('%s:%s' % (host, remote))
+        cmd.append(localpath)
+        t = Task(host, port, user, cmd, opts)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    for status in statuses:
+        if status == 255:
+            sys.exit(4)
+    for status in statuses:
+        if status != 0:
+            sys.exit(5)
+   
+if __name__ == "__main__":
+    opts, args = parse_args()
+    remote = args[0]
+    local = args[1]
+    if not re.match("^/", remote):
+        print("Remote path %s must be an absolute path" % remote)
+        sys.exit(3)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pslurp(hosts, remote, local, opts)
diff --git a/planetlab/pssh/bin/pssh b/planetlab/pssh/bin/pssh
new file mode 100755
index 0000000..f9af471
--- /dev/null
+++ b/planetlab/pssh/bin/pssh
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# -*- Mode: python -*-
+
+# Copyright (c) 2009, Andrew McNabb
+# Copyright (c) 2003-2008, Brent N. Chun
+
+"""Parallel ssh to the set of nodes in hosts.txt.
+
+For each node, this essentially does an "ssh host -l user prog [arg0] [arg1]
+...". The -o option can be used to store stdout from each remote node in a
+directory.  Each output file in that directory will be named by the
+corresponding remote node's hostname or IP address.
+"""
+
+import fcntl
+import os
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib import psshutil
+from psshlib.manager import Manager, FatalError
+from psshlib.task import Task
+from psshlib.cli import common_parser, common_defaults
+
+_DEFAULT_TIMEOUT = 60
+
+def option_parser():
+    parser = common_parser()
+    parser.usage = "%prog [OPTIONS] command [...]"
+    parser.epilog = "Example: pssh -h hosts.txt -l irb2 -o /tmp/foo uptime"
+
+    parser.add_option('-i', '--inline', dest='inline', action='store_true',
+            help='inline aggregated output for each server')
+    parser.add_option('-I', '--send-input', dest='send_input',
+            action='store_true',
+            help='read from standard input and send as input to ssh')
+    parser.add_option('-P', '--print', dest='print_out', action='store_true',
+            help='print output as we get it')
+
+    return parser
+
+def parse_args():
+    parser = option_parser()
+    defaults = common_defaults(timeout=_DEFAULT_TIMEOUT)
+    parser.set_defaults(**defaults)
+    opts, args = parser.parse_args()
+
+    if len(args) == 0 and not opts.send_input:
+        parser.error('Command not specified.')
+
+    if not opts.host_files and not opts.host_strings:
+        parser.error('Hosts not specified.')
+
+    return opts, args
+
+def do_pssh(hosts, cmdline, opts):
+    if opts.outdir and not os.path.exists(opts.outdir):
+        os.makedirs(opts.outdir)
+    if opts.errdir and not os.path.exists(opts.errdir):
+        os.makedirs(opts.errdir)
+    if opts.send_input:
+        stdin = sys.stdin.read()
+    else:
+        stdin = None
+    manager = Manager(opts)
+    for host, port, user in hosts:
+        cmd = ['ssh', host, '-o', 'NumberOfPasswordPrompts=1',
+                '-o', 'SendEnv=PSSH_NODENUM']
+        if opts.options:
+            for opt in opts.options:
+                cmd += ['-o', opt]
+        if user:
+            cmd += ['-l', user]
+        if port:
+            cmd += ['-p', port]
+        if opts.extra:
+            cmd.extend(opts.extra)
+        if cmdline:
+            cmd.append(cmdline)
+        t = Task(host, port, user, cmd, opts, stdin)
+        manager.add_task(t)
+    try:
+        statuses = manager.run()
+    except FatalError:
+        sys.exit(1)
+
+    if min(statuses) < 0:
+        # At least one process was killed.
+        sys.exit(3)
+    # The any builtin was introduced in Python 2.5 (so we can't use it yet):
+    #elif any(x==255 for x in statuses):
+    for status in statuses:
+        if status == 255:
+            sys.exit(4)
+    for status in statuses:
+        if status != 0:
+            sys.exit(5)
+   
+if __name__ == "__main__":
+    opts, args = parse_args()
+    cmdline = " ".join(args)
+    try:
+        hosts = psshutil.read_host_files(opts.host_files,
+            default_user=opts.user)
+    except IOError:
+        _, e, _ = sys.exc_info()
+        sys.stderr.write('Could not open hosts file: %s\n' % e.strerror)
+        sys.exit(1)
+    if opts.host_strings:
+        for s in opts.host_strings:
+            hosts.extend(psshutil.parse_host_string(s, default_user=opts.user))
+    do_pssh(hosts, cmdline, opts)
diff --git a/planetlab/pssh/bin/pssh-askpass b/planetlab/pssh/bin/pssh-askpass
new file mode 100755
index 0000000..beb9e8b
--- /dev/null
+++ b/planetlab/pssh/bin/pssh-askpass
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+parent, bindir = os.path.split(os.path.dirname(os.path.abspath(sys.argv[0])))
+if os.path.exists(os.path.join(parent, 'psshlib')):
+    sys.path.insert(0, parent)
+
+from psshlib.askpass_client import askpass_main
+askpass_main()