diff --git a/mussh/mussh b/mussh/mussh new file mode 100755 index 0000000..e5eee90 --- /dev/null +++ b/mussh/mussh @@ -0,0 +1,657 @@ +#!/bin/bash +# +# $Id: mussh,v 1.12 2006/12/26 21:57:22 doughnut Exp $ +MUSSH_VERSION="1.0" +# +# Description: This script is used to execute the same command(s) on +# many hosts. +# +# by doughnut +# + + + + +######################## +# INITIALIZE THIS CRAP # +######################## +DEBUG=0 +SSH_VERBOSE="" +QUIET=0 +FORCE_AGENT=0 +UNIQUE_HOSTS=1 +PROXY_SSH_ARGS="-o PasswordAuthentication=no" +SSH_ARGS="-o BatchMode=yes" +AGENT_LOADED=0 +REMOTE_SHELL='bash' +TEMP_BASE=/tmp +CONCURRENT=1 + +############################ +# GOTTA LOVE DOCUMENTATION # +############################ +USAGE="Usage: mussh [OPTIONS] <-h host.. | -H hostfile> [-c cmd] [-C scriptfile] +mussh --help for full help text" +HELPTEXT=" +Send a command or list of commands to multiple hosts. + +OPTIONS: + --help This text. + -d [n] Verbose debug. Prints each action, all hosts + and commands to be executed to STDERR. 'n' can + be from 0 to 2. + -v [n] Ssh debug levels. Can be from 0 to 3. + -m [n] Run concurrently on 'n' hosts at a time (asynchronous). + Use '0' (zero) for infinite. (default if -m) + -q No output unless necessary. + -i [identity ..] + Load an identity file. May be used + more than once. + -o Args to pass to ssh with -o option. + -a Force loading ssh-agent. + -A Do NOT load ssh-agent. + -b Print each hosts' output in a block without mingling + with other hosts' output. + -B Allow hosts' output to mingle. (default) + -u Unique. Eliminate duplicate hosts. (default) + -U Do NOT make host list unique. + -P Do NOT fall back to passwords on any host. This will + skip hosts where keys fail. + -l Use 'login' when no other is specified with hostname. + -L Force use of 'login' name on all hosts. + -s Path to shell on remote host. (Default: $REMOTE_SHELL) + -t Timeout setting for each session. + (requires openssh 3.8 or newer) + -V Print version info and exit. +PROXY ARGS: + -p [user@] + Host to use as proxy. (Must have mussh installed) + -po Args to pass to ssh on proxy with -o option. +HOST ARGS: + -h [user@] [[user@] ..] + Add a host to list of hosts. May be + used more than once. + -H [file ..] + Add contents of file(s) to list of hosts. + Files should have one host per line. Use + \"#\" for comments. +COMMAND ARGS: +If neither is specified, commands will be read from standard input. + -c Add a command or quoted list of commands and + args to list of commands to be executed on + each host. May be used more than once. + -C [file ..] + Add file contents to list of commands to be + executed on each host. May be used more + than once. + +At least one host is required. Arguments are in no particular order. + +EXAMPLES: +mussh -H ./linuxhosts -C spfiles/testscript.sh +mussh -c \"cat /etc/hosts\" -h myhost.mydomain.com + +Comments and Bug Reports: doughnut@doughnut.net +" + +########################### +# FUNCTIONS FOR SSH-AGENT # +########################### +load_keys() { + [ "$AGENT_LOADED" = "0" -a "$IDENT" = "" ] && return + [ "$DEBUG" -ge 1 ] && echo "DEBUG: Adding Keys" 1>&2 + ssh-add $* 1>&2 +} + +start_agent() { + [ "$DEBUG" -ge 1 ] && echo "DEBUG: Starting Agent" 1>&2 + if [ "$FORCE_AGENT" -ge 1 ] ; then + # Load it anyway + [ "$DEBUG" -ge 1 ] && echo "DEBUG: Forcing SSH Agent" 1>&2 + elif [ -S "$SSH_AUTH_SOCK" ] ; then + [ "$DEBUG" -ge 1 ] && echo "DEBUG: SSH Agent already loaded" 1>&2 + return + fi + eval $(ssh-agent -s) >/dev/null + AGENT_LOADED=1 +} +stop_agent() { + # get rid of the keys that we added (if any) + if [ "$IDENT" != "" ] ; then + [ "$DEBUG" -ge 1 ] && echo "DEBUG: Removing keys from agent" 1>&2 + ssh-add -d $IDENT > /dev/null 2>&1 + fi + [ "$AGENT_LOADED" = "0" ] && return + [ "$DEBUG" -ge 1 ] && echo "DEBUG: Stopping Agent" 1>&2 + eval $(ssh-agent -s -k) >/dev/null +} + +#################################### +# FUNCTIONS FOR REMOTE CONNECTIONS # +#################################### +proxy_connect() { + [ "$DEBUG" -ge 1 ] && echo "DEBUG: PROXY CONNECT $PROXY" 1>&2 + echo "$THESCRIPT" \ + | ssh -T $SSH_ARGS $PROXY \ + "mussh -h $HOSTLIST \ + -C - \ + -d$DEBUG \ + $PROXY_SSH_ARGS 2>&1 " \ + | while read SSH_LINE ; do + if [ "$QUIET" -lt 1 -a "$SSH_LINE" != "" ] ; then + echo "$SSH_LINE" | sed -e "s/^/$HOST: /" + fi + done +} + +ssh_connect() { + echo "$THESCRIPT" \ + | ssh -T $SSH_ARGS $HOST "$REMOTE_SHELL" 2>&1 \ + | while read SSH_LINE ; do + if [ "$QUIET" -lt 1 -a "$SSH_LINE" != "" ] ; then + echo "$SSH_LINE" | sed -e "s/^/$HOST: /" + fi + done +} + +############################## +# FUNCTIONS FOR FORKED PROCS # +############################## +set_hostlist() { +# Create a hostlist file. + [ "$DEBUG" -ge 2 ] && echo "DEBUG: BUILDING HOST LIST FILE $TEMP_DIR/hostlist" 1>&2 + rm -f $TEMP_DIR/hostlist || exit 1 + for HOST in $HOSTLIST ; do + echo $HOST >> "$TEMP_DIR/hostlist" || exit 1 + done +} + +get_next_host() { +# lock file + while [ 1 ] ; do + echo $CHILDNUM >> "$TEMP_DIR/hostlist.lock" + TOP_PID=$(head -1 "$TEMP_DIR/hostlist.lock" 2>/dev/null) + if [ "$TOP_PID" = $CHILDNUM ] ; then + break + fi + [ "$DEBUG" -ge 2 ] && echo "DEBUG[#$CHILDNUM]: hostlist file already locked. Sleep..." 1>&2 + #usleep 1000 + sleep 1 + done + [ "$DEBUG" -ge 2 ] && echo "DEBUG[#$CHILDNUM]: Locked hostfile." 1>&2 + +# get next host + NEXT_HOST=$(head -1 $TEMP_DIR/hostlist) + HOSTFILE_LEN=$(wc -l $TEMP_DIR/hostlist | awk '{print $1}') + if [ -z "$HOSTFILE_LEN" -o "$HOSTFILE_LEN" = 0 ] ; then + rm -f "$TEMP_DIR/hostlist.lock" + return + fi + [ "$DEBUG" -ge 2 ] && echo "DEBUG[#$CHILDNUM]: Next host: $NEXT_HOST" 1>&2 + +# re-write file removing new host + rm -f "$TEMP_DIR/hostlist.new" + echo tail -$(( $HOSTFILE_LEN - 1 )) $TEMP_DIR/hostlist > $TEMP_DIR/hostlist.new || exit 1 + tail -$(( $HOSTFILE_LEN - 1 )) $TEMP_DIR/hostlist > $TEMP_DIR/hostlist.new || exit 1 + rm -f "$TEMP_DIR/hostlist" + mv "$TEMP_DIR/hostlist.new" "$TEMP_DIR/hostlist" + +# unlock file + [ "$DEBUG" -ge 2 ] && echo "DEBUG[#$CHILDNUM]: Removing hostfile lock." 1>&2 + rm -f "$TEMP_DIR/hostlist.lock" + +# return hostname + echo $NEXT_HOST +} + +run_child() { + trap "exit 0" SIGHUP + CHILDNUM=$1 + [ "$DEBUG" -ge 2 ] && echo "DEBUG: FORKING CHILD #$CHILDNUM of $CONCURRENT (pid $!/$$)" 1>&2 + while [ 1 ] ; do + +# issue: Cannot call get_next_host inside $() or `` because our trap won't be able to kill that. +# solution: avoid subshell here by directing to a file. + rm -f $TEMP_DIR/$CHILDNUM.next_host + get_next_host >$TEMP_DIR/$CHILDNUM.next_host + HOST=$(<$TEMP_DIR/$CHILDNUM.next_host) + if [ -z "$HOST" ] ; then + rm -f "$TEMP_DIR/$CHILDNUM.pid" + break + fi + [ "$DEBUG" -ge 1 ] && echo "DEBUG[#$CHILDNUM]: CONNECT $HOST" 1>&2 + + rm -f "$TEMP_DIR/$CHILDNUM.active" + echo "$HOST" > "$TEMP_DIR/$CHILDNUM.active" + if [ -n "$BLOCKING" ] ; then + ssh_connect > $TEMP_DIR/$HOST.out + cat $TEMP_DIR/$HOST.out + else + ssh_connect + fi + done + [ "$DEBUG" -ge 2 ] && echo "DEBUG: CHILD #$CHILDNUM done" 1>&2 + rm -f "$TEMP_DIR/$CHILDNUM.pid" "$TEMP_DIR/$CHILDNUM.active" +} + + +########################### +# FUNCTIONS FOR TEMP DIRS # +########################### +create_temp() { + MKTEMP=$(which mktemp 2>/dev/null) + if [ -x "$MKTEMP" ] ; then + [ "$DEBUG" -ge 2 ] && echo "DEBUG: using mktemp ($MKTEMP)." 1>&2 + TEMP_DIR=$(mktemp -d $TEMP_BASE/$(basename $0).XXXXXX) || exit 1 + else + [ "$DEBUG" -ge 2 ] && echo "DEBUG: can't find mktemp... using alternate." 1>&2 + TEMP_DIR="$TEMP_BASE/$(basename $0).$(date +%s)" + [ "$DEBUG" -ge 2 ] && echo "DEBUG: Creating temp dir ($TEMP_DIR)." 1>&2 + if [ -e "$TEMP_DIR" ] ; then + echo "$0: Temp dir \"$TEMP_DIR\" already exists!" 1>&2 + exit 1 + fi + mkdir -m 700 $TEMP_DIR + fi +} + +destroy_temp() { + if [ -d "$TEMP_DIR" ] ; then + [ "$DEBUG" -ge 2 ] && echo "DEBUG: Removing temp dir ($TEMP_DIR)." 1>&2 + rm -rf "$TEMP_DIR" 2>/dev/null + fi +} + + + +######################################## +# REMEMBER TO CLEAN UP BEFORE WE PANIC # +######################################## +shutdown() { + [ "$DEBUG" -ge 1 ] && echo "DEBUG: shutting down children." 1>&2 + CPIDS=$(cat $TEMP_DIR/*.pid 2>/dev/null) + for CPID in $CPIDS ; do + [ "$DEBUG" -ge 2 ] && echo "DEBUG: Killing pid: $CPID" 1>&2 + kill -HUP $CPID + done + [ "$DEBUG" -ge 2 ] && echo "DEBUG: shutting down ssh-agent" 1>&2 + stop_agent + [ "$DEBUG" -ge 2 ] && echo "DEBUG: removing temp dir" 1>&2 + destroy_temp + [ "$DEBUG" -ge 2 ] && echo "DEBUG: done shutting down." 1>&2 + exit 1 +} + +spew_hostlist() { + echo "HOSTS RUNNING:" 1>&2 + cat $TEMP_DIR/*.active 2>/dev/null | sed 's/^/ /' 1>&2 + echo "HOSTS REMAINING:" 1>&2 + cat $TEMP_DIR/hostlist 2>/dev/null | sed 's/^/ /' 1>&2 + return +} + +trap shutdown SIGINT +trap shutdown SIGTERM +trap spew_hostlist SIGQUIT +trap "exit 0" SIGHUP + +############################# +# PARSE THE COMMAND OPTIONS # +############################# +while [ "$1" != "" ]; do + case "$1" in + ########### + # OPTIONS # + ########### + -A) + NO_AGENT=1 + shift + ;; + -a) + FORCE_AGENT=1 + shift + ;; + -b) + BLOCKING=1 + shift + ;; + -B) + unset BLOCKING + shift + ;; + -q) + QUIET=1 + DEBUG=0 + SSH_VERBOSE="-q" + shift + ;; + -o) + SSH_ARGS="$SSH_ARGS -o $2" + shift 2 + ;; + -u) + UNIQUE_HOSTS=1 + shift + ;; + -U) + UNIQUE_HOSTS=0 + shift + ;; + -l) + DEFAULT_LOGIN=$2 + shift 2 + ;; + -L) + FORCE_LOGIN=$2 + shift 2 + ;; + -s) + REMOTE_SHELL=$2 + shift 2 + ;; + -t*) + SSH_TIMEOUT="${1#-t}" + if [ -z "$SSH_TIMEOUT" -a -n "$2" -a "${2#-?*}" = "$2" ] ; then + SSH_TIMEOUT=$2 + shift + fi + if [ "${SSH_TIMEOUT//[^0-9]/}" != "$SSH_TIMEOUT" -o -z "$SSH_TIMEOUT" ] ; then + echo "mussh: Argument should be numeric: -t $SSH_TIMEOUT" 1>&2 + exit 1 + fi + shift + SSH_ARGS="$SSH_ARGS -o ConnectTimeout=$SSH_TIMEOUT" + ;; + -m*) # -m0 .. -m999 + CONCURRENT="${1#-m}" + if [ -z "$CONCURRENT" -a -n "$2" -a "${2#-?*}" = "$2" ] ; then + CONCURRENT=$2 + shift + elif [ -z "$CONCURRENT" ] ; then + CONCURRENT=0 + fi + if [ "${CONCURRENT//[^0-9]/}" != "$CONCURRENT" ] ; then + echo "mussh: Argument should be numeric: -m $CONCURRENT" 1>&2 + exit 1 + fi + shift + ;; + -d*) + DEBUG="${1#-d}" + if [ -z "$DEBUG" -a -n "$2" -a "${2#-?*}" = "$2" ] ; then + DEBUG=$2 + shift + elif [ -z "$DEBUG" ] ; then + DEBUG=1 + fi + if [ "${DEBUG//[^0-9]/}" != "$DEBUG" ] ; then + echo "mussh: Argument should be numeric: -d $DEBUG" 1>&2 + exit 1 + fi + shift + ;; + -v*) + TMP_ARG="${1#-v}" + if [ -z "$TMP_ARG" -a -n "$2" -a "${2#-?*}" = "$2" ] ; then + TMP_ARG=$2 + shift + elif [ -z "$TMP_ARG" ] ; then + TMP_ARG=1 + fi + if [ "${TMP_ARG//[^0-9]/}" != "$TMP_ARG" ] ; then + echo "mussh: Argument should be numeric: -v $TMP_ARG" 1>&2 + exit 1 + elif [ "${TMP_ARG//[^0-3]/}" != "$TMP_ARG" ] ; then + echo "mussh: Argument should be between 0 and 3: -v $TMP_ARG" 1>&2 + exit 1 + fi + SSH_VERBOSE="-v" + [ "$TMP_ARG" -ge 2 ] && SSH_VERBOSE="$SSH_VERBOSE -v" + [ "$TMP_ARG" -ge 3 ] && SSH_VERBOSE="$SSH_VERBOSE -v" + [ "$TMP_ARG" -eq 0 ] && SSH_VERBOSE="-q" + shift + ;; + -V) + echo "Version: $MUSSH_VERSION" + exit + ;; + -P) + SSH_ARGS="$SSH_ARGS -o PasswordAuthentication=no" + shift + ;; + --help) + # print help text + echo "$USAGE" + echo "$HELPTEXT" + exit + ;; + -i) + # Load the identity file in ssh-agent + while [ "$2" != "" -a "${2#-}" = "$2" ] ; do + IDENT="$IDENT $2" + shift + done + shift + ;; + ############## + # PROXY ARGS # + ############## + -p) + PROXY=$2 + SSH_ARGS="$SSH_ARGS -o ForwardAgent=yes" + shift 2 + ;; + -po) + PROXY_SSH_ARGS="$PROXY_SSH_ARGS -o $2" + shift 2 + ;; + ############# + # HOST ARGS # + ############# + -h) + while [ "$2" != "" -a "${2#-?*}" = "$2" ] ; do + HOSTLIST="$2 +$HOSTLIST" + shift + done + shift + ;; + -H) + while [ "$2" != "" -a "${2#-?*}" = "$2" ] ; do + HOSTFILE="$2" + if [ ! -e "$HOSTFILE" -a "$HOSTFILE" != "-" ] ; then + echo "mussh: Host file '$HOSTFILE' does not exist!" 1>&2 + exit 1 + fi + HOSTLIST="$(cat $HOSTFILE | sed -e 's/#.*//' | egrep -v "^ *$" ) +$HOSTLIST" + shift + done + shift + ;; + -n) + # I've left this undocumented for lack of testing. Volunteers? + NETGROUP=$2 + NETGROUP_LIST="$(getent netgroup $NETGROUP | xargs -n 1 echo | sed -n '/^(.*,$/s/[,(]//gp')" + if [ -z "$NETGROUP_LIST" ] ; then + echo "mussh: Failed to get netgroup: $NETGROUP" 2>&1 + exit 1 + fi + HOSTLIST="$NETGROUP_LIST +$HOSTLIST" + shift 2 + ;; + ################ + # COMMAND ARGS # + ################ + -c) + THESCRIPT="$THESCRIPT +$2" + #set "" ; shift + shift 2 + ;; + -C) + while [ "$2" != "" -a "${2#-?*}" = "$2" ] ; do + SCRIPTFILE="$2" + if [ ! -e "$SCRIPTFILE" -a "$SCRIPTFILE" != "-" ] ; then + echo "mussh: Script File '$SCRIPTFILE' does not exist!" 1>&2 + exit 1 + fi + THESCRIPT="$THESCRIPT +$(cat $SCRIPTFILE )" + shift + done + shift + ;; + *) + echo "mussh: invalid command - \"$1\"" 1>&2 + echo "$USAGE" 1>&2 + exit 1 + ;; + esac +done + +##################### +# CLEAN UP HOSTLIST # +##################### +HOSTLIST=$(echo "$HOSTLIST" | sed -e 's/#.*//' | egrep -v "^ *$" ) + +if [ "$FORCE_LOGIN" != "" ] ; then + [ "$DEBUG" -ge 1 ] && echo "DEBUG: FORCE_LOGIN: $FORCE_LOGIN" 1>&2 + HOSTLIST=$(echo "$HOSTLIST" | sed -e "s/^\(.*@\)\{0,1\}\([^@]*\)\$/$FORCE_LOGIN@\2/") +elif [ "$DEFAULT_LOGIN" != "" ] ; then + [ "$DEBUG" -ge 1 ] && echo "DEBUG: DEFAULT_LOGIN: $DEFAULT_LOGIN" 1>&2 + HOSTLIST=$(echo "$HOSTLIST" | sed -e "s/^\([^@]*\)\$/$DEFAULT_LOGIN@\1/") +fi +[ $UNIQUE_HOSTS -ge 1 ] && HOSTLIST=$(echo "$HOSTLIST" | sort -uf ) + + +################ +# CHECK SYNTAX # +################ +if [ "$HOSTLIST" = "" ] ; then + echo "mussh: ERROR: You must supply at least one host!" 1>&2 + echo "$USAGE" 1>&2 + exit 1 +fi + +################################### +# DEFAULT TO STDIN IF NO COMMANDS # +################################### +if [ "$THESCRIPT" = "" ] ; then + echo "Enter your script here. End with single \".\" or EOF." 1>&2 + while read THISLINE + do + if [ "$THISLINE" = "." ] ; then + break + fi + THESCRIPT="$THESCRIPT +$THISLINE" + done +fi + +################################################### +# INFINITE CONCURRENCY IS REALLY JUST ALL AT ONCE # +################################################### +COUNT_HOSTS=$(echo "$HOSTLIST" | wc -w) +if [ $CONCURRENT -eq 0 ] || [ $CONCURRENT -gt $COUNT_HOSTS ] ; then + CONCURRENT=$COUNT_HOSTS + [ "$DEBUG" -ge 1 ] && echo "DEBUG: setting concurrency (-m) to $CONCURRENT (all hosts)" 1>&2 +fi +[ "$DEBUG" -ge 1 ] && echo "DEBUG: Concurrency: $CONCURRENT" 1>&2 + +################################# +# ADD SSH VERBOSITY TO SSH ARGS # +################################# +if [ "$SSH_VERBOSE" ] ; then + SSH_ARGS="$SSH_VERBOSE $SSH_ARGS" +fi + +############################ +# PRINT VERBOSE DEBUG INFO # +############################ +if [ "$DEBUG" -ge 1 ] ; then + echo "DEBUG: DEBUG LEVEL: $DEBUG" 1>&2 + echo "DEBUG: SSH DEBUG LEVEL: $SSH_VERBOSE" 1>&2 +fi +if [ "$DEBUG" -ge 2 ] ; then + echo "DEBUG: HOSTLIST: " $HOSTLIST 1>&2 + echo "DEBUG: THE SCRIPT: $THESCRIPT" 1>&2 + echo "DEBUG: SSH ARGS: $SSH_ARGS" 1>&2 +fi + +############################ +# LOAD SSH-AGENT WITH KEYS # +############################ +if [ "$NO_AGENT" = "1" ] ; then + [ "$DEBUG" -ge 1 ] && echo "DEBUG: Not using ssh-agent" 1>&2 +elif [ "$IDENT" != "" ] ; then + start_agent + load_keys "$IDENT" +elif [ ! -f "$HOME/.ssh/identity" -a ! -f "$HOME/.ssh/id_dsa" ] ; then + [ "$DEBUG" -ge 1 ] && echo "DEBUG: No identity file found. Skipping agent." +else + start_agent + load_keys "$IDENT" +fi +#echo + +################### +# CREATE TEMP DIR # +################### +create_temp + +######################## +# EXECUTE THE COMMANDS # +######################## +if [ -z "$CONCURRENT" ] ; then +CONCURRENT=1 +fi + +if [ "$PROXY" != "" ] ; then + proxy_connect + +# Don't background process when we're only doing one at a time +#elif [ -z "$CONCURRENT" -o "$CONCURRENT" = 1 ] ; then +# set $HOSTLIST +# while [ "$1" != "" ] ; do +# HOST="$1" +# [ "$DEBUG" -ge 1 ] && echo "DEBUG: CONNECT $HOST" 1>&2 +# shift +# ssh_connect +# done +else +# Fork $CONCURRENT children + set_hostlist + CHILDNUM=1 + while [ $CHILDNUM -le $CONCURRENT ] ; do + run_child $CHILDNUM & + [ "$DEBUG" -ge 2 ] && echo "DEBUG: FORKED CHILD #$CHILDNUM with pid $!" 1>&2 + rm -f "$TEMP_DIR/$CHILDNUM.pid" + echo $! > "$TEMP_DIR/$CHILDNUM.pid" + (( CHILDNUM++ )) + done + wait +# since trap causes waits to stop waiting with a return value >128 +# We need to check that we really meant to stop waiting. + RETVAL=$? + while [ "$RETVAL" -gt 128 ] ; do + [ "$DEBUG" -ge 2 ] && echo "DEBUG: wait returned with a value of $RETVAL" 1>&2 + DO_WAIT=0 + for CPID in $(cat $TEMP_DIR/*.pid 2>/dev/null) ; do + if [ -d "/proc/$CPID" ] ; then + DO_WAIT=1 + [ "$DEBUG" -ge 2 ] && echo "DEBUG: $CPID is still running." 1>&2 + fi + done + [ "$DO_WAIT" = 0 ] && break + wait + done +fi + +############ +# CLEAN UP # +############ +destroy_temp +stop_agent +