queue-mgmt/qrun

   1 #!/bin/bash
   2
   3 usage_help_text="qrun [<OPTIONS>]
   4 Run next task in the queue.
   5 OPTIONS:
   6   -q, --quiet
   7   -qq, --very-quiet
   8   -d, --queue-dir <DIR>
   9   -s, --suppress-output
  10   -n, --dry-run
  11   -E, --no-task-exit-code"
  12
  13 qrun_suppress_output=''
  14 qrun_return_task_status=yes
  15
  16 . qadd-common
  17
  18
  19
  20 shopt -s nullglob
  21 next_item_id=''
  22
  23
  24 for itemfile in $(find "$queue_dir" -maxdepth 1 -name '*.comm' -printf '%f\n' | sort -n -t '.')
  25 do
  26         item_id=`basename "$itemfile" .comm`
  27         state=`qtask_state "$item_id"`
  28
  29         if [ "$state" = running ]
  30         then
  31                 # it's a running task. don't start any more.
  32                 if [ ! $quiet ]
  33                 then
  34                         pid=`cat "$queue_dir/$item_id.pid"`
  35                         pidns=`cat "$queue_dir/$item_id.pidns"`
  36                         this_pidns=`readlink /proc/self/ns/pid`
  37                         if [ "$pidns" = "$this_pidns" ]
  38                         then
  39                                 pidns_info=''
  40                         else
  41                                 pidns_info=" pidns=${pidns#*:}"
  42                         fi
  43                         task_command=`cat "$queue_dir/$item_id.comm" | { read -r -d $'\0' task_command rest; echo "$task_command"; }`
  44                         echo "qrun: task $item_id (pid=$pid$pidns_info cmd=$task_command) is running. not starting new one." >&2
  45                 fi
  46                 exit -6
  47         fi
  48
  49         if [ "$state" = queued ]
  50         then
  51                 if [ -z "$next_item_id" ]
  52                 then
  53                         next_item_id=$item_id
  54                         break
  55                 fi
  56         fi
  57 done
  58
  59 if [ -z "$next_item_id" ]
  60 then
  61         if [ ! $quiet ]
  62         then
  63                 echo "qrun: no next item found to run" >&2
  64         fi
  65         exit -5
  66 fi
  67
  68
  69 pwd=`cat "$queue_dir/$next_item_id.pwd"`
  70 umask=`cat "$queue_dir/$next_item_id.umask"`
  71 load_command_global "$queue_dir/$next_item_id.comm"
  72
  73 {
  74         if [ $dryrun ]
  75         then
  76                 exit 0
  77         fi
  78
  79         pidfile=$queue_dir/$next_item_id.pid
  80         echo $BASHPID > "$pidfile"
  81         readlink /proc/self/ns/pid > "${pidfile}ns"
  82
  83         set -e
  84
  85         # lock our pidfile and release the lock on queue_dir
  86         exec {task_lock_fd}>>"$pidfile"
  87         flock --exclusive "$task_lock_fd"
  88         exec {lock_fd}>&-
  89
  90         if [ $qrun_suppress_output ]
  91         then
  92                 exec >/dev/null
  93                 exec 2>/dev/null
  94         fi
  95
  96         # now we are turning into a task
  97         . "$queue_dir/$next_item_id.env.bash"
  98         if [ -e "$queue_dir/$next_item_id.func.bash" ]; then . "$queue_dir/$next_item_id.func.bash"; fi
  99         cd "$pwd"
 100         umask "$umask"
 101
 102         # setsid prevents SIGINT to sent to us when the user types ctrl-c on the main foreground process,
 103         # so leaving it to the main qrun process to reape our exit status.
 104         exec -- setsid "${command[@]}"
 105         exit 127
 106 } &
 107
 108 task_pid=$!
 109
 110 if [ ! $very_quiet ]
 111 then
 112         echo "qrun:${dryrun:+ DRYRUN} starting task $next_item_id, pid: $task_pid, command: ${command[*]}" >&2
 113 fi
 114
 115 # release the lock on queue_dir, letting other queue management happen while running the task
 116 exec {lock_fd}>&-
 117
 118 signal_handler()
 119 {
 120         interrupted=yes
 121 }
 122
 123 trap signal_handler INT
 124
 125 while true
 126 do
 127         interrupted=''
 128         set +e
 129         wait $task_pid
 130         task_exit_code=$?
 131         set -e
 132
 133         if [ $interrupted ]
 134         then
 135                 task_exit_code=''
 136                 kill -s INT $task_pid || true
 137         else
 138                 break
 139         fi
 140 done
 141
 142 if [ $dryrun ]
 143 then
 144         exit 0
 145 fi
 146
 147 echo -n $task_exit_code > "$queue_dir/$next_item_id.end"
 148
 149 if [ ! $very_quiet ]
 150 then
 151         echo "qrun: task $next_item_id returned $task_exit_code" >&2
 152 fi
 153
 154 if [ $qrun_return_task_status ]
 155 then
 156         exit $task_exit_code
 157 else
 158         exit 0
 159 fi