#!/bin/bash
#
# recycle_holds.sh - script to examine condor jobs that are on hold
#                    and see if they ever completed, either with error
#                    or success, and if not send them back to idle.
#
# author: richard.t.jones at uconn.edu
# version: september 26, 2024

logd="$(dirname $0)/../log7a.d"

for jobid in $(condor_q -hold | awk '{print $1}'); do
    if echo $jobid | grep -q '^[0-9]'; then
        echo -n "$jobid: "
        if grep -q 'Error .* cannot fetch' $logd/stdout.$jobid 2>/dev/null; then
            echo condor_release $jobid && condor_release $jobid
        elif grep -q 'running' $logd/stdout.$jobid 2>/dev/null; then
            echo -n "       "
            if grep -a 'Error' $logd/stdout.$jobid 2>/dev/null; then
                continue
            fi
        else
            echo condor_release $jobid && condor_release $jobid
        fi
    fi
done