osaris/lib/check-queue.sh at development · cryotools/osaris · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
#!/usr/bin/env bash

convertsecs() {
    ((h=${1}/3600))
    ((m=(${1}%3600)/60))
    ((s=${1}%60))
    printf "%02d:%02d:%02d\n" $h $m $s
}

start_monitor=`date +%s`

if [ $# -eq 1 ]; then
    name=$1
    duration=10
    display_gfx=0

elif [ $# -eq 2 ]; then
    name=$1
    duration=$2
    display_gfx=0

elif [ $# -eq 3 ]; then
    name=$1
    duration=$2
    display_gfx=$3

else
    echo "Warning: No parameters provided to identify job in SLURM queue."
    name=$( whoami | cut -c1-8 )
    echo "Trying by username $name"
    duration=60
fi

# jq=$( squeue -o "%.35j %.7i %.8u %.2t %.10M %.6D %R " | awk -F_ 'NR > 1 {printf"%s\n", $1}' | awk 'NR==1{ print $1}' )

echo
echo "Waiting for SLURM jobs to finish. This may take a while."
echo "Use the 'squeue' command to see the Slurm queue."
echo "Use 'tail -f <logfile>' to monitor processing logs in the Log directory."
echo

if [ $display_gfx -eq 1 ]; then
    case "$(( ( RANDOM % 4 )  + 1 ))" in
	"1")
	    echo
	    echo
	    echo "   //\\"
	    echo "   V  \\"
	    echo "    \\  \\_"
	    echo "     \\,'.\`-."
	    echo "      |\\ \\\`. \`. "
	    echo "      ( \  \`. \`-.                        _,.-:\\"
	    echo "       \ \   \`.  \`-._             __..--' ,-';/"
	    echo "        \\ \`.   \`-.   \`-..___..---'   _.--' ,'/"
	    echo "         \`. \`.    \`-._        __..--'    ,' /"
	    echo "           \`. \`-_     \`\`--..''       _.-' ,'"
	    echo "             \`-_ \`-.___        __,--'   ,'"
	    echo "                \`-.__  \`----\"\"\"    __.-'"
	    echo "                     \`--..____..--'"
	    echo
	    ;;
	"2")
	    echo
	    echo
	    echo "                        //"
	    echo "                       //"
	    echo "                      //"
	    echo "                     //"
	    echo "              _______||"
	    echo "         ,-'''       ||\`-."
	    echo "        (            ||   )"
	    echo "        |\`-..._______,..-'|"
	    echo "        |            ||   |"
	    echo "        |     _______||   |"
	    echo "        |,-'''_ _  ~ ||\`-.|"
	    echo "        |  ~ / \`-.\ ,-\' ~|"
	    echo "        |\`-...___/___,..-'|"
	    echo "        |    \`-./-'_ \/_| |"
	    echo "        | -'  ~~     || -.|"
	    echo "        (   ~      ~   ~~ )"
	    echo "         \`-..._______,..-'"
	    echo
	    ;;

	"3")
	    echo
	    echo
	    echo
	    echo "      ___  ___  ___  ___  ___.---------------."
	    echo "    .'\__\'\__\'\__\'\__\'\__,\`   .  ____ ___ \ "
	    echo "    |\/ __\/ __\/ __\/ __\/ _:\   |\`.  \  \___ \ "
	    echo "     \\\\'\__\'\__\'\__\'\__\'\_\`.__|\"\"\`. \  \___ \ "
	    echo "      \\\\/ __\/ __\/ __\/ __\/ _:                 \ "
	    echo "       \\\\'\__\'\__\'\__\ \__\'\_;-----------------\`"
	    echo "        \\\\/   \/   \/   \/   \/ :                 |"
	    echo "         \|______________________;________________| "
	    echo
	    ;;
	"4")
	    echo
	    echo
	    echo "                        ("
	    echo "                          )     ( "
	    echo "                   ___...(-------)-....___ "
	    echo "               .-\"\"       )    (          \"\"-. "
	    echo "         .-'``'|-._             )         _.-| "
	    echo "        /  .--.|   \`\"\"---...........---\"\"\`   | "
	    echo "       /  /    |                             | "
	    echo "       |  |    |                             | "
	    echo "        \  \   |                             | "
	    echo "         \`\ \`\ |                             | "
	    echo "           \`\ \`|                             | "
	    echo "           _/ /\\                             / "
	    echo "          (__/  \\                           / "
	    echo "       _..---\"\"\` \\                         /\`\"\"---.._ "
	    echo "    .-'           \\                       /          '-. "
	    echo "   :               \`-.__             __.-'              : "
	    echo "   :                  ) \"\"---...---\"\" (                 : "
	    echo "    '._               \`\"--...___...--\"\`              _.' "
	    echo "      \\\"\"--..__                              __..--\"\"/ "
	    echo "       '._     \"\"\"----.....______.....----\"\"\"     _.' "
	    echo "          \`\"\"--..,,_____            _____,,..--\"\"\` "
	    echo "                        \`\"\"\"----\"\"\"\` "
	    echo
	    ;;
    esac
fi

while true; do
    nq=$( squeue -o "%.35j %.7i %.8u %.2t %.10M %.6D %R " | grep $name | wc -l )
    npr=$( squeue -o "%.35j %.7i %.8u %.2t %.10M %.6D %R " | grep $name | awk 'NR==1{print $1}' )

    if [ $nq -eq 0 ]; then
	echo
	echo "SLURM jobs finished. Continuing with next processing step."
	echo
	break
    else
	#squeue_status=$(squeue -n $name)
	#echo $squeue_status
	#echo

	time_now=`date +%s`
	runtime_seconds=$((time_now-start_monitor))
	runtime_formated=$(convertsecs $runtime_seconds)
	echo "Running for $runtime_formated and still $nq jobs left in the queue."


	# squeue -o "%.20j %.7i %.8u %.2t %.10M %.6D %R "

	sleep ${duration}s

	#nlines=$( echo $squeue_status | wc -l) #Calculate number of lines for the output previously printed
	#for (( i=0; i <= $(($LINES)); i++ ));do #For each line printed as a result of "timedatectl"
	tput cuu1 #Move cursor up by one line
	tput el #Clear the line
	#done
    fi
done