-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy patharchiveOutput.sh
More file actions
executable file
·448 lines (385 loc) · 24 KB
/
archiveOutput.sh
File metadata and controls
executable file
·448 lines (385 loc) · 24 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
#!/bin/bash
redmine=hpcbio-redmine@igb.illinois.edu
if [ $# != 4 ]
then
MSG="Parameter mismatch."
echo -e "program=$0 stopped. Reason=$MSG" | mail -s 'Variant Calling Workflow failure message' "$redmine"
exit 1;
fi
set -x
echo `date`
scriptfile=$0
runfile=$1
email=$2
elog=$3
olog=$4
umask 0027
outputdir=$( cat $runfile | grep -w OUTPUTDIR | cut -d '=' -f2 )
deliveryfolder=$( cat $runfile | grep -w DELIVERYFOLDER | cut -d '=' -f2 )
TopOutputLogs=$outputdir/logs
pbsprj=$( cat $runfile | grep -w PBSPROJECTID | cut -d '=' -f2 )
pbsqueue=$( cat $runfile | grep -w PBSQUEUEWGEN| cut -d '=' -f2 )
pipeid=$( cat $TopOutputLogs/CONFIGUREpbs )
archivefolder=$outputdir/archive
dartcmd=/projects/sciteam/jti/builds/dart_0_8_5
skipmd5="YES" #this could be changed later on
# wrapping comments in echoes, so that the output logs would be easier to read: they will have more structure
set +x; echo -e "\n\n" >&2;
echo "####################################################################################################" >&2
echo "##################################### PARSING RUN INFO FILE ########################################" >&2
echo "##################################### AND SANITY CHECK ########################################" >&2
echo "####################################################################################################" >&2
echo -e "\n\n" >&2; set -x;
if [ ! -s $runfile ]
then
MSG="$runfile configuration file not found"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
if [ `expr ${#outputdir}` -lt 2 ]
then
MSG="$outputdir folder not specified in configuration file"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
if [ ! -d $outputdir ]
then
MSG="$outputdir folder not found"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
elif [ ! "$(ls -A $outputdir)" ]
then
MSG="$outputdir Is empty"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
if [ ! -d $TopOutputLogs ]
then
MSG="$TopOutputLogs folder not found"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
if [ `expr ${#deliveryfolder}` -lt 2 ]
then
delivery=$outputdir/delivery
else
delivery=$outputdir/$deliveryfolder
fi
if [ ! -d $delivery ]
then
MSG="$delivery folder not found"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
elif [ ! "$(ls -A $delivery)" ]
then
MSG="$delivery folder is empty. Execution of pipeline did not produce results. Nothing to archive. Exiting now"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
# names of output folders for each sample should be specified in SAMPLENAMES.list. Checking that the file exists
if [ ! -s $outputdir/SAMPLENAMES.list ]
then
MSG="$outputdir/SAMPLENAMES.list file not found"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
# this is where the dart files will go
if [ ! -d $archivefolder ]
then
echo -e "$archivefolder directory does not exist, creating it"
mkdir -p $archivefolder
else
echo -e "$archivefolder directory does exist, resetting it"
rm -r $archivefolder
mkdir -p $archivefolder
fi
chmod -R 770 $archivefolder
if [ ! -s $dartcmd ]
then
MSG="$dartcmd dart executable not found"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
#echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" | ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1;
fi
set +x; echo -e "\n\n" >&2;
echo "####################################################################################################" >&2
echo "####################### declaring END POINTS ########################################" >&2
echo "####################################################################################################" >&2
echo -e "\n\n" >&2; set -x;
###############################################################################################################
#this is how end points are normally specified
#Destination="ncsa#Nearline/projects/sciteam/jti/TestingAutoTransfer_baylor_barch_1"
#Source="ncsa#BlueWaters/scratch/sciteam/lmainzer/H3A_DiversityGenotyping_Space/DartPackages/Striped_Minus1"
#we need to declare them here for this particular project/batch
###############################################################################################################
thisbatch=`basename $outputdir`
#thisproject=/projects/sciteam/jti/Results
thisproject=/projects/sciteam/jti/TestingAutoTransfer_batch2
source_endpoint="ncsa#BlueWaters"
destination_endpoint="ncsa#Nearline"
source=${source_endpoint}${archivefolder}
destination=${destination_endpoint}${thisproject}/$thisbatch
#user=$( echo $email | cut -d '@' -f1 )
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "########## TESTING END POINTS: 1) that they exist 2) that we have access to them ########" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
#We assume that the userid is identical on both end points AND on globusonline
user=$( echo $USER )
if [ `expr ${#user}` -lt 2 ]
then
MSG="$user value is missing."
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1
elif [ $user == "rendong" -o $user == "grendon" ]
then
user="gloriarendon" #because userid on BW is different from userid on GO
elif [ $user == "botha" ]
then
user="grbot"
fi
echo "checking ssh cli.globusonline.org on $source"
ssh $user@cli.globusonline.org "ls $source"
if [ $? -ne 0 ]
then
MSG="ssh cli.globusonline.org command failed on $source."
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1
else
echo -e "$source is ok"
fi
echo "checking ssh cli.globusonline.org on $destination"
ssh $user@cli.globusonline.org "ls $destination"
if [ $? -ne 0 ]
then
echo "$destination folder does not exist, creating it now"
ssh $user@cli.globusonline.org "mkdir $destination"
else
echo -e "$destination is ok"
fi
echo `date`
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "##########################################################################################" >&2
echo "########## Create al the qsub jobs here ########" >&2
echo "########## md5sum commands will be executed from within a launcher ########" >&2
echo "### dart transfer commands will be launched from individual qsub jobs ####" >&2
echo "##########################################################################################" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
if [ $skipmd5 == "YES" ]
then
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "########## SKIP md5sum STEP because GO does it for us ########" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
else
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "########## Let's create the qsubs for the md5sum launcher ########" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
echo -e "resetting the joblists and qsub files ..."
truncate -s 0 $TopOutputLogs/md5sumBAMS.AnisimovJoblist
truncate -s 0 $TopOutputLogs/md5sumVCFS.AnisimovJoblist
truncate -s 0 $TopOutputLogs/md5sum.pbs
qsub_md5sumBAMS=$TopOutputLogs/qsub_md5sumBAMS.AnisimovJoblist
qsub_md5sumVCFS=$TopOutputLogs/qsub_md5sumVCFS.AnisimovJoblist
echo -e "walltimes for these jobs..."
md5sumBAMS_cputime="06:00:00"
md5sumVCFS_cputime="02:00:00"
echo -e "counters for the anisimov launchers"
md5sumBAMS_files=1
md5sumVCFS_files=1
echo -e "these are the lists of files that MAY need md5sum"
bamfiles1=`find $delivery -name *.bam -type f`
vcfFiles=`find $delivery -name *.vcf.gz -type f`
echo -e "populating the anisimov launcher for the md5sum of BAMS"
for bam in $bamfiles1
do
echo -e "nohup md5sum $bam > ${bam}.md5\n" >> $TopOutputLogs/md5sumBAMS.AnisimovJoblist
(( md5sumBAMS_files++ ))
done
echo -e "populating the anisimov launcher for the md5sum of VCFS"
for vcf in $vcfFiles
do
echo -e "nohup md5sum $vcf > ${vcf}.md5\n" >> $TopOutputLogs/md5sumVCFS.AnisimovJoblist
(( md5sumVCFS_files++ ))
done
echo -e "calculating number of nodes needed for each launcher"
NumberOfProcPerNode=16 #there are 32 cpus per node, each job uses 2 cpus
if [ $md5sumBAMS_files -lt 17 ]
then
md5sumBAMS_nodes=1
NumberOfProcPerNode=$md5sumBAMS_files
else
md5sumBAMS_nodes=$(( md5sumBAMS_files/NumberOfProcPerNode ))
if [ `expr $md5sumBAMS_nodes % $NumberOfProcPerNode` -gt 0 ]
then
(( md5sumBAMS_nodes++ )) # there is a remainder in that division, and we give those remaining jobs an extra node
fi
fi
if [ $md5sumVCFS_files -lt 17 ]
then
md5sumVCFS_nodes=1
NumberOfProcPerNode=$md5sumVCFS_files
else
md5sumVCFS_nodes=$(( md5sumVCFS_files/NumberOfProcPerNode ))
if [ `expr $md5sumVCFS_nodes % $NumberOfProcPerNode` -gt 0 ]
then
(( md5sumVCFS_nodes++ )) # there is a remainder in that division, and we give those remaining jobs an extra node
fi
fi
# we need to add one more node for the launcher
(( md5sumBAMS_nodes++ ))
(( md5sumVCFS_nodes++ ))
echo -e "NOW we have all the pieces together to form the qsub files for the TWO anisimov launchers"
cat $outputdir/qsubGenericHeader > $qsub_md5sumBAMS
echo "#PBS -N ${pipeid}_md5sumBAMS_Anisimov" >> $qsub_md5sumBAMS
echo "#PBS -o $TopOutputLogs/log.md5sumBAMS_Anisimov.ou" >> $qsub_md5sumBAMS
echo "#PBS -e $TopOutputLogs/log.md5sumBAMS_Anisimov.in" >> $qsub_md5sumBAMS
echo "#PBS -l nodes=${md5sumBAMS_nodes}:ppn=32:xe" >> $qsub_md5sumBAMS
echo "#PBS -l walltime=$md5sumBAMS_cputime" >> $qsub_md5sumBAMS
echo "aprun -n $md5sumBAMS_files -N $NumberOfProcPerNode -d 2 ~anisimov/scheduler/scheduler.x $TopOutputLogs/md5sumBAMS.AnisimovJoblist /bin/bash > $TopOutputLogs/log.md5sumBAMS_Anisimov.ou " >> $qsub_md5sumBAMS
md5sumBAMS_job=`qsub $qsub_md5sumBAMS`
`qhold -h u $md5sumBAMS_job`
echo $md5sumBAMS_job >> $TopOutputLogs/md5sumBAMS.AnisimovJoblist
echo $md5sumBAMS_job >> $TopOutputLogs/md5sum.pbs
cat $outputdir/qsubGenericHeader > $qsub_md5sumVCFS
echo "#PBS -N ${pipeid}_md5sumVCFS_Anisimov" >> $qsub_md5sumVCFS
echo "#PBS -o $TopOutputLogs/log.md5sumVCFS_Anisimov.ou" >> $qsub_md5sumVCFS
echo "#PBS -e $TopOutputLogs/log.md5sumVCFS_Anisimov.in" >> $qsub_md5sumVCFS
echo "#PBS -l nodes=${md5sumVCFS_nodes}:ppn=32:xe" >> $qsub_md5sumVCFS
echo "#PBS -l walltime=$md5sumVCFS_cputime" >> $qsub_md5sumVCFS
echo "aprun -n $md5sumVCFS_files -N $NumberOfProcPerNode -d 2 ~anisimov/scheduler/scheduler.x $TopOutputLogs/md5sumVCFS.AnisimovJoblist /bin/bash > $TopOutputLogs/log.md5sumVCFS_Anisimov.ou " >> $qsub_md5sumVCFS
md5sumVCFS_job=`qsub $qsub_md5sumVCFS`
`qhold -h u $md5sumVCF_job`
echo $md5sumVCFS_job >> $TopOutputLogs/md5sumVCFS.AnisimovJoblist
echo $md5sumVCFS_job >> $TopOutputLogs/md5sum.pbs
fi
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "###### Create the qsubs one for the for dart and one for transfer of sample folders ###" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
echo -e "this variable will be empty if we skip the md5sum. However, the code will still work"
if [ $skipmd5 != "YES" ]
then
md5sum_all=$( cat $TopOutputLogs/md5sum.pbs | sed "s/\..*//" | tr "\n" ":" )
fi
echo -e " resetting lists and resources"
truncate -s 0 $TopOutputLogs/globusTransfer.pbs
truncate -s 0 $TopOutputLogs/DART.pbs
globusTransfer_cputime="12:00:00"
DART_cputime="02:00:00"
DART_nodes=10
globus_nodes=1
echo `date`
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "########## Now lets create the README file and send a copy to archive ########" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
archivedFiles=$( cat $outputdir/SAMPLENAMES.list | sed "s/\n/.drt\n/g" )
readme=$delivery/docs/README.txt
chmod ug=rw $readme
truncate -s 0 $readme
thisDate=$( echo `date` )
echo -e "On $thisDate $user archived the sample folders \nfrom: $outputdir \nto: $destination\n\n" >> $readme
echo -e "The command used to archive was \n$dartcmd -C -e 20 -B 1000 -F archivefolder/{SampleFolder}.drt -r outputdir/SampleFolder \n\n" >> $readme
echo -e "The command to extract one or more files is \n$dartcmd -X -F $destination/{SampleFolder}.drt -r yourCurrentPath \n\n" >> $readme
echo -e "Notice that the extract command MUST be issued from the folder where the file will be EXTARCTED TO\n\n" >> $readme
echo -e "List of files in $destination \n\n" >> $readme
echo $archivedFiles >> $readme
echo -e "\n\n\nNote: Each drt file is a packaged sample folder\n\n\n" >> $readme
cp $readme $archivefolder/README.txt
ssh $user@cli.globusonline.org "transfer --label README --verify-checksum -- $source/README.txt $destination/README.txt "
if [ $? -ne 0 ]
then
MSG="transfer of $readme to $destination failed"
echo -e "program=$scriptfile stopped at line=$LINENO.\nReason=$MSG\n$LOGS" #| ssh iforge "mailx -s '[Support #200] variant identification pipeline' "$redmine,$email""
exit 1
else
echo -e "transfer of $readme to $destination was ok"
fi
echo `date`
set +x; echo -e "\n\n" >&2;
echo -e "##########################################################################################" >&2
echo -e "###### Next: the loop for schedulling dart and globus jobs, one per sample #####" >&2
echo -e "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
while read SampleFolder
do
if [ `expr ${#SampleFolder}` -lt 1 ]
then
set +x; echo -e "\n\n" >&2;
echo -e "##########################################################################################" >&2
echo -e "################ skipping empty line #######################" >&2
echo -e "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
else
set +x; echo -e "\n\n" >&2;
echo -e "##########################################################################################" >&2
echo -e "##### qsub preparation for DART packaging of $SampleFolder ######" >&2
echo -e "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
qsub_dart=$TopOutputLogs/qsub_DART_$SampleFolder
cat $outputdir/qsubGenericHeader > $qsub_dart
echo "#PBS -N ${pipeid}_DART_$SampleFolder" >> $qsub_dart
echo "#PBS -o $TopOutputLogs/log.DART_$SampleFolder.ou" >> $qsub_dart
echo "#PBS -e $TopOutputLogs/log.DART_$SampleFolder.er" >> $qsub_dart
echo "#PBS -l nodes=${DART_nodes}:ppn=32" >> $qsub_dart
echo "#PBS -l walltime=$DART_cputime" >> $qsub_dart
if [ $skipmd5 != "YES" ]
then
echo "#PBS -W depend=afterok:$md5sum_jobs " >> $qsub_dart
fi
echo "set -x" >> $qsub_dart
echo " aprun -n 80 -d 4 $dartcmd -C -e 20 -B 1000 -F $archivefolder/${SampleFolder}.drt -r $outputdir/$SampleFolder " >> $qsub_dart
dart_job=`qsub $qsub_dart`
`qhold -h u $dart_job`
echo $dart_job >> $TopOutputLogs/DART.pbs
set +x; echo -e "\n\n" >&2;
echo -e "##########################################################################################" >&2
echo -e "##### qsub preparation for GO archival of $SampleFolder ######" >&2
echo -e "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
qsubGO=$TopOutputLogs/qsub_globusTransfer_$SampleFolder
cat $outputdir/qsubGenericHeader > $qsubGO
cmd="\"transfer --label $SampleFolder --verify-checksum --encryp -- $source/${SampleFolder}.drt $destination/$SampleFolder/${SampleFolder}.drt \""
echo "#PBS -N ${pipeid}_globusTransfer_$SampleFolder" >> $qsubGO
echo "#PBS -o $TopOutputLogs/log.mglobusTransfer_$SampleFolder.ou" >> $qsubGO
echo "#PBS -e $TopOutputLogs/log.mglobusTransfer_$SampleFolder.er" >> $qsubGO
echo "#PBS -l nodes=${globus_nodes}:ppn=32" >> $qsubGO
echo "#PBS -l walltime=$globusTransfer_cputime" >> $qsubGO
echo "#PBS -W depend=afterok:$dart_job " >> $qsubGO
echo "ssh $user@cli.globusonline.org $cmd" >> $qsubGO
globus_job=`qsub $qsubGO`
`qhold -h u $globus_job`
echo $globus_job >> $TopOutputLogs/globusTransfer.pbs
fi
echo -e "##### NEXT SAMPLE PLEASE.... #####"
done < $outputdir/SAMPLENAMES.list
set +x; echo -e "\n\n" >&2;
echo "##########################################################################################" >&2
echo "########## Now we release all the jobs for execution ########" >&2
echo "##########################################################################################" >&2
echo -e "\n\n" >&2; set -x;
md5sum_all=$( cat $TopOutputLogs/md5sum.pbs | sed "s/\..*//" | tr "\n" " " )
globus_all=$( cat $TopOutputLogs/globusTransfer.pbs | sed "s/\..*//" | tr "\n" " " )
dart_all=$( cat $TopOutputLogs/DART.pbs | sed "s/\..*//" | tr "\n" " " )
`qrls -h u $md5sum_all`
`qrls -h u $dart_all`
`qrls -h u $globus_all`
echo `date`
echo -e "exiting now..."