Skip to content

[bugfix] Check that PBS output is written back to working directory before setting the job as completed #2519

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vkarak merged 3 commits into from
May 19, 2022
Merged

[bugfix] Check that PBS output is written back to working directory before setting the job as completed #2519

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
25196f2
Fix pbs stdout/stderr bug
ekouts May 18, 2022
995ae3c
Rename helper function
ekouts May 18, 2022
135ed0d
Address PR comments
ekouts May 18, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 17 additions & 10 deletions reframe/core/schedulers/pbs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,13 @@ def _update_nodelist(self, job, nodespec):
job._nodelist.sort()

def poll(self, *jobs):
def output_ready(job):
# We report a job as finished only when its stdout/stderr are
# written back to the working directory
stdout = os.path.join(job.workdir, job.stdout)
stderr = os.path.join(job.workdir, job.stderr)
return os.path.exists(stdout) and os.path.exists(stderr)

if jobs:
# Filter out non-jobs
jobs = [job for job in jobs if job is not None]
Expand All @@ -198,11 +205,12 @@ def poll(self, *jobs):
# Otherwise, it will return with return code 0 and print information
# only for the jobs it could find.
if completed.returncode in (153, 35):
self.log(f'Return code is {completed.returncode}: '
f'assuming all jobs completed')
self.log(f'Return code is {completed.returncode}')
for job in jobs:
job._state = 'COMPLETED'
job._completed = True
if job.cancelled or output_ready(job):
self.log(f'Assuming job {job.jobid} completed')
job._completed = True

return

Expand All @@ -224,10 +232,12 @@ def poll(self, *jobs):

for job in jobs:
if job.jobid not in jobinfo:
self.log(f'Job {job.jobid} not known to scheduler, '
f'assuming job completed')
self.log(f'Job {job.jobid} not known to scheduler')
job._state = 'COMPLETED'
job._completed = True
if job.cancelled or output_ready(job):
self.log(f'Assuming job {job.jobid} completed')
job._completed = True

continue

info = jobinfo[job.jobid]
Expand Down Expand Up @@ -259,10 +269,7 @@ def poll(self, *jobs):

# We report a job as finished only when its stdout/stderr are
# written back to the working directory
stdout = os.path.join(job.workdir, job.stdout)
stderr = os.path.join(job.workdir, job.stderr)
out_ready = os.path.exists(stdout) and os.path.exists(stderr)
done = job.cancelled or out_ready
done = job.cancelled or output_ready(job)
if done:
job._completed = True
elif (job.state in ['QUEUED', 'HELD', 'WAITING'] and
Expand Down