qsubget.m

function varargout = qsubget(jobid, varargin)

% QSUBGET get the output arguments after the remote job has been executed
% on the Torque, SGE, PBS or SLURM batch queue system.
%
% Use as
%   jobid  = qsubfeval(fname, arg1, arg2, ...)
%   argout = qsubget(jobid, ...)
%
% Optional arguments can be specified in key-value pairs and can include
%   StopOnError    = boolean (default = true)
%   timeout        = number, in seconds (default = 0; return immediately if output cannot be gotten)
%   sleep          = number, in seconds (default = 0.01)
%   output         = string, 'varargout' or 'cell' (default = 'varargout')
%   diary          = string, can be 'always', 'warning', 'error' (default = 'error')
%
% See also QSUBFEVAL, QSUBCELLFUN

% -----------------------------------------------------------------------
% Copyright (C) 2010-2016, Robert Oostenveld
%
% This program is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program.  If not, see <http://www.gnu.org/licenses/
%
% $Id$
% -----------------------------------------------------------------------

% the following are to speed up subsequent calls
persistent previous_varargin previous_timeout previous_sleep previous_output previous_diary previous_StopOnError

if isequal(previous_varargin, varargin)
  % prevent the ft_getopt function from being called, because it is slow
  % reuse the values from the previous call
  timeout     = previous_timeout;
  sleep       = previous_sleep;
  output      = previous_output;
  diary       = previous_diary;
  StopOnError = previous_StopOnError;
else
  % get the optional arguments
  timeout     = ft_getopt(varargin, 'timeout',     0);
  sleep       = ft_getopt(varargin, 'sleep',       0.010);
  output      = ft_getopt(varargin, 'output',      'varargout');
  diary       = ft_getopt(varargin, 'diary',       'error');
  StopOnError = ft_getopt(varargin, 'StopOnError', true);
end

% keep track of the time
stopwatch = tic;
completed = false;

while ~completed && (timeout == 0 || toc(stopwatch)<timeout)
  % the code is largely shared with fieldtrip/peer/peerget.m
  % this section is the only part where it is different between peer and qsub
  
  if qsublist('completed', jobid)
    % the job has completed and the output file exist
    
    curPwd       = getcustompwd();
    outputfile   = fullfile(curPwd, sprintf('%s_output.mat', jobid));
    logout       = fullfile(curPwd, sprintf('%s.o*', jobid)); % note the wildcard in the file name
    logerr       = fullfile(curPwd, sprintf('%s.e*', jobid)); % note the wildcard in the file name
    
    if exist(outputfile, 'file')
      tmp = load(outputfile);
      argout  = tmp.argout;
      options = tmp.optout;
    else
      argout  = {};
      options = {};
    end
    
    % the stderr log file should be empty, otherwise it potentially indicates an error
    direrr = dir(logerr);
    if ~exist(outputfile, 'file') && ~isempty(direrr) && direrr.bytes>0
      fid = fopen(fullfile(curPwd, direrr.name), 'r');
      stderr = fread(fid, [1 inf], 'char');
      fclose(fid);
    else
      stderr = '';
    end
    
    % the stdout log file can be used instead of the MATLAB diary
    dirout = dir(logout);
    if ~exist(outputfile, 'file') && ~isempty(dirout) && dirout.bytes>0
      fid = fopen(fullfile(curPwd, dirout.name), 'r');
      stdout = fread(fid, [1 inf], 'char');
      fclose(fid);
    else
      stdout = '';
    end
    
    % clean up all temporary files
    % delete(inputfile); % this one has already been deleted in qsubexec immediately after loading it
    if exist(outputfile, 'file'), delete(outputfile); end
    if ~isempty(dir(logout)), delete(logout); end % note the wildcard in the file name
    if ~isempty(dir(logerr)), delete(logerr); end % note the wildcard in the file name
    
    % remove the job from the persistent list
    qsublist('del', jobid);
    
    % indicate that the job has completed
    completed = true;
    
  elseif timeout == 0
    % only check once, no waiting here
    break;
    
  else
    % the job results have not arrived yet
    % wait a little bit and try again
    pausejava(sleep);
    continue
  end
  
end % while not completed

if completed
  
  if isempty(options)
    % the job did not complete normally, it might be aborted by the batch queuing system
    if ~isempty(stderr)
      % the stderr output can contain details
      err = sprintf('Error in job execution, the output on stderr is\n%s', stderr);
    else
      err = sprintf('Error in job execution');
    end
    % use the stderr and stdout output rather than the MATLAB results
    options = {'lasterr', err, 'diary', stdout};
  elseif isempty(ft_getopt(options, 'lasterr'))
    fprintf('job %s returned, it required %s and %s on %s\n', jobid, print_tim(ft_getopt(options, 'timused', nan)), print_mem(ft_getopt(options, 'memused', nan)), ft_getopt(options, 'hostname', 'unknown'));
  end
  
  % look at the optional arguments
  warn        = ft_getopt(options, 'lastwarn');
  err         = ft_getopt(options, 'lasterr');
  diarystring = ft_getopt(options, 'diary');
  
  % if there is an error, it needs to be represented as a message string
  % and optionally also as a strucure for rethrowing
  if ~isempty(err)
    if ischar(err)
      errmsg = err;
    elseif isstruct(err)
      errmsg = err.message;
    else
      errmsg = err.message;
      % convert the MEexception object into a structure to allow a rethrow further down in the code
      ws = warning('off', 'MATLAB:structOnObject');
      err = struct(err);
      warning(ws);
    end
  end
  
  % the information that is printed on screen starts and ends with a separator line
  separatorline = false;
  
  if ~isempty(diarystring)
    if strcmp(diary, 'error') && ~isempty(err)
      fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
      fprintf('%% an error was detected, the output of the remote execution follows \n');
      fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
      fprintf('%s', diarystring);
      separatorline = true;
    elseif strcmp(diary, 'warning') && ~isempty(warn)
      fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
      fprintf('%% a warning was detected, the output of the remote execution follows\n');
      fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
      fprintf('%s', diarystring);
      separatorline = true;
    elseif strcmp(diary, 'always')
      fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
      fprintf('%% no problem was detected, the output of the remote execution follows\n');
      fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
      fprintf('%s', diarystring);
      separatorline = true;
    end
  end
  
  if ~isempty(warn)
    fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
    warning(warn);
  end
  
  if ~isempty(err)
    fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
    if StopOnError
      if ischar(err)
        error(err);
      else
        rethrow(err);
      end
    else
      warning('error during remote execution: %s', errmsg);
    end
  end % ~isempty(err)
  
  if separatorline
    fprintf('%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n');
  end
  
  switch output
    case 'varargout'
      % return the output arguments, the options cannot be returned
      varargout = argout;
    case 'cell'
      % return the output arguments and the options
      varargout{1} = argout;
      varargout{2} = options;
    otherwise
      error('invalid output option');
  end
  
else
  warning('FieldTrip:qsub:jobNotAvailable', 'the job results are not yet available');
  switch output
    case 'varargout'
      % return empty output arguments
      varargout = cell(1, nargout);
    case 'cell'
      % return the output arguments and the options as empty cells
      varargout{1} = {};
      varargout{2} = {};
    otherwise
      error('invalid output option');
  end
end

% remember the input arguments to speed up subsequent calls
previous_varargin    = varargin;
previous_timeout     = timeout;
previous_sleep       = sleep;
previous_output      = output;
previous_diary       = diary;
previous_StopOnError = StopOnError;