New Test Suite

initCobraToolbox.m

@@ -40,9 +40,13 @@ function initCobraToolbox()
     global WAITBAR_TYPE;
     global ENV_VARS;
     global gitBashVersion;
+    global CBT_MISSING_REQUIREMENTS;
 
     % define a base version of gitBash that is tested
     gitBashVersion = '2.13.3';
+
+     % Set the Requirements Error Message ID:
+    CBT_MISSING_REQUIREMENTS = 'COBRA:RequirementsNotMet';
 
     % default waitbar is of type text
     if isempty(WAITBAR_TYPE)

src/base/solvers/getAvailableSolversByType.m

@@ -0,0 +1,34 @@
+function solvers = getAvailableSolversByType()
+% Get the available Solvers for the different Types on the system.
+%
+% USAGE:
+%    solvers = getAvailableSolversByType()
+%
+% OUTPUT:
+%    solvers:   struct containing one field per Problem type listing all
+%               solvers installed on the system for that problem type.
+%
+
+global OPT_PROB_TYPES;
+global SOLVERS;
+
+if isempty(SOLVERS) || isempty(OPT_PROB_TYPES)
+    ENV_VARS.printLevel = false;
+    initCobraToolbox;
+    ENV_VARS.printLevel = true;
+end
+
+solverNames = fieldnames(SOLVERS);
+solvers = struct();
+for i = 1:numel(OPT_PROB_TYPES)
+    solvers.(OPT_PROB_TYPES{i}) = {};
+end
+
+for i = 1:numel(solverNames)
+   if SOLVERS.(solverNames{i}).working
+       availableTypes = SOLVERS.(solverNames{i}).type;
+       for j = 1:numel(availableTypes)
+           solvers.(availableTypes{j}) = union(solvers.(availableTypes{j}), solverNames{i});
+       end
+   end
+end

test/COBRARequisitesFullfilled.m

@@ -0,0 +1,169 @@
+function [solversToUse] = COBRARequisitesFullfilled(varargin)
+% Checks the prerequisites of the test, and returns solvers depending on
+% the input parameters. If the requirements are NOT met, it will throw a
+% COBRA:RequirementsNotMet error.
+%
+% USAGE:
+%    [tf,solversToUse] = COBRARequisitesFullfilled(varargin)
+% 
+% INPUTS:
+%    varagin:       'ParameterName',value pairs with the following
+%                   Parameter options:
+%                   * 'Toolboxes'      - Names of toolboxes (the license
+%                                        feature name) (Default: {})
+%                   * 'ReqSolvers'     - Names of all solvers which MUST be
+%                                        available (Default: {})
+%                   * 'UseIfAvailable' - Names of solvers which should be
+%                                        used if they are available (will
+%                                        not throw an error if not).
+%                   * 'NeedsLP'        - Whether a LP solver is required.
+%                                       (Default = false);
+%                   * 'NeedsMILP'      - Whether a MILP solver is required.
+%                                       (Default = false);
+%                   * 'NeedsQP'        - Whether a QP solver is required.
+%                                       (Default = false);
+%                   * 'NeedsMIQP'      - Whether a MIQP solver is required.
+%                                       (Default = false);
+%                   * 'NeedsNLP'       - Whether a NLP solver is required.
+%                                       (Default = false);
+
+
+%Do some precomputation.
+global CBT_MISSING_REQUIREMENTS;
+
+persistent availableSolvers
+
+
+%Some Matlab Toolboxes currently in use in the COBRA Toolbox. 
+%This might have to be extended in the future. 
+toolboxInfo = struct('statistics_toolbox',{'Statistics and Machine Learning Toolbox','Statistics Toolbox'},...
+                     'bioinformatics_toolbox',{'Bioinformatics Toolbox'},...
+                     'distrib_computing_toolbox',{'Parallel Computing Toolbox'},...
+                     'optimization_toolbox',{'Optimization Toolbox'},...
+                     'global_optimization_toolbox',{'Global Optimization Toolbox'},...
+                     'image_toolbox','Image Processing Toolbox');
+
+if isempty(availableSolvers)
+    availableSolvers = getAvailableSolversByType();
+    fieldsWithSolvers = fieldnames(availableSolvers);
+    availableSolvers.ALL = {};
+    for i = 1:numel(fieldsWithSolvers)
+        availableSolvers.ALL = union(availableSolvers.ALL,availableSolvers.(fieldsWithSolvers{i}));
+    end
+end
+
+
+parser = inputParser()
+parser.addParamValue('Toolboxes',{},@iscell);
+parser.addParamValue('ReqSolvers',{},@iscell);
+parser.addParamValue('UseIfAvailable',{},@iscell);
+parser.addParamValue('NeedsLP',false,@(x) islogic(x) || x == 1 || x == 0 );
+parser.addParamValue('NeedsMILP',false,@(x) islogic(x) || x == 1 || x == 0 );
+parser.addParamValue('NeedsNLP',false,@(x) islogic(x) || x == 1 || x == 0 );
+parser.addParamValue('NeedsQP',false,@(x) islogic(x) || x == 1 || x == 0 );
+parser.addParamValue('NeedsMIQP',false,@(x) islogic(x) || x == 1 || x == 0 );
+
+parser.parse(varargin{:});
+
+UseQP = parser.Results.NeedsQP;
+UseLP = parser.Results.NeedsLP;
+UseMIQP = parser.Results.NeedsMIQP;
+UseNLP = parser.Results.NeedsNLP;
+UseMILP = parser.Results.NeedsMILP;
+
+Toolboxes = parser.Results.Toolboxes;
+RequiredSolvers = parser.Results.ReqSolvers;
+PreferredSolvers = parser.Results.UseIfAvailable;
+
+errorMessage = {};
+
+%First, check the required Solvers
+if ~isempty(RequiredSolvers) && ~all(ismember(RequiredSolvers,availableSolvers.ALL))
+    %We have required solvers and some are missing
+    missing = ~ismember(RequiredSolvers,availableSolvers.ALL);
+    errorMessage{end+1} = sprintf('%s are missing required solvers for the test.', strjoin(RequiredSolvers(missing),' and '));
+end
+
+%Now, Check the Toolboxes
+res = ver;
+missingTBs = struct('License',{},'Installation',{});
+for i = 1:numel(Toolboxes)
+    tbstring = lower(Toolboxes{i});
+    licpres = license('test',tbstring);
+    if any(tbstring,fieldnames(toolboxInfo))
+        tbpres = any(ismember(toolboxInfo.(Toolboxes{i}),{ver.name}));
+    else
+        %We will rely on the license....
+        tbpres = licpres;
+    end        
+    if ~tbpres
+        missingTBs.Installation{end+1} = tbstring;       
+    end
+    if ~licpres
+        missingTBs.License{end+1} = tbstring;       
+    end
+end
+%Append the error message.
+if ~isempty(missingTbs.License)
+    errorMessage{end+1} = sprintf('The test requires licenses for the following Toolboxes: %s', strjoin(missingTbs.License,' and '));     
+end
+if ~isempty(missingTbs.Installation)
+    errorMessage{end+1} = sprintf('The test the following Toolboxes to be installed: %s', strjoin(missingTbs.Installation,' and '));     
+end
+
+%Set up default solvers.
+if isempty(availableSolvers.LP)    
+    if needLP
+        errorMessage{end+1} = 'The test requires at least one LP solver but no solver is installed';
+    end    
+else
+    defaultLPSolver = availableSolvers.LP{1};     
+end
+
+if isempty(availableSolvers.QP)    
+    if needQP
+        errorMessage{end+1} = 'The test requires at least one QP solver but no solver is installed';
+    end    
+else
+    defaultQPSolver = availableSolvers.QP{1};     
+end
+
+if isempty(availableSolvers.MILP)    
+    if needMILP
+        errorMessage{end+1} = 'The test requires at least one MILP solver but no solver is installed';
+    end    
+else
+    defaultMILPSolver = availableSolvers.MILP{1};     
+end
+
+if isempty(availableSolvers.MIQP)    
+    if needMIQP
+        errorMessage{end+1} = 'The test requires at least one MIQP solver but no solver is installed';
+    end    
+else
+    defaultMIQPSolver = availableSolvers.MIQP{1};     
+end
+
+if isempty(availableSolvers.NLP)    
+    if needNLP
+        errorMessage{end+1} = 'The test requires at least one NLP solver but no solver is installed';
+    end    
+else
+    defaultNLPSolver = availableSolvers.NLP{1};     
+end
+
+
+
+if ~isempty(errorMessage)
+    error(CBT_MISSING_REQUIREMENTS,strjoin(errorMessage,'\n'));
+end
+
+%Ok, we are successfull. so lets collect the Used Solvers.
+solversToUse = struct();
+problemTypes = fieldnames(availableSolvers);
+for i = 1:problemTypes
+    solversToUse.(problemTypes{i}) = intersect(PreferredSolvers,availableSolvers.(problemTypes{i}));
+    if isempty(solversToUse.(problemTypes{i})) && ~isempty(availableSolvers.(problemTypes{i}))
+        eval(['solversToUse.' problemTypes{i} ' = default' problemTypes{i} 'Solver']);
+    end
+end

test/runCOBRATestSuite.m

@@ -0,0 +1,62 @@
+function results = runCOBRATestSuite()
+% This function runs all tests (i.e. files starting with 'test' in the
+% CBTDIR/test/ folder and returns the status.
+% It can distinguish between skipped and Failed tests. A test is considered
+% to be skipped if it throws a COBRA:RequirementsNotMet error.
+%
+% OUTPUTS:
+%
+%    results:   A structure array with one entry per test and the following fields:
+%               .passed - true if the test passed otherwise false
+%               .skipped - true if the test was skipped otherwise false
+%               .failed - true if the test failed, or was skipped,
+%                         otherwise false
+%               .status - a string representing the status of the test
+%                         ('failed','skipped' or'passed')
+%               .fileName - the fileName of the test 
+%               .time - the duration of the test (if passed otherwise NaN)
+%               .statusMessage - Informative string about potential
+%                                problems.
+%               .Error - The Error message received from a failed or skipped test 
+%
+% Author:
+%    - Thomas Pfau Jan 2018.
+
+
+global CBTDIR
+
+
+%Go to the test directory.
+testDir = [CBTDIR filesep 'test'];
+currentDir = cd(testDir);
+
+%Get all names of test files
+testFiles = rdir(['verifiedTests' filesep '**' filesep 'test*.m']);
+testFileNames = {testFiles.name};
+
+testFileNames = testFileNames(1:5);
+
+%Run the tests and show outputs.
+for i = 1:numel(testFileNames)
+    [~,file,ext] = fileparts(testFileNames{i});
+    testName = file;
+    fprintf('****************************************************\n\n');
+    fprintf('Running %s\n\n',testName);
+    results(i) = runScriptFile([file ext]);
+    fprintf('\n\n%s %s!\n',testName,results(i).status);    
+    if ~results(i).passed
+        if results(i).skipped
+            fprintf('Reason:\n%s\n',results(i).statusMessage);
+        else
+            trace = results(i).Error.getReport();
+            tracePerLine = strsplit(trace,'\n');
+            testSuitePosition = find(cellfun(@(x) ~isempty(strfind(x,'runCOBRATestSuite')),tracePerLine));
+            trace = sprintf(strjoin(tracePerLine(1:(testSuitePosition-7)),'\n')); % Remove the testSuiteTrace.
+            fprintf('Reason:\n%s\n',trace);        
+        end
+    end
+    fprintf('\n\n****************************************************\n');
+end
+
+%Change back to the original directory.
+cd(currentDir)

test/runScriptFile.m

@@ -0,0 +1,62 @@
+function result = runScriptFile(fileName)
+% This function runs the test in fileName 
+% It can distinguish between skipped and Failed tests. A test is considered
+% to be skipped if it throws a COBRA:RequirementsNotMet error.
+%
+% OUTPUTS:
+%
+%    result:    A structure array with the following fields:
+%               .passed - true if the test passed otherwise false
+%               .skipped - true if the test was skipped otherwise false
+%               .failed - true if the test failed, or was skipped,
+%                         otherwise false
+%               .status - a string representing the status of the test
+%                         ('failed','skipped' or'passed')
+%               .fileName - the fileName of the test 
+%               .time - the duration of the test (if passed otherwise NaN)
+%               .statusMessage - Informative string about potential
+%                                problems.
+%               .Error - The Error message received from a failed or skipped test 
+%
+% Author:
+%    - Thomas Pfau Jan 2018.
+
+global CBT_MISSING_REQUIREMENTS;
+
+COBRA_TESTSUITE_TESTFILE = fileName;
+
+%Get the timinig (and hope these values are not overwritten.
+COBRA_TESTSUITE_STARTTIME = clock();
+try
+    %Run the file
+    executefile(fileName);
+catch ME
+    %Catch errors and interpret them
+    clearvars -except ME COBRA_TESTSUITE_STARTTIME COBRA_TESTSUITE_TESTFILE    
+    result = struct('status','failed','failed',true,'passed',false,'fileName',...
+        COBRA_TESTSUITE_TESTFILE,'time',NaN, 'statusMessage', 'fail', 'Error',ME);    
+    if strcmp(ME.identifier,CBT_MISSING_REQUIREMENTS)
+        %Requirement missing, so the test was skipped.
+        result.status = 'skipped';  
+        result.skipped = true;
+        result.statusMessage = ME.message;
+    else
+        %Actual error in the test.
+        result.skipped = false;
+        result.status = 'failed';
+        result.statusMessage = ME.message;
+    end    
+    return
+end
+%Get the timinig.
+scriptTime = etime(clock(),COBRA_TESTSUITE_STARTTIME);
+
+result = struct('status','passed','failed',false,'passed',true,'skipped',false,'fileName',...
+    COBRA_TESTSUITE_TESTFILE, 'time', scriptTime,'statusMessage','success', 'Error',MException('',''));
+
+end
+
+function executefile(fileName)
+%Runs a script file (used to separate workspaces)
+run(fileName)
+end

test/untitled4.m

@@ -0,0 +1 @@
+disp('Uuuuuups');