From 86375e9ce1c4a54a82816005fced123af39b4e77 Mon Sep 17 00:00:00 2001 From: zagganas Date: Mon, 17 Jan 2022 19:17:59 +0000 Subject: [PATCH] Use TES for task execution --- components/RoCrateModal.php | 8 +- components/RunFormWidget.php | 3 +- controllers/SoftwareController.php | 422 ++++---------- controllers/WorkflowController.php | 208 +------ models/Software.php | 885 ++++++++++++----------------- models/Workflow.php | 57 +- scheduler_files/jobMonitor.py | 262 ++++----- views/software/logs.php | 5 +- views/software/run.php | 39 +- web/js/software/logs.js | 308 ++++------ web/js/software/run-index.js | 162 +----- 11 files changed, 786 insertions(+), 1573 deletions(-) diff --git a/components/RoCrateModal.php b/components/RoCrateModal.php index 4387f68..0397c60 100644 --- a/components/RoCrateModal.php +++ b/components/RoCrateModal.php @@ -79,8 +79,8 @@ public static function addModal($jobid) $fields=WorkflowInput::find()->where(['workflow_id'=>$software_id])->orderBy(['position'=> SORT_ASC])->all(); if (!empty($fields)) { - $fields=Workflow::getRerunFieldValues($jobid,$fields); - if ($fields==false) + $fields_tmp=Workflow::getRerunFieldValues($jobid,$fields); + if ($fields_tmp==false) { $fields=[]; } @@ -90,7 +90,6 @@ public static function addModal($jobid) $model=RoCrate::find()->where(['jobid'=>$jobid])->one(); - //print_r($history->jobid); $disabled_fields=false; if(!empty($model)) { @@ -188,8 +187,7 @@ public static function addModal($jobid) echo ''; echo "
"; $i=0; - // print_r($history->jobid); - // exit(0); + foreach ($fields as $field) { diff --git a/components/RunFormWidget.php b/components/RunFormWidget.php index 197d255..c2472fc 100644 --- a/components/RunFormWidget.php +++ b/components/RunFormWidget.php @@ -61,13 +61,14 @@ public static function showOutputField($commandsDisabled,$outFolder,$username,$t $runType='workflow'; } echo Html::hiddenInput('mountcaller',null ,['id'=>'mountcaller']); + $disabledBox=$commandsDisabled ? 'disabled-box' : ''; ?>

Output directory

- 'outFolder','class'=>'mount-field','readonly'=>true,])?> + 'outFolder','class'=>"mount-field $disabledBox",'readonly'=>true,])?> 'select-output-button btn btn-success btn-md','disabled'=>($commandsDisabled)])?> 'clear-output-button btn btn-danger btn-md','disabled'=>($commandsDisabled)])?>
diff --git a/controllers/SoftwareController.php b/controllers/SoftwareController.php index 5714c2a..29a6981 100644 --- a/controllers/SoftwareController.php +++ b/controllers/SoftwareController.php @@ -111,11 +111,6 @@ public function actions() public function actionIndex($selected_project='') { - // $session = Yii::$app->session; - // $session->set('user_id', '1234'); - // print_r($session['new_session']); - // exit(0); - /** * If the user-data folder for the current user does not exist, * create one @@ -164,14 +159,6 @@ public function actionIndex($selected_project='') 'images'=>$images, 'profiled'=>$profiled]); } - /* What the actual fuck is this???? */ - // public function actionIsProfiled($name,$version) - // { - // $profiled_value=Software::find()->select('profiled')->where(['name'=>$name])->andWhere(['version'=>$version])->scalar(); - // return $profiled_value; - - // } - /** * Action to run a docker image uploaded in the system */ @@ -204,8 +191,7 @@ public function actionRun($name, $version) } $software=Software::find()->where(['name'=>$name,'version'=>$version])->one(); - $software_id=$software->id; - $software_instructions=$software->instructions; + if (empty($software)) { return $this->render('no_software',['name'=>$name,'version'=>$version]); @@ -232,160 +218,39 @@ public function actionRun($name, $version) * or assign default values */ $jobid=isset($_POST['jobid']) ? $_POST['jobid'] : ''; - // $podid=isset($_POST['podid']) ? $_POST['podid'] : ''; - $machineType=isset($_POST['machineType']) ? $_POST['machineType'] : ''; $example=(isset($_POST['example']) && ($_POST['example']=="1")) ? true : false; $outFolder=(isset($_POST['outFolder'])) ? $_POST['outFolder'] : ''; - /* - * contMount variables contain the mountpoints inside the container as specified during the addition process - * SystemMount variableσ contain the local path that will be mounted to the container. - */ - // $containerMounts=$softwareModel::getContainerMountpoint($name,$version); - $icontMount=$software->imountpoint; - $ocontMount=$software->omountpoint; - $iocontMount=''; - if ((!empty($icontMount)) && (!empty($ocontMount))) - { - if ($icontMount==$ocontMount) - { - $iocontMount=$icontMount; - } - } - // print_r($iocontMount); - // exit(0); - - $isystemMountField=isset($_POST['isystemmount']) ? $_POST['isystemmount'] : '' ; - $osystemMountField=isset($_POST['osystemmount']) ? $_POST['osystemmount'] : '' ; - $iosystemMountField=isset($_POST['iosystemmount']) ? $_POST['iosystemmount'] : '' ; - - if ($example) - { - $iosystemMountField=''; - $isystemMountField=''; - $osystemMountField=''; - /* - * If iomount is empty, then see if imount and omount are filled - */ - if (!empty($iocontMount)) - { - $iosystemMountField='examples/' . $name . '/' . $version . '/io/'; - } - else - { - if (!empty($icontMount)) - { - $isystemMountField='examples/' . $name . '/' . $version . '/input/'; - } - if (!empty($ocontMount)) - { - $osystemMountField='examples/' . $name . '/' . $version . '/output/'; - } - } - - } - - - $iosystemMount=''; - $isystemMount=''; - $osystemMount=''; - /* - * If iomount is empty, then see if imount and omount are fi - */ - if (!empty($iocontMount)) - { - $iosystemMount=Yii::$app->params['userDataPath'] . explode('@',$user)[0] . '/' . $iosystemMountField; - } - else - { - if (!empty($icontMount)) - { - $isystemMount=Yii::$app->params['userDataPath'] . explode('@',$user)[0] . '/' . $isystemMountField; - } - if (!empty($ocontMount)) - { - $osystemMount=Yii::$app->params['userDataPath'] . explode('@',$user)[0] . '/' . $osystemMountField; - - } - } + $username=User::getCurrentUser()['username']; + $homeFolder=Yii::$app->params['userDataPath'] . explode('@',$user)[0] . '/'; - $mountpointExistError=false; - - - /* - * Check if i/o folders exist. If not, create them. Depends on whether there is one mountpoint for the - */ - - if (!empty($iocontMount)) + if (!is_dir($homeFolder)) { - if (!is_dir($iosystemMount)) - { - Software::exec_log("mkdir -p $iosystemMount"); - Software::exec_log("chmod 777 $iosystemMount"); - } - else - { - Software::exec_log("chmod 777 $iosystemMount -R"); - } - - } - else - { - if (!empty($icontMount)) - { - if (!is_dir($isystemMount)) - { - Software::exec_log("mkdir -p $isystemMount"); - Software::exec_log("chmod 777 $isystemMount"); - } - else - { - Software::exec_log("chmod 777 $isystemMount -R"); - } - } - if (!empty($ocontMount)) - { - // print_r($osystemMount); - // exit(0); - if (!is_dir($osystemMount)) - { - Software::exec_log("mkdir -p $osystemMount"); - Software::exec_log("chmod 777 $osystemMount"); - } - else - { - Software::exec_log("chmod 777 $osystemMount -R"); - } - } + Software::exec_log("mkdir $homeFolder"); + Software::exec_log("chmod 777 $homeFolder"); } + $oSystemFolder=$outFolder; + $iSystemFolder=''; + if ($example) { $exampleFolder=Yii::$app->params['userDataPath']. 'examples/' . $name . '/' . $version . '/input'; - if (!empty($iosystemMount)) - { - $folder=$iosystemMount; - } - else - { - $folder=$isystemMount; - } - Software::exec_log("cp -r $exampleFolder/* $folder"); - Software::exec_log("chmod 777 $folder"); + $ifolder=$homeFolder . 'examples/' . $name . '/' . $version . '/input/'; + $ofolder=$homeFolder . 'examples/' . $name . '/' . $version . '/output'; + $iSystemFolder='examples/' . $name . '/' . $version . '/input/'; + $oSystemFolder='examples/' . $name . '/' . $version . '/output/'; + $outFolder=$oSystemFolder; - } + Software::exec_log("mkdir -p $ifolder $ofolder"); + Software::exec_log("chmod 777 $ifolder"); + Software::exec_log("chmod 777 $ofolder"); + Software::exec_log("cp -r $exampleFolder/* $ifolder"); - /* - * If pod is already running, get its ID - * by using the jobid and the name of the software - */ - - $podid=Software::runningPodIdByJob($name,$jobid); - // print_r($podid); - // exit(0); - + + } /* * Add parameters for the active form @@ -408,8 +273,6 @@ public function actionRun($name, $version) $fields=SoftwareInput::find()->where(['softwareid'=>$software->id])->orderBy(['position'=> SORT_ASC])->all(); - // print_r($fields); - // exit(0); /* * If the form has posted load the field values. * This was changed because it didn't work with checkboxes. @@ -426,13 +289,11 @@ public function actionRun($name, $version) $superadmin=(User::hasRole("Admin", $superAdminAllowed = true)) ? 1 : 0; - $hasExample=$software->has_example; - $uploadedBy=$software->uploaded_by; /* * If the software uses the reference data stored in the shared folder * mount the shared folder to the pod */ - $sharedFolder=($software->shared)? Yii::$app->params['sharedDataFolder']:''; + // $sharedFolder=($software->shared)? Yii::$app->params['sharedDataFolder']:''; /* * If the software uses gpus, pass the appropriate argument for execution * in the limits section @@ -449,10 +310,17 @@ public function actionRun($name, $version) $field_count=0; } + /* + * $emptyFields is a variable that denotes whether all fields are empty. + */ $emptyFields=true; for ($index=0; $index<$field_count; $index++) { + /* + * If the example is used, the fields values are not empty + * and values from the example are used + */ if ($example) { $emptyFields=false; @@ -468,6 +336,13 @@ public function actionRun($name, $version) } else { + /* + * If the field values are empty, assign + * empty values to each field. Boolean fields + * get a false + * + * $emptyFields remains true + */ if (empty($field_values)) { if ($fields[$index]->field_type=='boolean') @@ -480,13 +355,15 @@ public function actionRun($name, $version) } } + /* + * Add the value to each respective field. + * For boolean values, strings are substituted + */ else { $emptyFields=false; if ($fields[$index]->field_type=='boolean') { - // print_r($field_values[$index]); - // print_r("
"); $fields[$index]->value=($field_values[$index]=="0") ? false : true; } else @@ -499,6 +376,10 @@ public function actionRun($name, $version) $container_command=''; $errors=[]; + /* + * If no inputs have been specified, + * the script is run without additional arguments + */ if (empty($fields)) { /* @@ -506,31 +387,29 @@ public function actionRun($name, $version) */ if (Yii::$app->request->getIsPost()) { - $script=$software->script; - $imountpoint=$icontMount; - $omountpoint=$ocontMount; - $iomountpoint=$iocontMount; - $container_command=$script; + $container_command=$software->script; } } else { + /* + * If values were provided for each field, + * create the command. + */ if (!$emptyFields) { $script=$software->script; - $imountpoint=$icontMount; - $omountpoint=$ocontMount; - $iomountpoint=$iocontMount; - $command_errors=Software::createCommand($script,$emptyFields,$fields,$imountpoint); + $ios=Software::getIOs($software,$fields,$iSystemFolder,$oSystemFolder); + $inputs=$ios[0]; + $output=$ios[1]; + $command_errors=Software::createCommand($software,$emptyFields,$fields); $errors=$command_errors[0]; $container_command=$command_errors[1]; - // print_r($container_command); - // exit(0); } else { /* - * Check if form has posted + * Check if form has posted and post an error */ if (Yii::$app->request->getIsPost()) { @@ -544,8 +423,7 @@ public function actionRun($name, $version) * Get the user-data folder of the current user. * If it does not exist, then create it. */ - $userFolder=Yii::$app->params['userDataPath'] . explode('@',User::getCurrentUser()['username'])[0]; - $user=User::getCurrentUser()['username']; + /** * Get the available jobs by getting the quota from eg-ci and counting the jobs @@ -555,8 +433,7 @@ public function actionRun($name, $version) if(empty($quotas) && (Yii::$app->params['standalone']==false)) - { - + { return $this->render('project_error',['project'=>$project]); } $quotas=$quotas[0]; @@ -576,13 +453,7 @@ public function actionRun($name, $version) ->count(); - if (!is_dir($userFolder)) - { - Software::exec_log("mkdir $userFolder"); - Software::exec_log("chmod 777 $userFolder"); - } - $runError=''; - $runPodId=''; + $runErrors=''; /** * Get the maximum value for memory an cpu from the project if: @@ -598,63 +469,51 @@ public function actionRun($name, $version) /* * Check if the pod for the current job ID is already running */ - $podRunning=Software::isAlreadyRunning($podid); - + $jobAlreadyRunning=Software::isAlreadyRunning($jobid); + $limits=['gpu'=>$gpu, 'cpu'=>$maxCores,'ram'=>$maxMem,'gpu'=>$gpu]; /* * If the form has posted, it is not empty and the pod is not already running, * run the command for the specific docker image. */ - if ((empty($errors)) && (!empty($container_command)) && ($podRunning==false) ) + if ( (empty($errors)) && (!empty($container_command)) && (!$jobAlreadyRunning) ) { - // print_r($οsystemMount); - // exit(0); - $jobid=uniqid(); - $result=Software::createAndRunJob($container_command, $fields, - $name, $version, $jobid, $user, - $podid, $machineType, - $isystemMount, $isystemMountField, - $osystemMount, $osystemMountField, - $iosystemMount, $iosystemMountField, - $project,$maxMem,$maxCores,$sharedFolder,$gpu); - - $runPodId=$result[0]; - $runError=$result[1]; - $machineType=$result[2]; + $software->container_command=$container_command; + $software->limits=$limits; + $software->inputs=$inputs; + $software->outputs=$output; + $software->user=$user; + $software->project=$project; + $software->fields=$fields; + $software->outFolder=$outFolder; + + /* + * Send the job to TES + */ + $software->runJob(); + + $jobid=$software->jobid; + $runErrors=$software->errors; } /* * If pod started without errors, then send its ID to the form. */ - if ($runPodId!='') - - { - $podid=$runPodId; - } - - - $type=1; - return $this->render('run', ['form_params'=>$form_params, 'name'=>$name, - 'version'=>$version, 'jobid'=>$jobid, 'software_id'=>$software_id, 'software_instructions'=>$software_instructions, - 'errors'=>$errors, 'runErrors'=>$runError, 'podid'=>$podid, 'machineType'=>$machineType, - 'fields'=>$fields,'isystemMount' => $isystemMountField, 'osystemMount' => $osystemMountField, - 'iosystemMount' => $iosystemMountField, 'example' => '0', 'hasExample'=>$hasExample, - 'username'=>$user,'icontMount'=>$icontMount,'ocontMount'=>$ocontMount, - 'iocontMount'=>$iocontMount,'mountExistError'=>false, - 'superadmin'=>$superadmin,'uploadedBy'=>$uploadedBy,'jobUsage'=>$jobUsage,'quotas'=>$quotas, - 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'project'=>$project, 'type'=>$type, 'outFolder'=>$outFolder]); + return $this->render('run', ['form_params'=>$form_params, 'software'=>$software, 'example' => '0', + 'errors'=>$errors, 'runErrors'=>$runErrors, + 'fields'=>$fields, 'username'=>$username, 'superadmin'=>$superadmin,'quotas'=>$quotas, + 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'type'=>1, 'outFolder'=>$outFolder]); } /* * Get logs from the running pod to show on the webpage */ - public function actionGetLogs($podid,$machineType,$jobid) + public function actionGetLogs($jobid) { - - $results=Software::getLogs($podid); + $results=Software::getLogs($jobid); $logs=$results[1]; $status=$results[0]; $time=$results[2]; @@ -662,18 +521,14 @@ public function actionGetLogs($podid,$machineType,$jobid) - return $this->renderPartial('logs',['logs'=>$logs, 'status'=>$status, 'machineType'=>$machineType,'time'=>$time,'project'=>$history->project]); + return $this->renderPartial('logs',['logs'=>$logs, 'status'=>$status, 'time'=>$time,'project'=>$history->project]); } - /* - * Clean up job and delete pod from the system + * Cancel job */ - public function actionCleanUp($name,$jobid,$status) + public function actionCancelJob($jobid) { - $model=new Software; - - $results=$model::cleanUp($name,$jobid,$status); - + Software::cancelJob($jobid); } /* @@ -807,7 +662,6 @@ public function actionUploadExisting() /* * Get the files and fields */ - // $model->imageFile=UploadedFile::getInstance($model, 'imageFile'); $model->cwlFile=UploadedFile::getInstance($model, 'cwlFile'); $model->dois=implode('|',$dois); /* @@ -1019,20 +873,9 @@ public function actionHistory($crate_id='') */ $user=User::getCurrentUser()['username']; - - // $inactiveJobs=Software::getInactiveJobs(); - // print_r($inactiveJobs); - // exit(0); - - // foreach ($inactiveJobs as $job) - // { - // Software::cleanUp($job[0],$job[1],'Complete'); - - // } $available=Software::getAvailableSoftware(); $available_workflows=Workflow::getAvailableWorkflows(); - // $results=Software::getUserHistory($user); $query=RunHistory::find()->where(['username'=>$user]) ->andWhere(['mpi_proc'=>null]) ->orderBy(['start'=>SORT_DESC]); @@ -1068,20 +911,14 @@ public function actionRerun($jobid) * or assign default values */ - $podid=''; - $machineType=''; $name=$history->softname; $version=$history->softversion; $project=$history->project; - $isystemMountField=$history->imountpoint; - $osystemMountField=$history->omountpoint; - $iosystemMountField=$history->iomountpoint; + $outFolder=$history->omountpoint; $maxMem=$history->max_ram; $maxCores=$history->max_cpu; - $commands=''; $software=Software::find()->where(['name'=>$name,'version'=>$version])->one(); - $software_instructions=$software->instructions; $uploadedBy=$software->uploaded_by; $fields=SoftwareInput::find()->where(['softwareid'=>$software->id])->orderBy(['position'=> SORT_ASC])->all(); @@ -1094,38 +931,6 @@ public function actionRerun($jobid) { return $this->render('cwl_changed'); } - - - $icontMount=$software->imountpoint; - $ocontMount=$software->omountpoint; - $iocontMount=''; - if ((!empty($icontMount)) && (!empty($ocontMount))) - { - if ($icontMount==$ocontMount) - { - $iocontMount=$icontMount; - } - } - - $mountExistError=false; - if (!empty($iosystemMountField)) - { - $folder=Yii::$app->params['userDataPath'] . explode('@',User::getCurrentUser()['username'])[0] . '/' .$iosystemMountField; - if (!is_dir($folder)) - { - $mountExistError=true; - } - } - else - { - $ifolder=Yii::$app->params['userDataPath'] . explode('@',User::getCurrentUser()['username'])[0] . '/' .$isystemMountField; - $ofolder=Yii::$app->params['userDataPath'] . explode('@',User::getCurrentUser()['username'])[0] . '/' .$osystemMountField; - - if ( (!is_dir($ifolder)) || (!is_dir($ofolder)) ) - { - $mountExistError=true; - } - } /* @@ -1143,7 +948,6 @@ public function actionRerun($jobid) ]; - $hasExample=$software->has_example; $username=User::getCurrentUser()['username']; $superadmin=(User::hasRole("Admin", $superAdminAllowed = true)) ? 1 : 0; $quotas=Software::getOndemandProjectQuotas($username,$project); @@ -1169,22 +973,15 @@ public function actionRerun($jobid) ->count(); $type=1; - return $this->render('run', ['form_params'=>$form_params, 'name'=>$name, - 'version'=>$version, 'jobid'=>'', - 'errors'=>'', 'runErrors'=>'', 'podid'=>'', 'machineType'=>'', - 'fields'=>$fields,'isystemMount' => $isystemMountField, 'osystemMount' => $osystemMountField, - 'iosystemMount' => $iosystemMountField, 'example' => '0', - 'hasExample'=>$hasExample, 'superadmin'=>$superadmin, - 'username'=>$username,'icontMount'=>$icontMount,'ocontMount'=>$ocontMount, - 'iocontMount'=>$iocontMount,'mountExistError'=>$mountExistError, - 'jobUsage'=>$jobUsage,'quotas'=>$quotas, - 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'project'=>$project, 'software_instructions'=>$software_instructions, 'type'=>$type, 'uploadedBy'=>$uploadedBy]); + return $this->render('run', ['form_params'=>$form_params, 'software'=>$software, 'example' => '0', + 'errors'=>'', 'runErrors'=>'', + 'fields'=>$fields, 'username'=>$username, 'superadmin'=>$superadmin,'quotas'=>$quotas, + 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'type'=>1, 'outFolder'=>$outFolder]); } public function actionReattach($jobid) { - // $result=$softwareModel::getRerunData($jobid); $history=RunHistory::find()->where(['jobid'=>$jobid])->one(); if (empty($history)) @@ -1199,27 +996,10 @@ public function actionReattach($jobid) $version=$history->softversion; $maxMem=$history->max_ram; $maxCores=$history->max_cpu; - $isystemMountField=$history->imountpoint; - $osystemMountField=$history->omountpoint; - $iosystemMountField=$history->iomountpoint; - $software=Software::find()->where(['name'=>$name])->andWhere(['version'=>$version])->one(); - $software_instructions=$software->instructions; - $uploadedBy=$software->uploaded_by; - - - $podid=Software::runningPodIdByJob($name,$jobid); + $outFolder=$history->omountpoint; $software=Software::find()->where(['name'=>$name,'version'=>$version])->one(); - $icontMount=$software->imountpoint; - $ocontMount=$software->omountpoint; - $iocontMount=''; - if ((!empty($icontMount)) && (!empty($ocontMount))) - { - if ($icontMount==$ocontMount) - { - $iocontMount=$icontMount; - } - } - + $software->jobid=$jobid; + $uploadedBy=$software->uploaded_by; /* * Add parameters for the active form @@ -1241,7 +1021,6 @@ public function actionReattach($jobid) */ $fields=Software::getRerunFieldValues($jobid,$fields); - $hasExample=$software->has_example; $username=User::getCurrentUser()['username']; $superadmin=(User::hasRole("Admin", $superAdminAllowed = true)) ? 1 : 0; @@ -1268,17 +1047,10 @@ public function actionReattach($jobid) ->count(); $type=1; - - return $this->render('run', ['form_params'=>$form_params, 'name'=>$name, - 'version'=>$version, 'jobid'=>$jobid, - 'errors'=>'', 'runErrors'=>'', 'podid'=>$podid, 'machineType'=>$machineType, - 'fields'=>$fields,'isystemMount' => $isystemMountField, 'osystemMount' => $osystemMountField, - 'iosystemMount' => $iosystemMountField, 'example' => '0', - 'hasExample'=>$hasExample, 'superadmin'=>$superadmin, - 'username'=>$username,'icontMount'=>$icontMount,'ocontMount'=>$ocontMount,'iocontMount'=>$iocontMount, - 'mountExistError'=>false, - 'jobUsage'=>$jobUsage,'quotas'=>$quotas, - 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'project'=>$project, 'software_instructions'=>$software_instructions,'type'=>$type, 'uploadedBy'=>$uploadedBy]); + return $this->render('run', ['form_params'=>$form_params, 'software'=>$software, 'example' => '0', + 'errors'=>'', 'runErrors'=>'', + 'fields'=>$fields, 'username'=>$username, 'superadmin'=>$superadmin,'quotas'=>$quotas, + 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'type'=>1, 'outFolder'=>$outFolder]); } diff --git a/controllers/WorkflowController.php b/controllers/WorkflowController.php index 2209201..d096988 100644 --- a/controllers/WorkflowController.php +++ b/controllers/WorkflowController.php @@ -222,8 +222,6 @@ public function actionRun($name, $version, $project) $jobid=isset($_POST['jobid']) ? $_POST['jobid'] : ''; $example=(isset($_POST['example']) && ($_POST['example']=="1")) ? true : false; $outFolder=(isset($_POST['outFolder'])) ? $_POST['outFolder'] : ''; - // print_r($outFolder); - // exit(0); if ($example) @@ -378,8 +376,6 @@ public function actionRun($name, $version, $project) $fields[$index]->dropdownValues[$item]=$item; } $fields[$index]->dropdownSelected=$fields[$index]->default_value; - // print_r($fields[$index]->dropdownValues); - // exit(0); } else { @@ -392,13 +388,9 @@ public function actionRun($name, $version, $project) */ else { - // print_r($field_values); - // exit(0); $emptyFields=false; if ($fields[$index]->field_type=='boolean') { - // print_r($field_values[$index]); - // print_r("
"); $fields[$index]->value=($field_values[$index]=="0") ? false : true; } else if ($fields[$index]->field_type=='enum') @@ -500,17 +492,13 @@ public function actionRun($name, $version, $project) $maxMem=(!isset($_POST['memory'])) || (empty($_POST['memory'])) || floatval($_POST['memory'])<=0 || (floatval($_POST['memory']) > floatval($quotas['ram'])) ? floatval($quotas['ram']) : floatval($_POST['memory']); $maxCores=(!isset($_POST['cores'])) || (empty($_POST['cores'])) || floatval($_POST['cores'])<=0 || (floatval($_POST['cores']) > floatval($quotas['cores'])) ? floatval($quotas['cores']) : floatval($_POST['cores']); - /* - * Check if the workflow for the current ID is already running - */ - $workflowRunning=Software::isAlreadyRunning($jobid); - - + $workflowAlreadyRunning=Workflow::isAlreadyRunning($jobid); + /* * If the form has posted, it is not empty and the pod is not already running, * run the command for the specific docker image. */ - if ((empty($errors)) && ($workflowRunning==false) && (!empty($workflowParams))) + if ((empty($errors)) && (!$workflowAlreadyRunning) && (!empty($workflowParams))) { /* * reset jobid because it might be filled from a previous submission @@ -553,13 +541,8 @@ public function actionGetLogs($jobid) $time=$results[0]; $history=RunHistory::find()->where(['jobid'=>$jobid])->one(); $project=$history->project; - // print_r($history->project); - // exit(0); - - //Workflow::checkStatus($status, $history) return $this->renderPartial('logs',['taskLogs'=>$taskLogs, 'status'=>$status, 'time'=>$time, 'project'=>$project]); - // return $this->render('logs',['taskLogs'=>$taskLogs, 'status'=>$status, 'time'=>$time]); } /* @@ -607,10 +590,6 @@ public function actionCleanUp($name,$jobid,$status) return; } - $model=new Software; - - $results=$model::cleanUp($name,$jobid,$status); - } /* @@ -1114,8 +1093,14 @@ public function actionReattach($jobid) * or assign default values */ + $errors=[]; $results=Workflow::getLogs($jobid); - Workflow::checkStatus($results[1], $history); + if (isset($results['error'])) + { + $jobid=$results['jobid']; + $errors[]=$results['error']; + } + // Workflow::checkStatus($results[1], $history); $name=$history->softname; $version=$history->softversion; @@ -1134,8 +1119,6 @@ public function actionReattach($jobid) */ $fields=Workflow::getRerunFieldValues($jobid,$fields); - - // $mountExistError=false; if (!empty($outFolder)) { $folder=Yii::$app->params['userDataPath'] . explode('@',User::getCurrentUser()['username'])[0] . '/' .$outFolder; @@ -1149,9 +1132,6 @@ public function actionReattach($jobid) Software::exec_log($command,$out,$ret); } } - - // print_r($outFolder); - // exit(0); /* * Add parameters for the active form @@ -1198,7 +1178,7 @@ public function actionReattach($jobid) return $this->render('run', ['form_params'=>$form_params, 'name'=>$name, 'version'=>$version, 'jobid'=>$jobid, - 'errors'=>'', 'runErrors'=>'','fields'=>$fields, + 'errors'=>$errors, 'runErrors'=>'','fields'=>$fields, 'example' => '0', 'hasExample'=>$hasExample, 'username'=>$username,'superadmin'=>$superadmin,'uploadedBy'=>$uploadedBy,'jobUsage'=>$jobUsage,'quotas'=>$quotas, 'maxMem'=>$maxMem, 'maxCores'=>$maxCores, 'project'=>$project,'outFolder' => $outFolder, 'workflow_instructions'=>$workflow_instructions, 'type'=>$type, 'visualize'=>$visualize]); @@ -1213,10 +1193,6 @@ public function actionDownloadFiles($name,$version) $filepath=$workflow->original_file; $fileexploded=explode('/',$filepath); $filename=end($fileexploded); - // print_r($filename); - // print_r("
"); - // print_r($filepath); - // exit(0); return Yii::$app->response->sendFile($filepath,$filename); } @@ -1264,38 +1240,18 @@ public function actionAddExample($name,$version) $valueErrors=[]; if (Yii::$app->request->getIsPost()) { - // print_r($_FILES); - // exit(0); $folder=Yii::$app->params['userDataPath'] . '/workflow_examples/' . $name . '/' . $version . '/input/'; $fieldPath='workflow_examples/' . $name . '/' . $version . '/input/'; - // print_r($folder); - // exit(0); + $command="mkdir -p $folder"; Software::exec_log($command,$output,$ret); $command="chmod 777 $folder"; Software::exec_log($command,$output,$ret); - // $i=0; - // $j=0; - // $k=0; + $values=[]; - // print_r($_FILES); - // print_r("
"); - // print_r($_POST); - // exit(0); - // $fieldValues=isset($_POST['field_values']) ? $_POST['field_values'] : []; - // print_r($_FILES['field_values']); - // exit(0); - // $fileNames=$_FILES['field_values']['name']; - // $tmpFiles=$_FILES['field_values']['tmp_name']; - // print_r($_POST); - // exit(0); - // print_r($fileNames); - // print_r("
"); - // print_r($tmpFiles); - // exit(0); if (!empty($fields)) { @@ -1430,144 +1386,6 @@ public function actionAddExample($name,$version) return $this->render('add_example',[ 'fields' => $fields, 'name'=>$name, 'version'=>$version, 'valueErrors'=>$valueErrors]); } - - - - public function actionJobDetails($jobid) - { - - // $softwareModel=new Software; - $result=Software::getJobDetails($jobid); - if (empty($result)) - { - return $this->render('job_not_found',['jobid'=>$jobid]); - } - - $machineType=$result['machinetype']; - $status=empty($result['status']) ? 'Running' : $result['status']; - $start=date("F j, Y, H:i:s",strtotime($result['start'])); - $stop=empty($result['stop']) ? 'Not available yet' : date("F j, Y, H:i:s",strtotime($result['stop'])); - $isystemMountField=$result['imountpoint']; - $osystemMountField=$result['omountpoint']; - $iosystemMountField=$result['iomountpoint']; - if(empty($result['stop'])) - { - $totalExecutionTime='Not available yet'; - } - else - { - $totalExecutionTime=date("H:i:s",(strtotime($stop)-strtotime($start))); - } - - $maxRam=empty($result['ram']) ? 'Not available' : $result['ram']; - $maxCpu=empty($result['cpu']) ? 'Not available' : $result['cpu']/1000; - - - // rint_r(strtotime($stop)); - // print_r(""); - // print_r(strtotime($start)); - - - // $rows=[ - // 'Software name' => $result['softname'], - // 'Software version' => $result['softversion'], - // 'Status'=> $status, - // 'Started on'=>$start, - // 'Stopped on' => $stop, - // 'Total execution time' => $totalExecutionTime, - // 'Maximum memory footprint (GB)' => $maxRam, - // 'Maximum CPU load (cores)' => $maxCpu, - // 'Machine Type' => $machineType, - // ]; - - // if (!empty($iosystemMountField)) - // { - // $rows['I/O Mountpoint']=$iosystemMountField; - // } - // else - // { - // if (!empty($isystemMountField)) - // { - // $rows['Input Mountpoint']=$isystemMountField; - // } - - // if (!empty($osystemMountField)) - // { - // $rows['Output Mountpoint']=$osystemMountField; - // } - // } - - return $this->render('job_details',['name'=>$result['softname'],'version'=>$result['softversion'], - 'status'=>$status, 'start'=>$start, 'stop'=>$stop, 'execTime'=>$totalExecutionTime, - 'ram'=>$maxRam,'cpu'=>$maxCpu,'machineType' =>$machineType, - 'iomount'=>$iosystemMountField, 'imount'=>$isystemMountField, 'omount'=>$osystemMountField, - 'jobid'=>$jobid, - ]); - - - } - - - public function actionUserStatistics() - { - - $softwareModel=new Software; - $user=User::getCurrentUser()['username']; - $projectTotals=$softwareModel::getUserStatistics($user); - $projectAggr=$softwareModel::getUserStatisticsPerProject($user); - $quotas=Software::getActiveProjectQuotas($user); - // $tim=str_replace(":", "," , $result[2]); - // $time=str_replace("," , "." , $result[2]); - - - - - // print_r($result); - // exit(0); - - // $rows=[ - // 'Number of jobs'=>$result[0], - // 'Number of completed jobs'=>$result[1], - // // 'Started on'=>$start, - // // 'Stopped on' => $stop, - // 'Total execution time (in hours)' => $result[2], - // 'Average memory footprint (GB)' => $result[3], - // 'Average CPU load (cores)' => round($result[4])/1000, - - // ]; - - - - return $this->render('user_statistics',['projectTotals'=>$projectTotals,'projectAggr'=>$projectAggr, 'quotas'=>$quotas]); - - - } - - public function actionImageDescription($name,$version) - { - $model=Software::find()->where(['name'=>$name, 'version'=>$version])->one(); - - return $this->renderAjax('image_description',['model'=>$model]); - } - - public function actionDownloadLogs($jobid) - { - $filepath=Yii::$app->params['tmpFolderPath'] . $jobid . '/logs.txt'; - $filename='logs.txt'; - // print_r($filename); - // print_r("
"); - // print_r($filepath); - // exit(0); - if (file_exists($filepath)) - { - return Yii::$app->response->sendFile($filepath,$filename); - } - else - { - return false; - } - - } } diff --git a/models/Software.php b/models/Software.php index 204c0d2..72e6d62 100644 --- a/models/Software.php +++ b/models/Software.php @@ -54,7 +54,17 @@ class Software extends \yii\db\ActiveRecord { - /** + public $container_command=[]; + public $fields=[]; + public $inputs=[]; + public $outputs=[]; + public $limits=[]; + public $project=''; + public $user=''; + public $errors=[]; + public $jobid=''; + public $outFolder=''; + /* * {@inheritdoc} */ public static function tableName() @@ -116,8 +126,6 @@ public static function getSoftwareNames($softUser) $query->where(['visibility'=>'public']) ->orWhere(['and',['visibility'=>'private','uploaded_by'=>$softUser]]); } - // echo $query->createCommand()->getRawSql(); - // exit(0); $rows=$query->all(); $results=[]; @@ -148,16 +156,11 @@ public static function getSoftwareNames($softUser) } } - // $results[$name][$uploader][$version[0]]=$visibility; } - - // print_r($results); - // exit(0); return $results; - // return $rows; } @@ -281,8 +284,8 @@ public static function getSoftwareDescriptions($softUser) $query->where(['visibility'=>'public']) ->orWhere(['and',['visibility'=>'private','uploaded_by'=>$softUser]]); } - // echo $query->createCommand()->getRawSql(); - // exit(0); + + $rows=$query->all(); $results=[]; @@ -292,16 +295,11 @@ public static function getSoftwareDescriptions($softUser) $version=$row['version']; $description=$row['description']; $results[]=['name'=>$name, 'version'=>$version, 'description'=>$description]; - // $results[$name][$uploader][$version[0]]=$visibility; } - // print_r($results); - // exit(0); - return $results; - // return $rows; } public static function getAvailableSoftware() @@ -311,8 +309,6 @@ public static function getAvailableSoftware() $query->select('id,name,version,mpi') ->where(['mpi'=>false]) ->from('software'); - // echo $query->createCommand()->getRawSql(); - // exit(0); $rows=$query->all(); $results=[]; @@ -330,7 +326,6 @@ public static function getAvailableSoftware() } return $results; - // return $rows; } public static function getIndicators($softUser) @@ -420,55 +415,48 @@ public static function getSoftwarePreviousCwl($name, $version) - public static function getContainerMountpoint($name, $version) - { - $query=new Query; + // public static function getContainerMountpoint($name, $version) + // { + // $query=new Query; - $query->select(['imountpoint','omountpoint']) - ->from('software ') - ->where(['name'=>$name, 'version'=>$version]); - $rows=$query->one(); + // $query->select(['imountpoint','omountpoint']) + // ->from('software ') + // ->where(['name'=>$name, 'version'=>$version]); + // $rows=$query->one(); - $imount=$rows['imountpoint']; - $omount=$rows['omountpoint']; + // $imount=$rows['imountpoint']; + // $omount=$rows['omountpoint']; - $iomount=''; + // $iomount=''; - if ((!empty($imount)) && (!empty($omount))) - { - if ($imount==$omount) - { - $iomount=$imount; - } - } + // if ((!empty($imount)) && (!empty($omount))) + // { + // if ($imount==$omount) + // { + // $iomount=$imount; + // } + // } - // print_r($iomount); - // print_r("
"); - // print_r($imount); - // print_r("
"); - // print_r($omount); - // print_r("
"); - // exit(0); - return [$imount, $omount, $iomount]; - } + // return [$imount, $omount, $iomount]; + // } - public static function getUserHistory($softUser) - { - $query=new Query; + // public static function getUserHistory($softUser) + // { + // $query=new Query; - $query->select(['start','stop','command','status','softname', 'softversion','jobid', 'ram', 'cpu', 'machinetype', 'project','software_id']) - ->from('run_history') - ->where(['username'=>$softUser]); + // $query->select(['start','stop','command','status','softname', 'softversion','jobid', 'ram', 'cpu', 'machinetype', 'project','software_id']) + // ->from('run_history') + // ->where(['username'=>$softUser]); - $pages = new Pagination(['totalCount' => $query->count()]); - $pages->setPageSize(10); + // $pages = new Pagination(['totalCount' => $query->count()]); + // $pages->setPageSize(10); - $results = $query->orderBy('start DESC')->offset($pages->offset)->limit($pages->limit)->all(); + // $results = $query->orderBy('start DESC')->offset($pages->offset)->limit($pages->limit)->all(); - return [$pages,$results]; - } + // return [$pages,$results]; + // } @@ -476,62 +464,113 @@ public static function getUserHistory($softUser) * If the user has uploaded a CWL file, * then get the custom form fields. */ - public static function getSoftwareFields($name,$version) - { - $query=new Query; - - $query->select('si.name, si.position, si.field_type, si.prefix, si.default_value, si.optional, si.separate si.example') - ->from('software sf') - ->join('INNER JOIN', 'software_inputs si', 'si.softwareid=sf.id') - ->where(['sf.name'=>$name, 'sf.version'=>$version]) - ->orderBY(['si.position'=>SORT_ASC]); - $rows=$query->all(); - - return $rows; - } + // public static function getSoftwareFields($name,$version) + // { + // $query=new Query; + + // $query->select('si.name, si.position, si.field_type, si.prefix, si.default_value, si.optional, si.separate si.example') + // ->from('software sf') + // ->join('INNER JOIN', 'software_inputs si', 'si.softwareid=sf.id') + // ->where(['sf.name'=>$name, 'sf.version'=>$version]) + // ->orderBY(['si.position'=>SORT_ASC]); + // $rows=$query->all(); + + // return $rows; + // } /* * If the user has uploaded a CWL file, * then get the docker image script name. */ - public static function getScript($name,$version) + // public static function getScript($name,$version) + // { + // $query=new Query; + + // $query->select('script, imountpoint,omountpoint') + // ->from('software') + // ->where(['name'=>$name, 'version'=>$version]); + // $path=$query->one(); + + // $imount=$path['imountpoint']; + // $omount=$path['omountpoint']; + // $iomount=''; + // if ((!empty($imount)) || (empty($omount))) + // { + // if ($imount==$omount) + // { + // $iomount=$imount; + // } + // } + + // return [$path['script'],$imount,$omount,$iomount]; + // } + public static function getIOs($software,$fields,$iSystemFolder,$oSystemFolder) { - $query=new Query; + $userFolder=Yii::$app->params['userDataPath'] . explode('@',User::getCurrentUser()['username'])[0]; + $ofolder=$userFolder . '/' . $oSystemFolder; + if (!is_dir($ofolder)) + { + self::exec_log("mkdir -p $ofolder"); + self::exec_log("chmod 777 $ofolder"); + } + /* + * We installed schema on a server instead of deploying with helm + * and our FTP is jailed, with the root being the user-data folder. + * Change does not affect helm deployments, because the parameter does not exist. + */ + if (isset(Yii::$app->params['ftpJailPath']) and (!empty(Yii::$app->params['ftpJailPath']))) + { + $userFolder=str_replace(Yii::$app->params['ftpJailPath'],'',$userFolder); + } - $query->select('script, imountpoint,omountpoint') - ->from('software') - ->where(['name'=>$name, 'version'=>$version]); - $path=$query->one(); + /* + * Prepare output directory location. + */ - $imount=$path['imountpoint']; - $omount=$path['omountpoint']; - $iomount=''; - if ((!empty($imount)) || (empty($omount))) + $url= "ftp://" . Yii::$app->params['ftpIp'] . '//' . $userFolder . '/' . $oSystemFolder; + $outputs=[['type'=>'DIRECTORY','path'=>$software->omountpoint,'url'=>$url]]; + $inputs=[]; + foreach ($fields as $field) { - if ($imount==$omount) + if (($field->field_type!='Directory') && ($field->field_type!='File')) { - $iomount=$imount; + continue; + } + $input=[]; + if ($field->field_type=='Directory') + { + $url="ftp://" . Yii::$app->params['ftpIp'] . '//' . $userFolder . '/' . $iSystemFolder . $field->value; + $path=$software->imountpoint; + $type='DIRECTORY'; } + else + { + $url="ftp://" . Yii::$app->params['ftpIp'] . '//' . $userFolder . '/' . $iSystemFolder . $field->value; + $filename=explode('/',$field->value); + $filename=end($filename); + $path=$software->imountpoint . '/' . $filename; + $type='FILE'; + } + $inputs[]=['type'=>$type,'path'=>$path,'url'=>$url]; } - return [$path['script'],$imount,$omount,$iomount]; + return [$inputs,$outputs]; } - /* * Check if the commands posted are empty. */ - public static function createCommand($script,$emptyFields,$fields,$mountpoint) + public static function createCommand($software,$emptyFields,$fields) { $errors=[]; /** * Return one command with the mountpoint attached and one without it */ - $command=$script; + $command=$software->script; if ( (!$emptyFields)) { - // print_r(3); + foreach ($fields as $field) { if ($field->field_type=='boolean') @@ -575,7 +614,7 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) { if (($field->field_type=='File') || ($field->field_type=='Directory')) { - $command.= ' ' . $field->prefix . $field_gap . $mountpoint . '/' . $field->value; + $command.= ' ' . $field->prefix . $field_gap . $software->imountpoint . '/' . $field->value; } else { @@ -590,7 +629,7 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) { if (($field->field_type=='File') || ($field->field_type=='Directory')) { - $command.= ' ' . $field->prefix . $field_gap . $mountpoint . '/' . $field->value; + $command.= ' ' . $field->prefix . $field_gap . $software->imountpoint . '/' . $field->value; } else { @@ -643,7 +682,7 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) { if (($field->field_type=='File') || ($field->field_type=='Directory')) { - $finalValue.= ' ' . $field->prefix . $field_gap . $mountpoint . '/' . $val; + $finalValue.= ' ' . $field->prefix . $field_gap . $software->imountpoint . '/' . $val; } else { @@ -686,7 +725,7 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) { if (($field->field_type=='File') || ($field->field_type=='Directory')) { - $finalValue.= $mountpoint . '/' . $val . $separator; + $finalValue.= $software->imountpoint . '/' . $val . $separator; } else { @@ -731,7 +770,7 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) { if (($field->field_type=='File') || ($field->field_type=='Directory')) { - $finalValue.= ' ' . $field->prefix . $field_gap . $mountpoint . '/' . $field->value; + $finalValue.= ' ' . $field->prefix . $field_gap . $software->imountpoint . '/' . $field->value; } else { @@ -774,7 +813,7 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) { if (($field->field_type=='File') || ($field->field_type=='Directory')) { - $finalValue.= $mountpoint . '/' . $field->value . $separator; + $finalValue.= $software->imountpoint . '/' . $field->value . $separator; } else { @@ -797,59 +836,83 @@ public static function createCommand($script,$emptyFields,$fields,$mountpoint) /* * This function creates the job YAML file and sends it to Kubernetes */ - public static function createAndRunJob($commands, $fields, - $name, $version, $jobid, $user, - $podid, $machineType, - $isystemMount, $isystemMountField, - $osystemMount, $osystemMountField, - $iosystemMount, $iosystemMountField, - $project,$maxMem,$maxCores,$sharedFolder,$gpu) + public function runJob() { - + $data=[]; /* - * Scheduler (python) script physical location + * Create job name. Leftover from the old method, but it works. */ - $scheduler=self::sudoWrap(Yii::$app->params['scriptsFolder'] . "scheduler.py"); - $stats=self::sudoWrap(Yii::$app->params['scriptsFolder'] . "jobMonitor.py"); - - /* - * Get image repository location from the DB + $taskName=str_replace('_','-',$this->name); + $taskName=str_replace(' ','-',$taskName); + $taskName=str_replace("\t",'-',$taskName); + $taskName.= '-' . $this->version; + + /* + * Get container command and eliminate empty strings */ - $software=Software::find()->where(['name'=>$name, 'version'=>$version])->one(); - $image=$software->image; - $workingdir=$software->workingdir; - $imountpoint=$software->imountpoint; - $omountpoint=$software->omountpoint; - $softwareId=$software->id; - - $iomountpoint=''; - if ((!empty($imountpoint)) || (empty($omountpoint))) + $command_str=$this->container_command; + $tmp=explode(' ',$this->container_command); + $this->container_command=[]; + foreach ($tmp as $token) { - if ($imountpoint==$omountpoint) + if ($token=='') { - $iomountpoint=$imountpoint; + continue; } + $this->container_command[]=$token; + } + + /* + * Get data to be sent to TES + */ + $data['name']=$taskName; + $data['inputs']=$this->inputs; + $data['outputs']=$this->outputs; + + /* + * Get resources or limits depending on the type of TES + */ + if (isset(Yii::$app->params['schemaTes']) && (!empty(Yii::$app->params['schemaTes']))) + { + $resources=['cpu_cores'=>$this->limits['cpu'], 'ram_gb'=>$this->limits['ram'],'gpu'=>$this->limits['gpu']]; + $data['limits']=$resources; + $data['resources']=$resources; + } + else + { + $resources=['cpu_cores'=>$this->limits['cpu'], 'ram_gb'=>$this->limits['ram'],'disk_gb'=>'30']; + $data['resources']=$resources; + } + + /* + * Create TES executor + */ + $executor=[]; + $executor['image']=$this->image; + $executor['command']=$this->container_command; + $executor['workdir']=$this->omountpoint; + $data['executors']=[$executor]; + $url=self::getTesUrl(); + $client=new Client(); + $response = $client->createRequest() + ->setFormat(Client::FORMAT_JSON) + ->setData($data) + ->setMethod('POST') + ->setUrl($url) + ->send(); + + if (!$response->getIsOk()) + { + $this->errors=['There was an error sending the job to TESK.
Please contact an administrator']; + return; } - // $nameNoQuotes=$name; - $softName=$name; - $nameNoQuotes=str_replace('_','-',$name); - $versionNoQuotes=$version; - $name=self::enclose($name); - $version=self::enclose($version); - $iomountpoint=self::enclose($iomountpoint); - $imountpoint=self::enclose($imountpoint); - $omountpoint=self::enclose($omountpoint); - $iosystemMount=self::enclose($iosystemMount); - $isystemMount=self::enclose($isystemMount); - $osystemMount=self::enclose($osystemMount); - $workingdir=self::enclose($workingdir); - $sharedFolder=self::enclose($sharedFolder); - $gpu=self::enclose($gpu); + + $this->jobid=$response->data['id']; /* - * Create the tmp folder to store the YAML file + * Create the tmp folder to store the aux files */ - $folder=Yii::$app->params['tmpFolderPath'] . "$jobid/"; + $folder=Yii::$app->params['tmpFolderPath'] . "$this->jobid/"; if (file_exists($folder)) { @@ -865,361 +928,210 @@ public static function createAndRunJob($commands, $fields, * Store inputs in a file in the tmp directory */ $fieldValues=[]; - foreach($fields as $field) + foreach($this->fields as $field) { $fieldValues[$field->name]=$field->value; } - $fieldValues=json_encode($fieldValues,JSON_UNESCAPED_SLASHES); - - $machineType=SoftwareProfiler::getMachineType($fields,$software,$folder,$isystemMount,$iosystemMount,$maxMem); + $fieldValues=json_encode($fieldValues,JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT); - $filename=$folder . 'fields.txt'; + $filename=$folder . 'fields.json'; file_put_contents($filename, $fieldValues); /* - * Store the commands in a file in the tmp directory + * Store the json call in a file in the tmp directory */ - $filename=$folder . 'commands.txt'; - - file_put_contents($filename, $commands); + $filename=$folder . 'tesCall.json'; + $tesCall=json_encode($data,JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT); + file_put_contents($filename, $tesCall); Software::exec_log("chmod 777 $folder -R",$out,$ret); - /* - * Classify software, create YAML job configuration file, - * send the file to Kubernetes to run and get the machine type. + * Insert to DB */ - $arguments=[ - $name,$version,$image, - $jobid,$folder, $workingdir, - $imountpoint, $isystemMount, - $omountpoint, $osystemMount, - $iomountpoint, $iosystemMount, - $maxMem,$maxCores*1000, Yii::$app->params['nfsIp'], - $machineType,$sharedFolder,$gpu]; - - $schedulerCommand=$scheduler . ' ' . implode(' ',$arguments) . " 2>&1"; - - // print_r($schedulerCommand); - // exit(0); - Software::exec_log($schedulerCommand,$output,$ret); + $history=new RunHistory(); + $history->username=User::getCurrentUser()['username']; + $history->jobid=$this->jobid; + $history->command=$command_str; + $history->omountpoint=$this->outFolder; + $history->start='NOW()'; + $history->softname=$this->name; + $history->softversion=$this->version; + $history->project=$this->project; + $history->max_ram=$this->limits['cpu']; + $history->max_cpu=$this->limits['ram']; + $history->software_id=$this->id; + $history->type='job'; + $history->save(false); - // print_r($output); - // exit(0); - $jobName=strtolower($nameNoQuotes) . '-' . $jobid; /* - * This replacements are performed in configFileCreator.py too. - * Anything changed here should be changed there. - * Job cancelling is also affected. + * Run the monitor script in the background */ - $jobName=str_replace('_','-',$jobName); - $jobName=str_replace(' ','-',$jobName); - $jobName=str_replace("\t",'-',$jobName); - $arguments=[$jobName, $jobid, $folder]; - $statsCommand=$stats . ' ' . implode(' ', $arguments); + $monitor=self::sudoWrap(Yii::$app->params['scriptsFolder'] . "jobMonitor.py"); + $jobid=self::enclose($this->jobid); + $enfolder=self::enclose($folder); + $tesEndpoint=self::enclose(Yii::$app->params['teskEndpoint']); + $schemaTes=(isset(Yii::$app->params['schemaTes']) && (!empty(Yii::$app->params['schemaTes'])))?"'1'":"'0'"; + $arguments=[$jobid, $folder,$tesEndpoint,$schemaTes]; + $monitorCommand=$monitor . ' ' . implode(' ', $arguments); $monitorLog=$folder . 'monitorLog.txt'; /* * Uncomment the following line and comment the other one to debug the monitor */ - // shell_exec(sprintf("%s > $monitorLog 2>&1 &", $statsCommand)); - shell_exec(sprintf("%s > /dev/null 2>&1 &", $statsCommand)); - - - /* - * Read the output and check if the pod was created. - * If not, add error message. - */ - if (!preg_match("/job\.batch\/[a-z0-9:space:]+/",$output[0])) - { - $error=""; - foreach ($output as $out) - { - $error.=$out . "
"; - } - $error.="Please contact the system administrator."; - return [$podid, $error, $machineType]; - } - - /* - * Get the pod ID by using the job name. - */ - - - $query=Yii::$app->db->createCommand()->insert('run_history', - [ - - "username"=>User::getCurrentUser()['username'], - "jobid" => $jobid, - "command" => $commands, - "imountpoint" => $isystemMountField, - "omountpoint" => $osystemMountField, - "iomountpoint" => $iosystemMountField, - "start" => 'NOW()', - "softname" => $softName, - "softversion"=> $versionNoQuotes, - "machinetype"=> $machineType, - "project"=>$project, - "max_ram"=> $maxMem, - "max_cpu" => $maxCores, - 'software_id' => $softwareId, - 'type'=>'job', - - ] - )->execute(); - - // print_r($statsCommand); - // exit(0); + shell_exec(sprintf("%s > $monitorLog 2>&1 &", $monitorCommand)); + // shell_exec(sprintf("%s > /dev/null 2>&1 &", $monitorCommand)); - unset($output); - - - if (isset(Yii::$app->params['namespaces']['jobs'])) - { - $namespace=Yii::$app->params['namespaces']['jobs']; - $command=self::sudoWrap("kubectl get pods --no-headers -n $namespace 2>&1") . " | grep $jobName | tr -s ' ' "; - - Software::exec_log($command,$output,$ret); - } - else - { - $command=self::sudoWrap("kubectl get pods --no-headers 2>&1") . " | grep $jobName | tr -s ' ' "; - Software::exec_log($command,$output,$ret); - } - - $podString=''; - foreach ($output as $out) - { - // print_r($out); - // print_r("
"); - if (strpos($out,$jobName)!== false) - { - $podString=explode(' ', $out)[0]; - break; - } - - } - - if (empty($podString)) - { - $error="There was an error submitting the job to Kubernetes.
Please reload the page and try again or contact an administrator."; - return [$podid, $error, $machineType]; - } - return [$podString,'', $machineType]; + return; } /* * Get pod logs by using the pod ID */ - public static function getLogs($podid) + public static function getLogs($jobid) { - if (isset(Yii::$app->params['namespaces']['jobs'])) + $url=self::getTesUrl(); + if (isset(Yii::$app->params['schemaTes']) && (!empty(Yii::$app->params['schemaTes']))) { - $namespace=Yii::$app->params['namespaces']['jobs']; - $logsCommand=self::sudoWrap("kubectl logs $podid -n $namespace 2>&1"); - $command=self::sudoWrap("kubectl get pods --no-headers $podid -n $namespace 2>&1"); + $urlArray=[$url, 'jobid'=>$jobid, 'view'=>'FULL']; } else { - $logsCommand=self::sudoWrap("kubectl logs $podid 2>&1"); - $command=self::sudoWrap("kubectl get pods --no-headers $podid 2>&1"); + $urlArray=[$url . "/$jobid", 'view'=>'FULL']; } - Software::exec_log($logsCommand,$logs,$ret); + $client=new Client(); + $response = $client->createRequest() + ->setFormat(Client::FORMAT_JSON) + ->setMethod('GET') + ->setUrl($urlArray) + ->send(); - Software::exec_log($command,$output,$ret); - $splt=preg_split('/[\s]+/', $output[0]); - $status=$splt[2]; - $time=$splt[4]; - if ($status=='server') + if (!$response->getIsOk()) { - $status='Completed'; - $time='N/A'; + return['COMPLETE',['Job logs unavailable. Please visit the "Job History" page to download the logs.'],"N/A"]; } - - - return [$status,$logs,$time]; - } - /* - * Erase job after it is completed or terminated. - */ - public static function cleanUp($name,$jobid,$status) - { - $folder=$userFolder=Yii::$app->params['tmpFolderPath'] . "/$jobid/"; - - #Clear job - $jobName=strtolower($name). '-' . $jobid; - $jobName=str_replace('_','-',$jobName); - $jobName=str_replace(' ','-',$jobName); - $jobName=str_replace("\t",'-',$jobName); - $yaml=$folder . $jobName . '.yaml'; - - if ($status=='Completed') - { + $status=$response->data['state']; + try + { + $logs=$response->data['logs'][0]['logs'][0]['stdout']; + } + catch (\Exception $e) + { + $logs=""; + } - $command=self::sudoWrap("kubectl describe job $jobName 2>&1"); - Software::exec_log($command, $output, $ret); - // print_r($output); - // exit(0); - $start=''; - $stop=''; - foreach ($output as $line) + $logs=explode("\n",$logs); + try + { + $start=$response->data['logs'][0]['logs'][0]['start_time']; + $start=str_replace('T',' ',$start); + $start=str_replace('Z',' ',$start); + $startdate=new \DateTime($start); + $enddate=new \DateTime('NOW'); + $diff=$enddate->diff($startdate); + + $timestr=$diff->s . "s"; + if ($diff->i>0) { - $startStr="Start Time:"; - $startlen=strlen($startStr); - $stopStr="Completed At:"; - $stoplen=strlen($stopStr); - // $cancelStr="Canceled At:"; - // $cancellen=strlen($cancelStr); - - if (substr($line,0,$startlen)==$startStr) - { - $tokens=explode(',', $line)[1]; - $tokens=explode('+', $tokens)[0]; - $datetime=strtotime($tokens); - $start=date("Y-m-d H:i:s", $datetime); - } - - if (substr($line,0,$stoplen)==$stopStr) - { - $tokens=explode(',', $line)[1]; - $tokens=explode('+', $tokens)[0]; - $datetime=strtotime($tokens); - $stop=date("Y-m-d H:i:s", $datetime); - } - - // if (substr($line,0,$cancellen)==$cancelStr) - // { - // $tokens=explode(',', $line)[1]; - // $tokens=explode('+', $tokens)[0]; - // $datetime=strtotime($tokens); - // $stop=date("Y-m-d H:i:s", $datetime); - // } - + $timestr=$diff->i . 'm ' . $timestr; + } + if ($diff->h>0) + { + $timestr=$diff->h . 'h ' . $timestr; + } + if ($diff->d>0) + { + $timestr=$diff->d . 'd, ' . $timestr; + } + if ($diff->m>0) + { + $timestr=$diff->m . 'm ' . $timestr; + } + if ($diff->y>0) + { + $timestr=$diff->y . 'y ' . $timestr; } - Yii::$app->db->createCommand()->update('run_history', - [ - 'start' => $start, - 'stop' => $stop, - 'status' => $status, - ], - "jobid='$jobid'" - )->execute(); - // print_r($start); - // print_r("
"); - // print_r($stop); - // exit(0); } - else + catch (\Exception $e) { - Yii::$app->db->createCommand()->update('run_history', - [ - 'stop' => 'NOW()', - 'status' => $status, - ], - "jobid='$jobid'" - )->execute(); - } + $timestr="N/A"; - - // print_r( Yii::$app->db->createCommand()->update('run_history', - // [ - // 'start' => $start, - // 'stop' => $stop, - // 'status' => $status, - // ], - // "softname='$name' AND username='$user' AND jobid='$jobid'" - // )->getRawSql()); - - $podid=self::runningPodIdByJob($name,$jobid); - $command=self::sudoWrap("kubectl logs $podid 2>&1"); - Software::exec_log($command,$logs,$ret); - file_put_contents($folder . 'logs.txt', implode("\n",$logs)); - - $command=self::sudoWrap("kubectl delete -f $yaml"); - Software::exec_log($command,$out,$ret); + } + + + return [$status,$logs,$timestr]; } - /* - * Check is podid is running - */ - public static function isAlreadyRunning($podid) + public static function isAlreadyRunning($jobid) { - if ($podid=='') - { - return false; - } - /* - * The following could have been implemented by using the "--no-headers" - * flag and checking whether the output is empty instead of count($output)==1. + * If jobid is empty, the page is loaded for the first time. + * If not, ask TES whether the job exists in the system. + * If the job results in 404 error, then it has run and deleted + * or not has not run at all. */ - $command=self::sudoWrap("kubectl get pods $podid 2>&1"); - Software::exec_log($command,$output,$ret); - if (count($output)==1) + if (empty($jobid)) { return false; } + $url=self::getTesUrl(); + if (isset(Yii::$app->params['schemaTes']) && (!empty(Yii::$app->params['schemaTes']))) + { + $urlArray=[$url, 'jobid'=>$jobid]; + } else + { + $urlArray=[$url . "/$jobid"]; + } + $client=new Client(); + $response = $client->createRequest() + ->setFormat(Client::FORMAT_JSON) + ->setMethod('GET') + ->setUrl($urlArray) + ->send(); + + if ($response->getIsOk()) { return true; } + return false; } - /* - * Use name and job ID to get the pod ID. - */ - public static function runningPodIdByJob($name,$jobid) + public static function cancelJob($jobid) { - $podid=''; - if (isset(Yii::$app->params['namespaces']['jobs'])) + /* + * Cancel job by making the appropriate call + * to the TES api. + */ + $url=self::getTesUrl(); + if (isset(Yii::$app->params['schemaTes']) && (!empty(Yii::$app->params['schemaTes']))) { - $namespace=Yii::$app->params['namespaces']['jobs']; - $command=self::sudoWrap("kubectl get pods -n $namespace --no-headers 2>&1"); + $urlArray=[$url . '/cancel', 'jobid'=>$jobid]; } else { - $command=self::sudoWrap("kubectl get pods --no-headers 2>&1"); - } - - - Software::exec_log($command,$out,$ret); - - foreach ($out as $row) - { - $podTokens=preg_split('/[\s]+/', $row); - - $podString=$podTokens[0]; - $podStatus=$podTokens[2]; - $tokens=explode('-',$podString); - if ((strtolower($name)==$tokens[0]) && ($jobid==$tokens[1])) - { - $podid=$podString; - break; - } + $urlArray=[$url . "/$jobid:cancel"]; } - if ($podStatus=='Terminating') - { - $podid=''; - } - return $podid; - + $client=new Client(); + $response = $client->createRequest() + ->setFormat(Client::FORMAT_JSON) + ->setMethod('POST') + ->setUrl($urlArray) + ->send(); } + public static function listDirectories($directory) { - // $files = array_filter(scandir($directory),'is_dir'); $files = scandir($directory); $results=[]; - // print_r($files); - // exit(0); foreach($files as $key => $value) { @@ -1247,15 +1159,10 @@ public static function softwareRemove($name,$version) $success=''; $error=''; - // print_r($command); - // print_r("

"); - // exit(0); session_write_close(); Software::exec_log($command,$out,$ret); session_start(); - // print_r($out); - // print_r($ret); - // exit(0); + if ($ret==0) { $success="Successfully deleted image $name v.$version!"; @@ -1288,7 +1195,7 @@ public static function getRerunFieldValues($jobid,$fields) { $folder=Yii::$app->params['tmpFolderPath'] . '/' . $jobid . '/'; - $file=$folder . 'fields.txt'; + $file=$folder . 'fields.json'; if (file_exists($file)) { @@ -1296,9 +1203,23 @@ public static function getRerunFieldValues($jobid,$fields) } else { - return $fields; + /* + * try fields.txt for backward compatibility + */ + $file=$folder . 'fields.txt'; + $content=file_get_contents($file); + if (file_exists($file)) + { + $content=file_get_contents($file); + } + else + { + return $fields; + } + + } - // print_r($file); + $json=json_decode($content,true); if (empty($json)) @@ -1328,19 +1249,26 @@ public static function getRerunFieldValues($jobid,$fields) $fields[$i]->value=trim($field_values[$i]); } } - // print_r($fields[--$i]->value?'true':'false'); - // exit(0); } else { - foreach ($fields as $field) + if (!empty($fields)) + { + $fieldCount=count($fields); + } + else { - if (!array_key_exists($field->name,$json)) + $fieldCount=0; + } + + for ($i=0; $i<$fieldCount; $i++) + { + if (!array_key_exists($fields[$i]->name,$json)) { - return false; + continue; } - $field->value=$json[$field->name]; + $fields[$i]->value=$json[$fields[$i]->name]; } } return $fields; @@ -1382,92 +1310,11 @@ public static function updateExampleFields($name,$version,$values) } - public static function hasExample($name,$version) - { - $query=new Query; - $result=$query->select(['has_example']) - ->from('software') - ->where(['name'=>$name, 'version'=>$version]) - ->one(); - - return $result['has_example']; - - } - public static function uploadedBy($name,$version) - { - $query=new Query; - $result=$query->select(['uploaded_by']) - ->from('software') - ->where(['name'=>$name, 'version'=>$version]) - ->one(); - - return $result['uploaded_by']; - - } - - public static function getInactiveJobs() - { - - $command=self::sudoWrap('kubectl get jobs --no-headers 2>&1'); - - Software::exec_log($command,$output,$ret); - - // print_r($output); - // exit(0); - if (trim($output[0])=="No resources found in default namespace.") - { - return []; - } - if (trim($output[0])=="No resources found.") - { - return []; - } - - $inactive=[]; - $active=[]; - - foreach ($output as $line) - { - $tokens_tmp=explode(' ',$line); - $tokens=[]; - foreach ($tokens_tmp as $token) - { - if (!empty($token)) - $tokens[]=$token; - } - // print_r($tokens); - // exit(0); - $completed_tokens=explode('/',$tokens[1]); - $completed=intval($completed_tokens[0]); - $total=intval($completed_tokens[1]); - // var_dump($total); - // var_dump($completed); - // exit(0); - if ($completed!=$total) - continue; - // $status='Completed'; - $job=$tokens[0]; - // if ($status=='Completed') - // { - $job=explode('-',$job); - $inactive[]=$job; - // } - - - } - // print_r($inactive); - // exit(0); - return $inactive; - - - } public static function getActiveProjects() { $username=User::getCurrentUser()['username']; - // print_r("https://egci-beta.imsi.athenarc.gr/index.php?r=api/active-ondemand-quotas&username=$username"); - // exit(0); $client = new Client(); $response = $client->createRequest() ->setMethod('GET') @@ -1475,8 +1322,6 @@ public static function getActiveProjects() ->send(); $projects=$response->data; - // print_r($projects); - // exit(0); $projectNames=[]; $projectJobs=[]; @@ -1503,8 +1348,6 @@ public static function getActiveProjects() ]) ->groupBy('project') //->createCommand()->getRawSql(); ->all(); - // print_r($projectUsage); - // exit(0); $projectsDB=[]; foreach($projectUsage as $project) { @@ -1530,7 +1373,7 @@ public static function getActiveProjects() - public static function getUserStatistics($softUser) + public static function getUserStatistics($softUser) { $query=new Query; @@ -1717,4 +1560,18 @@ public static function exec_log($command, array &$out=null, int &$ret=null) error_log(implode(" ", $out)); } } + + public static function getTesUrl() + { + if (isset(Yii::$app->params['schemaTes']) && (!empty(Yii::$app->params['schemaTes']))) + { + return Yii::$app->params['teskEndpoint'] . "/v1/tasks"; + } + else + { + return Yii::$app->params['teskEndpoint'] . "/v1/tasks"; + } + } + + } diff --git a/models/Workflow.php b/models/Workflow.php index 3e29c82..e7db2b9 100644 --- a/models/Workflow.php +++ b/models/Workflow.php @@ -189,18 +189,6 @@ public static function getParameters($fields) return [json_encode($params,JSON_UNESCAPED_SLASHES),$errors]; } - /* - * If the job did not submit, then return false - */ - public static function isAlreadyRunning($jobid) - { - if ($jobid=='') - { - return false; - } - - } - /* * Returns a list of folders to be used in the * select folder popup @@ -598,7 +586,6 @@ public static function runWorkflow($workflow, $newLocation, $tmpWorkflowFolder, ->setMethod('POST') ->setUrl($url) ->send(); - // ->toString(); } catch (Exception $e) { @@ -645,8 +632,6 @@ public static function runWorkflow($workflow, $newLocation, $tmpWorkflowFolder, $data=$response->data; $jobid=$data['run_id']; - // print_r($data); - // exit(0); $query=Yii::$app->db->createCommand()->insert('run_history', [ @@ -717,6 +702,37 @@ public static function runWorkflow($workflow, $newLocation, $tmpWorkflowFolder, } + public static function isAlreadyRunning($jobid) + { + $stopStates=['COMPLETE'=>false,'EXECUTOR_ERROR'=>false,'SYSTEM_ERROR'=>false,'CANCELED'=>false,'CANCELING'=>false,]; + if (empty($jobid)) + { + return false; + } + $url=Yii::$app->params['wesEndpoint'] . '/ga4gh/wes/v1/runs/' . $jobid; + $client = new Client(); + $response = $client->createRequest() + ->addHeaders(['Content-Type'=>'application/json','Accept'=>'application/json']) + ->setMethod('GET') + ->setUrl($url) + ->send(); + if ($response->getIsOk()) + { + $state=$response->data['state']; + if (isset($stopStates[$state])) + { + return false; + } + else + { + return true; + } + } + + return false; + + } + public static function getLogs($jobid) { $url=Yii::$app->params['wesEndpoint'] . '/ga4gh/wes/v1/runs/' . $jobid; @@ -727,6 +743,7 @@ public static function getLogs($jobid) ->setUrl($url) ->send(); $statusCode=$response->getStatusCode(); + if ($statusCode==400) { $error='Malformed request. Please contact an administrator'; @@ -742,6 +759,11 @@ public static function getLogs($jobid) $error='Requester not authorized to perform this action. Please contact an administrator'; return ['jobid'=>'','error'=>$error]; } + else if ($statusCode==404) + { + $error='Job not found in the workflows system. Please contact an administrator'; + return ['jobid'=>'','error'=>$error]; + } else if ($statusCode==500) { $error='An unexpected error occurred. Please contact an administrator'; @@ -807,12 +829,9 @@ public static function getLogs($jobid) $logs=[]; $i=1; - // print_r($taskLogs); - // exit(0); + foreach ($taskLogs as $index=>$log) { - // print_r($status); - // exit(0); if (!empty($log)) { $value=[ diff --git a/scheduler_files/jobMonitor.py b/scheduler_files/jobMonitor.py index e099832..c75fef4 100755 --- a/scheduler_files/jobMonitor.py +++ b/scheduler_files/jobMonitor.py @@ -24,13 +24,15 @@ import sys import subprocess import time +import requests import psycopg2 as psg import json from datetime import datetime -jobName=sys.argv[1] -jobid=sys.argv[2] -folder=sys.argv[3] +jobid=sys.argv[1] +folder=sys.argv[2] +endpoint=sys.argv[3] +tesType=sys.argv[4] configFileName=os.path.dirname(os.path.abspath(__file__)) + '/configuration.json' configFile=open(configFileName,'r') @@ -41,189 +43,119 @@ dbuser=db['username'] passwd=db['password'] dbname=db['database'] -namespaces=config.get('namespaces',None) -jobNamespace=None -if namespaces is not None: - jobNamespace=namespaces.get('jobs',None) -if jobNamespace is not None: - command="kubectl get pods -n " + jobNamespace + " --no-headers -l job-name=" + jobName + " | tr -s ' '" -else: - command="kubectl get pods --no-headers -l job-name=" + jobName + " | tr -s ' '" -try: - out=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') -except subprocess.CalledProcessError as exc: - print(exc.output) - -podid='' -out=out.split(' ') -podid=out[0] -status=out[2] -status_code=0 cpu=0 memory=0 -stopStatuses=set(['Completed','Error','StartError','ErrImagePullBackOff', "ContainerCannotRun","RunContainerError","OOMKilled", 'OutOfcpu']) -while status not in stopStatuses: - - code=0 - if jobNamespace is not None: - command="kubectl top pod --use-protocol-buffers --no-headers " + podid + ' -n ' + jobNamespace - else: - command="kubectl top pod --use-protocol-buffers --no-headers " + podid - try: - out=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') - except subprocess.CalledProcessError as exc: - code=exc.returncode +if tesType=='1': + taskUrl=endpoint + '/v1/tasks?id=' + jobid + '&view=FULL' +else: + taskUrl=endpoint + '/v1/tasks/' + jobid + '?view=FULL' - if code==0: - command='echo "'+ out + "\" | tr -s ' ' " +headers={'Accept':'application/json', 'Content-Type': 'application/json'} +response = requests.get(taskUrl,headers=headers) +if not response.ok: + print(2,'Job not found.') + exit(2) +try: + content=json.loads(response.content) +except Exception as e: + print(3,e) + exit(3) + +state=content['state'] +states=['UNKNOWN','QUEUED','INITIALIZING','RUNNING','PAUSED','COMPLETE','EXECUTOR_ERROR','SYSTEM_ERROR','CANCELED'] +stopStates=['COMPLETE','CANCELED']; +errorStatues=['EXECUTOR_ERROR','SYSTEM_ERROR'] +sqlStates={'COMPLETE':'Complete','EXECUTOR_ERROR':'Error','SYSTEM_ERROR':'Error','CANCELED':'Canceled'} + +toleration=0; +while state not in stopStates: + if tesType=='1': try: - out=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') - except subprocess.CalledProcessError as exc: - code=exc.returncode - tokens=out.split(' ') - topmem=float(tokens[2][:-2]) - - memmeasure=tokens[2][-2:] - - if memmeasure=='Mi': - topmem/=1024.0 - elif memmeasure=='Ki': - topmem/=(1024.0*1024) - - if memory1: - for line in out: - line=line.split(' ') - status=line[2] - print(out) - if status=='OOMKilled': - break - else: - out=out[0] - out=out.split(' ') - status=out[2] - -if status!='Canceled': - #Get start and end times - if jobNamespace is not None: - command="kubectl get job " + jobName + " -o=jsonpath='{.status.completionTime}' -n " + jobNamespace + resources=content['current_resources'] + tmp_cpu=resources['cpu_cores'] + tmp_memory=resources['ram_gb'] + if memory2: + break + toleration+=1 + +k8sformat="%Y-%m-%dT%H:%M:%S.%fZ" +sqlformat="%Y-%m-%d %H:%M:%S" +while 'end_time' not in content['logs'][0]: + headers={'Accept':'application/json', 'Content-Type': 'application/json'} + response = requests.get(taskUrl,headers=headers) + if not response.ok: + print(6,'Job not found by loop.') + exit(6) try: - out=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') - except subprocess.CalledProcessError as exc: - print(exc.output) - start=out.replace('T',' ').replace('Z',' ').strip() + content=json.loads(response.content) + except Exception as e: + print(7,e) + exit(7) + time.sleep(1) - if status=='Completed': - status='Complete' - +endTime=content['logs'][0]['end_time'] +startTime=content['logs'][0]['start_time'] +endobj=datetime.strptime(endTime,k8sformat) +startobj=datetime.strptime(startTime,k8sformat) +status=sqlStates[state] conn=psg.connect(host=host, user=dbuser, password=passwd, dbname=dbname) cur=conn.cursor() -sql="SELECT status FROM run_history WHERE jobid='" + jobid + "'" -cur.execute(sql) -result=cur.fetchone() -if result[0]=='Canceled': - status='Canceled' - -if status=='Canceled': - #everything is done with PHP on the interface - pass -elif status=='Complete': - query="UPDATE run_history SET ram=" + str(memory) + ", cpu=" + str(cpu) + ", start='" + start +"', stop='" + str(end) + "', status='" + status +"' WHERE jobid='" + jobid + "'" - status_code=0 - cur.execute(query) - conn.commit() - -elif status=='OOMKilled': - query="UPDATE run_history SET stop='NOW()', status='Out_of_RAM', remote_status_code=-10 WHERE jobid='" + jobid + "'" - status_code=-10 - cur.execute(query) - conn.commit() -elif status=='OutOfcpu': - query="UPDATE run_history SET stop='NOW()', status='Out_οf_CPU', remote_status_code=-10 WHERE jobid='" + jobid + "'" - status_code=-11 - cur.execute(query) - conn.commit() -else: - query="UPDATE run_history SET stop='NOW()', status='Error', remote_status_code=-9 WHERE jobid='" + jobid + "'" - status_code=-2 - cur.execute(query) - conn.commit() - - +query="UPDATE run_history SET ram=" + str(memory) + ", cpu=" + str(cpu) + ", start='" + startobj.strftime(sqlformat) +"', stop='" + endobj.strftime(sqlformat) + "', status='" + status +"' WHERE jobid='" + jobid + "'" +cur.execute(query) +conn.commit() conn.close() +try: + logs=content['logs'][0]['logs'][0]['stdout'] +except: + logs="" -if (status!='Canceled'): - #Get logs - if jobNamespace is not None: - command="kubectl get pods -n " + jobNamespace + " --no-headers -l job-name=" + jobName + " | tr -s ' '" - else: - command="kubectl get pods --no-headers -l job-name=" + jobName + " | tr -s ' '" - try: - out=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') - except subprocess.CalledProcessError as exc: - print(exc.output) +logFile=folder + '/logs.txt' +g=open(logFile,'w') +g.write(logs) +g.close() - podid='' - out=out.split(' ') - podid=out[0] +if config['cleanTeskJobs']: + namespaces=config.get('namespaces',None) + jobNamespace=None + if namespaces is not None: + jobNamespace=namespaces.get('tesk',None) if jobNamespace is not None: - command="kubectl logs " + podid + " -n " + jobNamespace + " 2>&1" + command="for j in $(kubectl get jobs -n " + jobNamespace + " --no-headers | grep '" + jobid + "' | tr -s ' ' | cut -d ' ' -f 1); do kubectl delete jobs -n " + jobNamespace + " $j; done" else: - command="kubectl logs " + podid + " 2>&1" - + command="kubectl get jobs --no-headers -l | grep '" + jobid + "' | tr -s ' '| cut -d ' ' -f 1); do kubectl delete jobs $j; done " try: - logs=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') + out=subprocess.check_output(command,stderr=subprocess.STDOUT,shell=True, encoding='utf-8') except subprocess.CalledProcessError as exc: - logs='' print(exc.output) - except UnicodeDecodeError as exc: - logs='' - print(exc) - - logFile=folder + '/logs.txt' - g=open(logFile,'w') - g.write(logs) - g.close() - -#Clean job -yamlFile=folder + '/' + jobName + '.yaml' -returnCode=subprocess.call(['kubectl','delete','-f',yamlFile]) diff --git a/views/software/logs.php b/views/software/logs.php index 52656d4..0eedc04 100644 --- a/views/software/logs.php +++ b/views/software/logs.php @@ -45,14 +45,13 @@ -
Type of machine:
Status:
-
">
+
">
Running time:
-
">
+
">
diff --git a/views/software/run.php b/views/software/run.php index 64a98a6..e01f3bc 100644 --- a/views/software/run.php +++ b/views/software/run.php @@ -37,9 +37,9 @@ echo Html::CssFile('@web/css/software/run.css'); $this->registerJsFile('@web/js/software/run-index.js', ['depends' => [\yii\web\JqueryAsset::className()]]); -$this->title = "New job ($name v.$version) "; +$this->title = "New job ($software->name v.$software->version) "; -$commandsDisabled= ($jobid!='') ? true : false; +$commandsDisabled= ($software->jobid!='') ? true : false; if($commandsDisabled) { $commandBoxClass= 'disabled-box'; @@ -49,7 +49,7 @@ $commandBoxClass=''; } -if ($hasExample) +if ($software->has_example) { $exampleBtnLink='javascript:void(0);'; } @@ -91,31 +91,28 @@ jobid,$software->name,$software->version,$example, $software->has_example); RunFormWidget::showArguments($fields,$type,$commandBoxClass,$commandsDisabled); RunFormWidget::showResources($quotas,$maxCores,$maxMem,$commandsDisabled,$commandBoxClass); - RunFormWidget::showRunButtons($superadmin,$username,$uploadedBy,$hasExample,$commandsDisabled,$type,$name,$version,$software_instructions); + RunFormWidget::showRunButtons($superadmin,$username,$software->uploaded_by,$software->has_example,$commandsDisabled,$type,$software->name,$software->version,$software->instructions); ?> -
-
- - - -
- "; - echo "

Runtime Info:

"; - echo "Status:
QUEUED

"; - echo $this->registerJsFile('@web/js/software/logs.js', ['depends' => [\yii\web\JqueryAsset::className()]] ); - - }?> -
+ +
+ "; + echo "

Runtime Info:

"; + echo "Status:
INITIALIZING

"; + echo $this->registerJsFile('@web/js/software/logs.js', ['depends' => [\yii\web\JqueryAsset::className()]] ); + + }?> + + diff --git a/web/js/software/logs.js b/web/js/software/logs.js index e7405b6..2e1b1dc 100644 --- a/web/js/software/logs.js +++ b/web/js/software/logs.js @@ -27,179 +27,11 @@ $(document).ready(function() $(".cancel-button-container").css('display','inline-block'); - function cleanUp() + function enableForm() { - var jobid=$("#hidden_jobid_input").val(); - var name=$("#hidden_name_input").val(); - // $.ajax({ - // url: "index.php?r=software/clean-up", - // type: "GET", - // data: { "jobid": jobid, "name": name, 'status': 'Complete'}, - // dataType: "html", - // success: function (data) - // { - // $("#initial-status").hide(); - // $('#pod-logs').html(data); - - // status=$("#status-value").text(); - // if (status == "Completed") - // { - // clearInterval(refreshId); - // } - // }, - // retries: 2, - // complete: function() - // { - // // Schedule the next request when the current one's complete - // setInterval(sendRequest, 5000); // The interval set to 5 seconds - // }, - // }); - } - function cancelClean() - { - var jobid=$("#hidden_jobid_input").val(); - var name=$("#hidden_name_input").val(); - $.ajax({ - url: "index.php?r=software/clean-up", - type: "GET", - data: { "jobid": jobid, "name": name, "status": "Canceled"}, - dataType: "html", - success: function (data) - { - clearInterval(refreshId); - $("#status-value").html("Canceled"); - $(".job-output").hide(); - $(".running-logo").hide(); - $("#command-text-box").removeClass('disabled-box'); - $("#command-text-box").prop('readonly',false); - $(".cancel-button-container").css('display','none'); - $(".select-mount-button").removeAttr('disabled'); - $(".clear-mount-button").removeAttr('disabled'); - $("#software-start-run-button").removeAttr('disabled'); - } - // $("#initial-status").hide(); - // $('#pod-logs').html(data); - - // status=$("#status-value").text(); - // if (status == "Completed") - // { - // clearInterval(refreshId); - // } - // }, - // retries: 2, - // complete: function() - // { - // // Schedule the next request when the current one's complete - // setInterval(sendRequest, 5000); // The interval set to 5 seconds - // }, - }); - } - function sendRequest(refId) - { - var podid=$("#hidden_podid_input").val(); - var machineType=$("#hidden_machineType_input").val(); - var jobid=$("#hidden_jobid_input").val(); - // window.alert(jobid); - $.ajax({ - url: "index.php?r=software/get-logs", - type: "GET", - data: { "podid": podid, "machineType": machineType, "jobid" : jobid }, - dataType: "html", - success: function (data) - { - $("#initial-status").hide(); - $('#pod-logs').html(data); - $('.container-logs').scrollTop($('.container-logs').prop('scrollHeight')); - - - status=$("#status-value").text(); - if ( (status == "Completed") || (status == "Error") || (status == "ImagePullBackOff") || (status == "Terminating") || (status == "Canceled") || (status=="OOMKilled")) - { - clearInterval(refId); - setTimeout(cleanUp(),2000); - totalFields=Number($("#hidden_fieldsNum").val()); - var i=0; - for (i=0; i