From 486bf95865b1bf15ddf74c4d23f3430e297d077d Mon Sep 17 00:00:00 2001 From: zagganas Date: Mon, 16 May 2022 16:53:26 +0000 Subject: [PATCH] Uploader can now specify a workingdir --- models/Software.php | 2 +- models/SoftwareUpload.php | 8 ++++---- models/SoftwareUploadExisting.php | 8 ++++---- scheduler_files/existingImageUploader.py | 22 ++++++++++++--------- scheduler_files/imageUploader.py | 5 ++--- scheduler_files/uploadFunctions.py | 11 +++++------ views/software/software_edit.php | 1 + views/software/software_upload.php | 1 + views/software/software_upload_existing.php | 1 + 9 files changed, 32 insertions(+), 27 deletions(-) diff --git a/models/Software.php b/models/Software.php index 71b78c6..3dcd5ef 100644 --- a/models/Software.php +++ b/models/Software.php @@ -822,7 +822,7 @@ public function runJob() $executor=[]; $executor['image']=$this->image; $executor['command']=$this->container_command; - $executor['workdir']=empty($this->omountpoint) ? $this->imountpoint : $this->omountpoint; + $executor['workdir']=empty($this->workingdir) ? $this->omountpoint : $this->workingdir; $data['executors']=[$executor]; // var_dump($data); // exit(0); diff --git a/models/SoftwareUpload.php b/models/SoftwareUpload.php index dc07c75..c1c4723 100644 --- a/models/SoftwareUpload.php +++ b/models/SoftwareUpload.php @@ -113,7 +113,7 @@ public function attributeLabels() 'visibility' => 'Visible to', 'imountpoint'=>'Input folder mount point (where users can provide input data inside the container). Leave empty if no mount is required', 'omountpoint'=>'Output folder mount point (folder inside the container where users can find the output). Leave empty if no mount is required', - 'workingdir'=>'Working directory (inside the container). If left empty, /data will be used.', + 'workingdir'=>'Working directory (inside the container). If left empty, the output mountpoint will be used.', 'description'=> 'Software description * ', 'imageInDockerHub'=>'Image exists in DockerHub and is specified in the CWL file', 'iomount' => 'Image requires disk I/O', @@ -180,18 +180,18 @@ public function upload() $imageFileName=$this->quotes($imageFileName); $this->name=$this->quotes($this->name); $this->version=$this->quotes($this->version); - $mpi=($this->mpi=='1') ? $this->quotes('t') : $this->quotes('f'); + $this->workingdir=$this->quotes($this->workingdir); $username=$this->quotes($username); $arguments=[$this->name, $this->version, $imageFileName, $imageFileExt, $cwlFileName, $username, $this->visibility, $this->imountpoint, $this->omountpoint, - $this->description, $this->biotools, $doiFile, $mpi, $this->covid19, $this->instructions,$this->gpu]; + $this->description, $this->biotools, $doiFile, $this->workingdir, $this->covid19, $this->instructions,$this->gpu]; $command=Software::sudoWrap(Yii::$app->params['scriptsFolder'] . "imageUploader.py "); $command.= implode(" ", $arguments) . " "; $command.= "2>&1"; - + print_r($command);exit(0); Software::exec_log($command,$out,$ret); $errors=''; diff --git a/models/SoftwareUploadExisting.php b/models/SoftwareUploadExisting.php index 2ac640d..7316c2a 100644 --- a/models/SoftwareUploadExisting.php +++ b/models/SoftwareUploadExisting.php @@ -114,7 +114,7 @@ public function attributeLabels() 'visibility' => 'Visible to', 'imountpoint'=>'Input folder mount point (where users can provide input data inside the container). Leave empty if no mount is required', 'omountpoint'=>'Output folder mount point (folder inside the container where users can find the output). Leave empty if no mount is required', - 'workingdir'=>'Working directory (inside the container). If left empty, /data will be used.', + 'workingdir'=>'Working directory (inside the container). If left empty, the output mountpoint a will be used.', 'description'=> 'Software description * ', 'imageInDockerHub'=>'Image exists in DockerHub and is specified in the CWL file', 'iomount' => 'Image requires disk I/O', @@ -140,12 +140,13 @@ public function upload() $workingdir=$previous->workingdir; $original=$previous->original_image; $dockerhub=($previous->docker_or_local) ? "'t'" : "'f'"; - $workingdir=$this->quotes($workingdir); + $this->workingdir=$this->quotes($this->workingdir); $original=$this->quotes($original); $this->covid19=($this->covid19=='1') ? "'t'" : "'f'"; $this->biotools=$this->quotes($this->biotools); $this->instructions=$this->quotes($this->instructions); $this->gpu=$this->quotes($this->gpu); + $this->workingdir=$this->quotes($this->workingdir); //add dois string in a file and pass it on to python $dataFolder=Yii::$app->params['tmpImagePath'] . $username . '/' . str_replace(' ','-',$this->name) . '/' . str_replace(' ','-',$this->version) . '/'; @@ -178,13 +179,12 @@ public function upload() $this->name=$this->quotes($this->name); $this->version=$this->quotes($this->version); - $mpi=($this->mpi=='1') ? $this->quotes('t') : $this->quotes('f'); $username=$this->quotes($username); $arguments=[ $this->name, $this->version, $this->image, $cwlFileName, $username, $this->visibility, $this->imountpoint, $this->omountpoint, - $this->description, $this->biotools, $doiFile, $mpi, $workingdir, + $this->description, $this->biotools, $doiFile, $this->workingdir, $original,$dockerhub,$this->covid19, $this->instructions,$this->gpu]; $command=Software::sudoWrap(Yii::$app->params['scriptsFolder'] . "existingImageUploader.py "); diff --git a/scheduler_files/existingImageUploader.py b/scheduler_files/existingImageUploader.py index e2561df..e6b17d6 100755 --- a/scheduler_files/existingImageUploader.py +++ b/scheduler_files/existingImageUploader.py @@ -39,13 +39,12 @@ description=sys.argv[9] biotools=sys.argv[10] doiFile=sys.argv[11] -mpi=sys.argv[12] -workingDir=sys.argv[13] -original=sys.argv[14] -docker_or_local=sys.argv[15] -covid19=sys.argv[16] -instructions=sys.argv[17] -gpu=sys.argv[18] +workingDir=sys.argv[12] +original=sys.argv[13] +docker_or_local=sys.argv[14] +covid19=sys.argv[15] +instructions=sys.argv[16] +gpu=sys.argv[17] def quoteEnclose(string): return "'" + string + "'" @@ -77,11 +76,16 @@ def quoteEnclose(string): workingDir='/' uf.imageStore(softName,softVersion, image,script,user,visibility, - workingDir,imountPoint,omountPoint,description,cwlPath,biotools,doiFile,mpi,original,docker_or_local,covid19,instructions,gpu) + workingDir,imountPoint,omountPoint,description,cwlPath,biotools,doiFile,original,docker_or_local,covid19,instructions,gpu) if 'inputs' not in cwlContent: cwlContent['inputs']=[]; -exit_value=uf.inputStore(softName,softVersion, cwlContent['inputs']) +if isinstance(cwlContent['inputs'],dict): + exit_value=uf.inputStoreDict(softName,softVersion, cwlContent['inputs']) +elif isinstance(cwlContent['inputs'],list): + exit_value=uf.inputStoreList(softName,softVersion, cwlContent['inputs']) +else: + exit_value=100 exit(exit_value) diff --git a/scheduler_files/imageUploader.py b/scheduler_files/imageUploader.py index 402ed6d..a23d9ab 100755 --- a/scheduler_files/imageUploader.py +++ b/scheduler_files/imageUploader.py @@ -44,7 +44,7 @@ description=sys.argv[10] biotools=sys.argv[11] doiFile=sys.argv[12] -mpi=sys.argv[13] +workingDir=sys.argv[13] covid19=sys.argv[14] instructions=sys.argv[15] gpu=sys.argv[16] @@ -121,7 +121,6 @@ def quoteEnclose(string): # imageName=softName.lower() # imageVersion=softVersion.lower() -workingDir='/' if (dockerHub=='t'): @@ -161,7 +160,7 @@ def quoteEnclose(string): conn.commit() uf.imageStore(softName,softVersion, imageFull,script,user,visibility, - workingDir,imountPoint,omountPoint,description,cwlPath,biotools,doiFile,mpi,original,dockerHub,covid19,instructions,gpu) + workingDir,imountPoint,omountPoint,description,cwlPath,biotools,doiFile,original,dockerHub,covid19,instructions,gpu) if 'inputs' not in cwlContent: cwlContent['inputs']=[]; diff --git a/scheduler_files/uploadFunctions.py b/scheduler_files/uploadFunctions.py index d2984cb..74ae8f9 100755 --- a/scheduler_files/uploadFunctions.py +++ b/scheduler_files/uploadFunctions.py @@ -422,7 +422,7 @@ def inputStoreList(softName,softVersion, inputs): return 0 def imageStore(name,version,image,script,user,visibility, - workingDir,imountPoint,omountPoint, description,cwlPath,biotools,doiFile,mpi,original,docker_or_local,covid19,instructions,gpu): + workingDir,imountPoint,omountPoint, description,cwlPath,biotools,doiFile,original,docker_or_local,covid19,instructions,gpu): softFull=name+ '-' + version name=quoteEnclose(name) @@ -437,7 +437,6 @@ def imageStore(name,version,image,script,user,visibility, description=quoteEnclose(description) cwlPath=quoteEnclose(cwlPath) biotools=quoteEnclose(biotools) - mpi=quoteEnclose(mpi) original=quoteEnclose(original) docker_or_local=quoteEnclose(docker_or_local) covid19=quoteEnclose(covid19) @@ -459,19 +458,19 @@ def imageStore(name,version,image,script,user,visibility, date="NOW()" values=[name,version,image,script,user, date, visibility, workingDir, imountPoint, - omountPoint, description, cwlPath,biotools,dois,mpi,original,docker_or_local,covid19, instructions,gpu] + omountPoint, description, cwlPath,biotools,dois,original,docker_or_local,covid19, instructions,gpu] sql1='INSERT INTO software_upload (name,version, image,script,uploaded_by, date, visibility, workingdir, imountpoint, omountpoint,\ - description, cwl_path,biotools,dois,mpi,original_image,docker_or_local,covid19, instructions,gpu) ' + description, cwl_path,biotools,dois,original_image,docker_or_local,covid19, instructions,gpu) ' sql1+='VALUES (' + ','.join(values) + ')' values=[name,version,image,script,user, visibility, workingDir, imountPoint, omountPoint,description,cwlPath,biotools, - dois,mpi,original,docker_or_local,covid19, instructions,gpu] + dois,original,docker_or_local,covid19, instructions,gpu] sql2='INSERT INTO software (name,version,image,script,uploaded_by,\ visibility, workingdir, imountpoint, omountpoint, description,\ - cwl_path,biotools,dois,mpi,original_image,docker_or_local,covid19, instructions,gpu) ' + cwl_path,biotools,dois,original_image,docker_or_local,covid19, instructions,gpu) ' sql2+='VALUES (' + ','.join(values) + ')' # print() diff --git a/views/software/software_edit.php b/views/software/software_edit.php index 4a00797..b394f7a 100644 --- a/views/software/software_edit.php +++ b/views/software/software_edit.php @@ -75,6 +75,7 @@ field($model, 'shared') -> checkbox(['id'=>'shared', "uncheck"=>'0',]):'' ?> + field($model, 'workingdir')?> field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?> field($model, 'biotools')->textInput() ?> diff --git a/views/software/software_upload.php b/views/software/software_upload.php index 643b702..403a852 100644 --- a/views/software/software_upload.php +++ b/views/software/software_upload.php @@ -76,6 +76,7 @@ field($model, 'imountpoint') ?> field($model, 'omountpoint') ?> + field($model, 'workingdir')?> field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?> field($model, 'biotools') ?> diff --git a/views/software/software_upload_existing.php b/views/software/software_upload_existing.php index 832768a..b92ba25 100644 --- a/views/software/software_upload_existing.php +++ b/views/software/software_upload_existing.php @@ -74,6 +74,7 @@ field($model, 'imountpoint') ?> field($model, 'omountpoint') ?> + field($model, 'workingdir')?> field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?> field($model, 'biotools') ?>