From d0b1adc97275505eb4e2097462e14fd2b74a9bf6 Mon Sep 17 00:00:00 2001 From: zagganas Date: Tue, 5 Apr 2022 17:04:40 +0000 Subject: [PATCH] Added jupyter gpu capability. Added spawning state --- controllers/JupyterController.php | 16 +++++++++-- database_schema/schema_db.sql | 2 ++ models/JupyterImages.php | 3 ++ models/JupyterServer.php | 41 ++++++++++++++++++++++++--- scheduler_files/jupyterConfig.py | 6 ++-- scheduler_files/jupyterServerStart.py | 11 +++---- views/jupyter/edit_image.php | 2 ++ views/jupyter/image_list.php | 8 ++++-- views/jupyter/index.php | 32 +++++++++++++++++---- views/jupyter/new_image.php | 1 + views/jupyter/start_server.php | 2 +- web/js/jupyter/start_server.js | 5 +++- 12 files changed, 105 insertions(+), 24 deletions(-) diff --git a/controllers/JupyterController.php b/controllers/JupyterController.php index 3b327e4..6fce8f0 100644 --- a/controllers/JupyterController.php +++ b/controllers/JupyterController.php @@ -134,7 +134,12 @@ public function actionIndex() $images=[]; foreach ($img as $i) { - $images[$i->image]=$i->description; + $description=$i->description; + if ($i->gpu) + { + $description.=' (GPU)'; + } + $images[$i->image]=$description; } return $this->render('index',['projects'=>$projects,'images'=>$images]); @@ -190,11 +195,15 @@ public function actionStartServer($project) $imageDrop=[]; foreach ($images as $image) { - $imageDrop[$image->image]=$image->description; + $description=$image->description; + if ($image->gpu) + { + $description.=" (GPU enabled)"; + } + $imageDrop[$image->id]=$description; } $model = new JupyterServer; - if ($model->load(Yii::$app->request->post()) && $model->validate()) { $model->cpu=$quotas['cores']; @@ -356,6 +365,7 @@ public function actionEditImage($id) if ($model->load(Yii::$app->request->post()) && $model->validate()) { + $model->gpu=($model->gpu==1) ? true : false; $model->save(); Yii::$app->session->setFlash('success',"Image $model->image saved!"); diff --git a/database_schema/schema_db.sql b/database_schema/schema_db.sql index 2e0333a..4ff87e7 100644 --- a/database_schema/schema_db.sql +++ b/database_schema/schema_db.sql @@ -1783,3 +1783,5 @@ alter table workflow add column workflow_type varchar(100); alter table workflow_upload add column workflow_type varchar(100); create index workflow_type_idx on workflow_upload(workflow_type); create index workflow_upload_type_idx on workflow_upload(workflow_type); +alter table jupyter_images add column gpu boolean default false; +alter table jupyter_server add column state varchar(20); diff --git a/models/JupyterImages.php b/models/JupyterImages.php index 960daf2..3e07acf 100644 --- a/models/JupyterImages.php +++ b/models/JupyterImages.php @@ -28,6 +28,8 @@ public function rules() { return [ [['description', 'image'], 'string'], + [['gpu'],'boolean'], + [['gpu'],'required'], ]; } @@ -40,6 +42,7 @@ public function attributeLabels() 'id' => 'ID', 'description' => 'Descriptive name', 'image' => 'Image on dockerhub', + 'gpu' => 'Image uses GPU' ]; } } diff --git a/models/JupyterServer.php b/models/JupyterServer.php index 5c1223f..8bc9b2c 100644 --- a/models/JupyterServer.php +++ b/models/JupyterServer.php @@ -4,6 +4,7 @@ use Yii; use app\models\Softare; +use app\models\JupyterImages; use webvimark\modules\UserManagement\models\User; use yii\helpers\Html; use yii\httpclient\Client; @@ -46,7 +47,7 @@ public function rules() [['active'], 'boolean'], [['manifest', 'project'], 'string', 'max' => 100], [['server_id'], 'string', 'max' => 20], - [['password','image'],'required'] + [['password', 'image_id'],'required'] ]; } @@ -109,7 +110,28 @@ public static function matchServersWithProjects($projects) */ if (isset($projects[$server->project])) { - $projects[$server->project]['server']=$server; + $projects[$server->project]['server']=$server; + if ($server->state=='spawning') + { + try + { + $client = new Client(); + $response = $client->createRequest() + ->setMethod('GET') + ->setUrl($server->url) + ->send(); + if ($response->getIsOk()) + { + $server->state='running'; + $server->save(false); + } + + } + catch (\Exception $e) + { + + } + } } } @@ -142,6 +164,13 @@ public function startServer() $username=User::getCurrentUser()['username']; $user=explode('@',$username)[0]; + $image=JupyterImages::find()->where(['id'=>$this->image_id])->one(); + if (empty($image)) + { + $error='Image not found. Please try again or contact an administrator'; + return ['',$error]; + } + $data=[]; if (file_exists('/data/containerized')) { @@ -153,7 +182,9 @@ public function startServer() } $data['nfs']=$nfs; - $data['image']=$this->image; + $data['image']=$image->image; + $data['image_id']=$this->image_id; + $data['gpu']=$image->gpu; $data['id']=$sid; $data['folder']=Yii::$app->params['tmpFolderPath'] . '/' . $sid . '/'; $data['mountFolder']=Yii::$app->params['userDataPath'] . $user . '/'; @@ -205,7 +236,7 @@ public function startServer() } } - catch (yii\httpclient\Exception $e) + catch (\Exception $e) { /* * This block is left empty on purpose. @@ -256,6 +287,8 @@ public function startServer() if (!$isDown) { $success='Server was started successfully! It can be accessed here.'; + $server->state='running'; + $server->save(false); } else { diff --git a/scheduler_files/jupyterConfig.py b/scheduler_files/jupyterConfig.py index f9ddcdf..e6c0f16 100644 --- a/scheduler_files/jupyterConfig.py +++ b/scheduler_files/jupyterConfig.py @@ -2,7 +2,7 @@ from notebook.auth import passwd -def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, domain, platform): +def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, domain, platform, gpu): manifest=folder + '/' + sid + '-jupyter.yaml' appName=sid + '-jupyter' @@ -10,7 +10,6 @@ def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, d volumes=[] containers=[] pod={} - pod['replicas']=1 pod['selector']={'matchLabels':{'app':appName}} pod['template']={'metadata':{'labels':{'app':appName}}} @@ -36,6 +35,9 @@ def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, d container['env'].append({'name':'JUPYTER_ENABLE_LAB', 'value': 'yes'}) container['resources']={'limits':{'cpu':str(cpu), 'memory':str(mem) + 'Gi'}, 'requests':{'cpu':str(cpu), 'memory':str(mem) + 'Gi'}} + if gpu: + container['resources']['limits']['nvidia.com/gpu']=1 + volumeMounts=[] if nfs=='container': vmount={'name': vname, 'mountPath': '/home/jovyan/work', 'subPath': mount.replace('/data/','')} diff --git a/scheduler_files/jupyterServerStart.py b/scheduler_files/jupyterServerStart.py index b604867..69b9de8 100755 --- a/scheduler_files/jupyterServerStart.py +++ b/scheduler_files/jupyterServerStart.py @@ -30,6 +30,8 @@ def enclose(s): sConfigFile.close() sid=sconfig['id'] +image_id=sconfig['image_id'] +gpu=sconfig['gpu'] folder=sconfig['folder'] cpu=sconfig['resources']['cpu'] mem=sconfig['resources']['mem'] @@ -42,14 +44,13 @@ def enclose(s): expires=sconfig['expires'] -manifest,url=cf.createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs,namespace,domain,platform) -print(manifest) -print(url) +manifest,url=cf.createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs,namespace,domain,platform,gpu) + subprocess.call(['kubectl', 'apply', '-f', manifest]) -values=[enclose(manifest), enclose(project), enclose(sid), enclose(image), 'NOW()', enclose(user), enclose('https://' + url),"'t'", enclose(expires)] -sql='INSERT INTO jupyter_server(manifest,project,server_id,image,created_at,created_by,url,active, expires_on) VALUES (' + ','.join(values) + ')' +values=[enclose(manifest), enclose(project), enclose(sid), enclose(image), 'NOW()', enclose(user), enclose('https://' + url),"'t'", enclose(expires), enclose(image_id), enclose('spawning')] +sql='INSERT INTO jupyter_server(manifest,project,server_id,image,created_at,created_by,url,active, expires_on, image_id, state) VALUES (' + ','.join(values) + ')' conn=psg.connect(host=host, user=dbuser, password=passwd, dbname=dbname) cur=conn.cursor() diff --git a/views/jupyter/edit_image.php b/views/jupyter/edit_image.php index e4a2f07..7372c41 100644 --- a/views/jupyter/edit_image.php +++ b/views/jupyter/edit_image.php @@ -51,8 +51,10 @@ field($model,'description')?> field($model,'image')?> + field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?> 'btn btn-primary submit-btn'])?> 'btn btn-secondary cancel-btn'])?> + diff --git a/views/jupyter/image_list.php b/views/jupyter/image_list.php index 8c567ca..2f87442 100644 --- a/views/jupyter/image_list.php +++ b/views/jupyter/image_list.php @@ -36,6 +36,8 @@ $back_icon=''; $add_icon=''; +$gpu_en_icon=''; +$gpu_dis_icon=''; Headers::begin() ?> $this->title, @@ -53,8 +55,9 @@
- + + @@ -63,8 +66,9 @@ { ?> - + +
DescriptionDescription Dockerhub imageGPU
description?>description?> image?>gpu) ? $gpu_en_icon : $gpu_dis_icon?> '; diff --git a/views/jupyter/index.php b/views/jupyter/index.php index 92df5b1..8b0159b 100644 --- a/views/jupyter/index.php +++ b/views/jupyter/index.php @@ -41,9 +41,11 @@ [ ], -]) +]); +?> + -
@@ -53,7 +55,8 @@ - + + '; $stop_icon=''; - $access_icon=''; + $start_url=Url::to(['/jupyter/start-server','project'=>$name]); $stop_url=Url::to(['/jupyter/stop-server','project'=>$name]); if (isset($resources['server'])) @@ -88,7 +91,21 @@ $start_class="btn start-btn disabled"; $stop_class="btn stop-btn"; $access_class="btn access-btn"; - $access_url=$resources['server']->url; + if ($resources['server']->state=='running') + { + $access_url=$access_url=$resources['server']->url; + $access_title='Access server.'; + $access_icon=''; + $access_target='_blank'; + } + else + { + $access_url=''; + $access_title='Please wait a few minutes and reload the page to get the access link.'; + $access_icon=''; + $access_target=''; + } + } else { @@ -97,11 +114,14 @@ $stop_class="btn stop-btn disabled"; $access_class="btn access-btn disabled"; $access_url=''; + $access_title='Please start the server'; + $access_icon=''; + $access_target=''; } ?> $start_class, 'title'=> "Start server"])?> $stop_class, 'title'=> "Stop server" ]) : ''?> - $access_class, 'title'=> "Access server", "target"=>"_blank"])?> + $access_class, 'title'=> $access_title, "target"=>$access_target])?> diff --git a/views/jupyter/new_image.php b/views/jupyter/new_image.php index c6696d1..969e2d2 100644 --- a/views/jupyter/new_image.php +++ b/views/jupyter/new_image.php @@ -51,6 +51,7 @@ field($model,'description')?> field($model,'image')?> + field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?> 'btn btn-primary submit-btn'])?> 'btn btn-secondary cancel-btn'])?> diff --git a/views/jupyter/start_server.php b/views/jupyter/start_server.php index 8774132..884121a 100644 --- a/views/jupyter/start_server.php +++ b/views/jupyter/start_server.php @@ -52,7 +52,7 @@ - field($model,'image')->dropDownList($imageDrop)?> + field($model,'image_id')->dropDownList($imageDrop)?> field($model,'password')->passwordInput()?> 'btn btn-success submit-btn'])?> diff --git a/web/js/jupyter/start_server.js b/web/js/jupyter/start_server.js index 57cdae9..67fc6bb 100644 --- a/web/js/jupyter/start_server.js +++ b/web/js/jupyter/start_server.js @@ -2,6 +2,9 @@ $(document).ready(function() { $(".submit-btn").click(function(){ - $("#creatingModal").modal({backdrop: 'static', keyboard: false}); + password=$("#jupyterserver-password").val(); + if (password.length!=0){ + $("#creatingModal").modal({backdrop: 'static', keyboard: false}); + } }); }); \ No newline at end of file
RAM (GB) Image Expires on