Added jupyter gpu capability. Added spawning state

athenarc · Apr 5, 2022 · d0b1adc · d0b1adc
1 parent 5d00611
commit d0b1adc
Show file tree

Hide file tree

Showing 12 changed files with 105 additions and 24 deletions.
diff --git a/controllers/JupyterController.php b/controllers/JupyterController.php
@@ -134,7 +134,12 @@ public function actionIndex()
         $images=[];
         foreach ($img as $i)
         {
-            $images[$i->image]=$i->description;
+            $description=$i->description;
+            if ($i->gpu)
+            {
+                $description.=' (GPU)';
+            }
+            $images[$i->image]=$description;
         }
 
         return $this->render('index',['projects'=>$projects,'images'=>$images]);
@@ -190,11 +195,15 @@ public function actionStartServer($project)
         $imageDrop=[];
         foreach ($images as $image)
         {
-            $imageDrop[$image->image]=$image->description;
+            $description=$image->description;
+            if ($image->gpu)
+            {
+                $description.=" (GPU enabled)";
+            }
+            $imageDrop[$image->id]=$description;
         }
 
         $model = new JupyterServer;
-
         if ($model->load(Yii::$app->request->post()) && $model->validate()) 
         {
             $model->cpu=$quotas['cores'];
@@ -356,6 +365,7 @@ public function actionEditImage($id)
 
         if ($model->load(Yii::$app->request->post()) && $model->validate()) 
         {
+            $model->gpu=($model->gpu==1) ? true : false;
             $model->save();
 
             Yii::$app->session->setFlash('success',"Image $model->image saved!");

diff --git a/database_schema/schema_db.sql b/database_schema/schema_db.sql
@@ -1783,3 +1783,5 @@ alter table workflow add column workflow_type varchar(100);
 alter table workflow_upload add column workflow_type varchar(100);
 create index workflow_type_idx on workflow_upload(workflow_type);
 create index workflow_upload_type_idx on workflow_upload(workflow_type);
+alter table jupyter_images add column gpu boolean default false;
+alter table jupyter_server add column state varchar(20);
diff --git a/models/JupyterImages.php b/models/JupyterImages.php
@@ -28,6 +28,8 @@ public function rules()
     {
         return [
             [['description', 'image'], 'string'],
+            [['gpu'],'boolean'],
+            [['gpu'],'required'],
         ];
     }
 
@@ -40,6 +42,7 @@ public function attributeLabels()
             'id' => 'ID',
             'description' => 'Descriptive name',
             'image' => 'Image on dockerhub',
+            'gpu'   => 'Image uses GPU'
         ];
     }
 }
diff --git a/models/JupyterServer.php b/models/JupyterServer.php
@@ -4,6 +4,7 @@
 
 use Yii;
 use app\models\Softare;
+use app\models\JupyterImages;
 use webvimark\modules\UserManagement\models\User;
 use yii\helpers\Html;
 use yii\httpclient\Client;
@@ -46,7 +47,7 @@ public function rules()
             [['active'], 'boolean'],
             [['manifest', 'project'], 'string', 'max' => 100],
             [['server_id'], 'string', 'max' => 20],
-            [['password','image'],'required']
+            [['password', 'image_id'],'required']
         ];
     }
 
@@ -109,7 +110,28 @@ public static function matchServersWithProjects($projects)
              */
             if (isset($projects[$server->project]))
             {
-                $projects[$server->project]['server']=$server;   
+                $projects[$server->project]['server']=$server;
+                if ($server->state=='spawning')
+                {
+                    try
+                    {
+                        $client = new Client();
+                        $response = $client->createRequest()
+                                ->setMethod('GET')
+                                ->setUrl($server->url)
+                                ->send();
+                        if ($response->getIsOk())
+                        {
+                            $server->state='running';
+                            $server->save(false);
+                        }
+
+                    }
+                    catch (\Exception $e)
+                    {
+
+                    }
+                }
             }
         }
 
@@ -142,6 +164,13 @@ public function startServer()
         $username=User::getCurrentUser()['username'];
         $user=explode('@',$username)[0];
 
+        $image=JupyterImages::find()->where(['id'=>$this->image_id])->one();
+        if (empty($image))
+        {
+            $error='Image not found. Please try again or contact an administrator';
+            return ['',$error];
+        }
+
         $data=[];
         if (file_exists('/data/containerized'))
         {
@@ -153,7 +182,9 @@ public function startServer()
         }
 
         $data['nfs']=$nfs;
-        $data['image']=$this->image;
+        $data['image']=$image->image;
+        $data['image_id']=$this->image_id;
+        $data['gpu']=$image->gpu;
         $data['id']=$sid;
         $data['folder']=Yii::$app->params['tmpFolderPath'] . '/' . $sid . '/';
         $data['mountFolder']=Yii::$app->params['userDataPath'] . $user . '/';
@@ -205,7 +236,7 @@ public function startServer()
                     }
 
                 }
-                catch (yii\httpclient\Exception $e)
+                catch (\Exception $e)
                 {
                     /*
                      * This block is left empty on purpose.
@@ -256,6 +287,8 @@ public function startServer()
             if (!$isDown)
             {
                 $success='Server was started successfully! It can be accessed <a href="' . $server->url . '" target="blank">here</a>.';
+                $server->state='running';
+                $server->save(false);
             }
             else
             {

diff --git a/scheduler_files/jupyterConfig.py b/scheduler_files/jupyterConfig.py
@@ -2,15 +2,14 @@
 from notebook.auth import passwd
 
 
-def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, domain, platform):
+def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, domain, platform, gpu):
     manifest=folder + '/' + sid + '-jupyter.yaml'
     appName=sid + '-jupyter'
 
     deployment={}
     volumes=[]
     containers=[]
     pod={}
-
     pod['replicas']=1
     pod['selector']={'matchLabels':{'app':appName}}
     pod['template']={'metadata':{'labels':{'app':appName}}}
@@ -36,6 +35,9 @@ def createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs, namespace, d
     container['env'].append({'name':'JUPYTER_ENABLE_LAB', 'value': 'yes'})
 
     container['resources']={'limits':{'cpu':str(cpu), 'memory':str(mem) + 'Gi'}, 'requests':{'cpu':str(cpu), 'memory':str(mem) + 'Gi'}}
+    if gpu:
+        container['resources']['limits']['nvidia.com/gpu']=1
+
     volumeMounts=[]
     if nfs=='container':
         vmount={'name': vname, 'mountPath': '/home/jovyan/work', 'subPath': mount.replace('/data/','')}

diff --git a/scheduler_files/jupyterServerStart.py b/scheduler_files/jupyterServerStart.py
@@ -30,6 +30,8 @@ def enclose(s):
 sConfigFile.close()
 
 sid=sconfig['id']
+image_id=sconfig['image_id']
+gpu=sconfig['gpu']
 folder=sconfig['folder']
 cpu=sconfig['resources']['cpu']
 mem=sconfig['resources']['mem']
@@ -42,14 +44,13 @@ def enclose(s):
 expires=sconfig['expires']
 
 
-manifest,url=cf.createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs,namespace,domain,platform)
-print(manifest)
-print(url)
+manifest,url=cf.createServerConfig(sid,cpu,mem,password,folder,image,mount,nfs,namespace,domain,platform,gpu)
+
 
 subprocess.call(['kubectl', 'apply', '-f', manifest])
 
-values=[enclose(manifest), enclose(project), enclose(sid), enclose(image), 'NOW()', enclose(user), enclose('https://' + url),"'t'", enclose(expires)]
-sql='INSERT INTO jupyter_server(manifest,project,server_id,image,created_at,created_by,url,active, expires_on) VALUES (' + ','.join(values) + ')'
+values=[enclose(manifest), enclose(project), enclose(sid), enclose(image), 'NOW()', enclose(user), enclose('https://' + url),"'t'", enclose(expires), enclose(image_id), enclose('spawning')]
+sql='INSERT INTO jupyter_server(manifest,project,server_id,image,created_at,created_by,url,active, expires_on, image_id, state) VALUES (' + ','.join(values) + ')'
 
 conn=psg.connect(host=host, user=dbuser, password=passwd, dbname=dbname)
 cur=conn.cursor()

diff --git a/views/jupyter/edit_image.php b/views/jupyter/edit_image.php
@@ -51,8 +51,10 @@
 
     <?=$form->field($model,'description')?>
     <?=$form->field($model,'image')?>
+    <?= $form->field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?>
 
     <?=Html::submitButton($save_icon . '&nbsp;Save',['class'=> 'btn btn-primary submit-btn'])?>
     <?=Html::submitButton($cancel_icon . '&nbsp;Cancel',['class'=> 'btn btn-secondary cancel-btn'])?>
+
 <?php ActiveForm::end(); ?>
 
diff --git a/views/jupyter/image_list.php b/views/jupyter/image_list.php
@@ -36,6 +36,8 @@
 
 $back_icon='<i class="fas fa-arrow-left"></i>';
 $add_icon='<i class="fas fa-plus"></i>';
+$gpu_en_icon='<i class="fas fa-check"></i>';
+$gpu_dis_icon='<i class="fas fa-times"></i>';
 Headers::begin() ?>
 <?php echo Headers::widget(
 ['title'=>$this->title, 
@@ -53,8 +55,9 @@
 <div class=" table-responsive">
     <table class="table table-striped">
         <thead>
-            <th class="col-md-3">Description</th>
+            <th class="col-md-2">Description</th>
             <th class="col-md-3">Dockerhub image</th>
+            <th class="col-md-1">GPU</th>
             <th class="col-md-1"></th>
         </thead>
         <tbody>
@@ -63,8 +66,9 @@
         {
         ?>
         <tr>
-            <td class="col-md-3"><?=$image->description?></td>
+            <td class="col-md-2"><?=$image->description?></td>
             <td class="col-md-3"><?=$image->image?></td>
+            <td class="col-md-2"><?=($image->gpu) ? $gpu_en_icon : $gpu_dis_icon?></td>
             <td class="col-md-1">
                 <?php
                     $edit_icon='<i class="fas fa-edit"></i>';

diff --git a/views/jupyter/index.php b/views/jupyter/index.php
@@ -41,9 +41,11 @@
     [
 
     ],
-])
+]);
+?>
+<?php 
+    Headers::end();
 ?>
-<?php Headers::end()?>
 
 <div class=" table-responsive">
     <table class="table table-striped">
@@ -53,7 +55,8 @@
             <th class="col-md-1">RAM (GB)</th>
             <th class="col-md-3">Image</th>
             <th class="col-md-2">Expires on</th>
-            <th class="col-md-2"></th>
+
+            <th class="col-md-1"></th>
         </thead>
         <tbody>
         <?php
@@ -79,7 +82,7 @@
                 <?php
                     $start_icon='<i class="fas fa-play"></i>';
                     $stop_icon='<i class="fas fa-stop"></i>';
-                    $access_icon='<i class="fas fa-external-link-alt"></i>';
+
                     $start_url=Url::to(['/jupyter/start-server','project'=>$name]);
                     $stop_url=Url::to(['/jupyter/stop-server','project'=>$name]);
                     if (isset($resources['server']))
@@ -88,7 +91,21 @@
                         $start_class="btn start-btn disabled";
                         $stop_class="btn stop-btn";
                         $access_class="btn access-btn";
-                        $access_url=$resources['server']->url;
+                        if ($resources['server']->state=='running')
+                        {
+                            $access_url=$access_url=$resources['server']->url;
+                            $access_title='Access server.';
+                            $access_icon='<i class="fas fa-external-link-alt"></i>';
+                            $access_target='_blank';
+                        }
+                        else
+                        {
+                            $access_url='';
+                            $access_title='Please wait a few minutes and reload the page to get the access link.';
+                            $access_icon='<i class="fas fa-sync fa-spin"></i>';
+                            $access_target='';
+                        }
+
                     }
                     else
                     {
@@ -97,11 +114,14 @@
                         $stop_class="btn stop-btn disabled";
                         $access_class="btn access-btn disabled";
                         $access_url='';
+                        $access_title='Please start the server';
+                        $access_icon='<i class="fas fa-external-link-alt"></i>';
+                        $access_target='';
                     }
                 ?>
                 <?=$started ? '' : Html::a($start_icon,$start_url,['class'=>$start_class, 'title'=> "Start server"])?>
                 <?=$started ? Html::a($stop_icon,$stop_url,['class'=>$stop_class, 'title'=> "Stop server" ]) : ''?>
-                <?=Html::a($access_icon,$access_url,['class'=>$access_class, 'title'=> "Access server", "target"=>"_blank"])?>
+                <?=Html::a($access_icon,$access_url,['class'=>$access_class, 'title'=> $access_title, "target"=>$access_target])?>
             </td>
 
         </tr>

diff --git a/views/jupyter/new_image.php b/views/jupyter/new_image.php
@@ -51,6 +51,7 @@
 
     <?=$form->field($model,'description')?>
     <?=$form->field($model,'image')?>
+    <?= $form->field($model, 'gpu') -> checkbox(['id'=>'gpu', "uncheck"=>'0']) ?>
 
     <?=Html::submitButton($add_icon . '&nbsp;Add',['class'=> 'btn btn-primary submit-btn'])?>
     <?=Html::submitButton($cancel_icon . '&nbsp;Cancel',['class'=> 'btn btn-secondary cancel-btn'])?>

diff --git a/views/jupyter/start_server.php b/views/jupyter/start_server.php
@@ -52,7 +52,7 @@
 
 <?php $form=ActiveForm::begin($form_params); ?>
 
-    <?=$form->field($model,'image')->dropDownList($imageDrop)?>
+    <?=$form->field($model,'image_id')->dropDownList($imageDrop)?>
     <?=$form->field($model,'password')->passwordInput()?>
 
     <?=Html::submitButton($start_icon . '&nbsp;Start',['class'=> 'btn btn-success submit-btn'])?>

diff --git a/web/js/jupyter/start_server.js b/web/js/jupyter/start_server.js
@@ -2,6 +2,9 @@ $(document).ready(function()
 {
 
     $(".submit-btn").click(function(){
-        $("#creatingModal").modal({backdrop: 'static', keyboard: false});
+        password=$("#jupyterserver-password").val();
+        if (password.length!=0){
+            $("#creatingModal").modal({backdrop: 'static', keyboard: false});
+        }
     });
 });