Introduce defaultTimeout for predictor and apply in tfserving (kubefl…

…ow#1149) * Introduce serving defaultTimeout in configmap and apply in Tensorflow model server * Add tests for serving defaultTimeout in tfserving * typo in test configmap * Run gen-crd-apidocs to add the serving defaultTimeout of tfserving * Add description for serving defaultTimeout in apidocs * Change defaultTimeout on v1alpha2 to int64 instead of string * get timeout from user input or else use the component default timeout * define defaultTimeout of v1alpha2 in seconds and improve tests based on the review * run the apidocs to get the correct defaultTimeout description * set timeout locally
magdalenakuhn17 · Dec 18, 2020 · 42d044d · 42d044d
1 parent 121e3dd
commit 42d044d
Show file tree

Hide file tree

Showing 14 changed files with 290 additions and 12 deletions.
diff --git a/config/configmap/inferenceservice.yaml b/config/configmap/inferenceservice.yaml
@@ -10,6 +10,7 @@ data:
             "image": "tensorflow/serving",
             "defaultImageVersion": "1.14.0",
             "defaultGpuImageVersion": "1.14.0-gpu",
+            "defaultTimeout": "60",
             "supportedFrameworks": [
               "tensorflow"
             ],

diff --git a/config/overlays/test/configmap/inferenceservice.yaml b/config/overlays/test/configmap/inferenceservice.yaml
@@ -9,7 +9,8 @@ data:
         "tensorflow": {
             "image": "tensorflow/serving",
             "defaultImageVersion": "1.14.0",
-            "defaultGpuImageVersion": "1.14.0-gpu"
+            "defaultGpuImageVersion": "1.14.0-gpu",
+            "defaultTimeout": "60"
         },
         "onnx": {
             "image": "mcr.microsoft.com/onnxruntime/server",

diff --git a/docs/apis/README.md b/docs/apis/README.md
@@ -10,6 +10,92 @@
 </p>
 Resource Types:
 <ul></ul>
+<h3 id="serving.kubeflow.org/v1alpha2.AIXExplainerSpec">AIXExplainerSpec
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#serving.kubeflow.org/v1alpha2.ExplainerSpec">ExplainerSpec</a>)
+</p>
+<p>
+<p>AIXExplainerSpec defines the arguments for configuring an AIX Explanation Server</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>type</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.AIXExplainerType">
+AIXExplainerType
+</a>
+</em>
+</td>
+<td>
+<p>The type of AIX explainer</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>storageUri</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>The location of a trained explanation model</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>runtimeVersion</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>Defaults to latest AIX Version</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>resources</code></br>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements
+</a>
+</em>
+</td>
+<td>
+<p>Defaults to requests and limits of 1CPU, 2Gb MEM.</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>config</code></br>
+<em>
+map[string]string
+</em>
+</td>
+<td>
+<p>Inline custom parameter settings for explainer</p>
+</td>
+</tr>
+</tbody>
+</table>
+<h3 id="serving.kubeflow.org/v1alpha2.AIXExplainerType">AIXExplainerType
+(<code>string</code> alias)</p></h3>
+<p>
+(<em>Appears on:</em>
+<a href="#serving.kubeflow.org/v1alpha2.AIXExplainerSpec">AIXExplainerSpec</a>)
+</p>
+<p>
+</p>
 <h3 id="serving.kubeflow.org/v1alpha2.AlibiExplainerSpec">AlibiExplainerSpec
 </h3>
 <p>
@@ -422,6 +508,19 @@ AlibiExplainerSpec
 </tr>
 <tr>
 <td>
+<code>aix</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.AIXExplainerSpec">
+AIXExplainerSpec
+</a>
+</em>
+</td>
+<td>
+<p>Spec for AIX explainer</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>custom</code></br>
 <em>
 <a href="#serving.kubeflow.org/v1alpha2.CustomSpec">
@@ -478,6 +577,18 @@ ExplainerConfig
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>aix</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.ExplainerConfig">
+ExplainerConfig
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="serving.kubeflow.org/v1alpha2.InferenceService">InferenceService
@@ -897,6 +1008,60 @@ Kubernetes core/v1.ResourceRequirements
 </tr>
 </tbody>
 </table>
+<h3 id="serving.kubeflow.org/v1alpha2.PMMLSpec">PMMLSpec
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#serving.kubeflow.org/v1alpha2.PredictorSpec">PredictorSpec</a>)
+</p>
+<p>
+<p>PMMLSpec defines arguments for configuring PMML model serving.</p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>storageUri</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>The URI of the trained model which contains model.pmml</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>runtimeVersion</code></br>
+<em>
+string
+</em>
+</td>
+<td>
+<p>PMML KFServer docker image version which defaults to latest release</p>
+</td>
+</tr>
+<tr>
+<td>
+<code>resources</code></br>
+<em>
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#resourcerequirements-v1-core">
+Kubernetes core/v1.ResourceRequirements
+</a>
+</em>
+</td>
+<td>
+<p>Defaults to requests and limits of 1CPU, 2Gb MEM.</p>
+</td>
+</tr>
+</tbody>
+</table>
 <h3 id="serving.kubeflow.org/v1alpha2.Predictor">Predictor
 </h3>
 <p>
@@ -905,6 +1070,7 @@ Kubernetes core/v1.ResourceRequirements
 </h3>
 <p>
 (<em>Appears on:</em>
+<a href="#serving.kubeflow.org/v1alpha2.PredictorProtocols">PredictorProtocols</a>, 
 <a href="#serving.kubeflow.org/v1alpha2.PredictorsConfig">PredictorsConfig</a>)
 </p>
 <p>
@@ -947,6 +1113,59 @@ string
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>defaultTimeout,string</code></br>
+<em>
+int64
+</em>
+</td>
+<td>
+<p>Default timeout of predictor for serving a request, in seconds</p>
+</td>
+</tr>
+</tbody>
+</table>
+<h3 id="serving.kubeflow.org/v1alpha2.PredictorProtocols">PredictorProtocols
+</h3>
+<p>
+(<em>Appears on:</em>
+<a href="#serving.kubeflow.org/v1alpha2.PredictorsConfig">PredictorsConfig</a>)
+</p>
+<p>
+</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>
+<code>v1</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.PredictorConfig">
+PredictorConfig
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
+<tr>
+<td>
+<code>v2</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.PredictorConfig">
+PredictorConfig
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="serving.kubeflow.org/v1alpha2.PredictorSpec">PredictorSpec
@@ -1060,6 +1279,19 @@ PyTorchSpec
 </tr>
 <tr>
 <td>
+<code>pmml</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.PMMLSpec">
+PMMLSpec
+</a>
+</em>
+</td>
+<td>
+<p>Spec for PMML predictor</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>DeploymentSpec</code></br>
 <em>
 <a href="#serving.kubeflow.org/v1alpha2.DeploymentSpec">
@@ -1119,8 +1351,8 @@ PredictorConfig
 <td>
 <code>xgboost</code></br>
 <em>
-<a href="#serving.kubeflow.org/v1alpha2.PredictorConfig">
-PredictorConfig
+<a href="#serving.kubeflow.org/v1alpha2.PredictorProtocols">
+PredictorProtocols
 </a>
 </em>
 </td>
@@ -1131,8 +1363,8 @@ PredictorConfig
 <td>
 <code>sklearn</code></br>
 <em>
-<a href="#serving.kubeflow.org/v1alpha2.PredictorConfig">
-PredictorConfig
+<a href="#serving.kubeflow.org/v1alpha2.PredictorProtocols">
+PredictorProtocols
 </a>
 </em>
 </td>
@@ -1163,6 +1395,18 @@ PredictorConfig
 <td>
 </td>
 </tr>
+<tr>
+<td>
+<code>pmml</code></br>
+<em>
+<a href="#serving.kubeflow.org/v1alpha2.PredictorConfig">
+PredictorConfig
+</a>
+</em>
+</td>
+<td>
+</td>
+</tr>
 </tbody>
 </table>
 <h3 id="serving.kubeflow.org/v1alpha2.PyTorchSpec">PyTorchSpec

diff --git a/pkg/apis/serving/v1alpha2/configmap.go b/pkg/apis/serving/v1alpha2/configmap.go
@@ -59,6 +59,8 @@ type PredictorConfig struct {
 
 	DefaultImageVersion    string `json:"defaultImageVersion"`
 	DefaultGpuImageVersion string `json:"defaultGpuImageVersion"`
+	// Default timeout of predictor for serving a request, in seconds
+	DefaultTimeout int64 `json:"defaultTimeout,string,omitempty"`
 }
 
 // +kubebuilder:object:generate=false

diff --git a/pkg/apis/serving/v1alpha2/framework_tensorflow.go b/pkg/apis/serving/v1alpha2/framework_tensorflow.go
@@ -46,6 +46,7 @@ func (t *TensorflowSpec) GetContainer(modelName string, parallelism int, config
 		"--rest_api_port=" + TensorflowServingRestPort,
 		"--model_name=" + modelName,
 		"--model_base_path=" + constants.DefaultModelLocalMountPath,
+		"--rest_api_timeout_in_ms=" + fmt.Sprint(1000*config.Predictors.Tensorflow.DefaultTimeout),
 	}
 
 	return &v1.Container{

diff --git a/pkg/apis/serving/v1alpha2/framework_tensorflow_test.go b/pkg/apis/serving/v1alpha2/framework_tensorflow_test.go
@@ -90,6 +90,7 @@ func TestTensorflowContainer(t *testing.T) {
 			Tensorflow: PredictorConfig{
 				ContainerImage:      "someOtherImage",
 				DefaultImageVersion: "0.1.0",
+				DefaultTimeout:      60,
 			},
 		},
 	}
@@ -110,6 +111,7 @@ func TestTensorflowContainer(t *testing.T) {
 			"--rest_api_port=" + TensorflowServingRestPort,
 			"--model_name=someName",
 			"--model_base_path=/mnt/models",
+			"--rest_api_timeout_in_ms=60000",
 		},
 		LivenessProbe: &v1.Probe{
 			Handler: v1.Handler{

diff --git a/pkg/apis/serving/v1alpha2/v1alpha2_suite_test.go b/pkg/apis/serving/v1alpha2/v1alpha2_suite_test.go
@@ -74,7 +74,8 @@ func TestMain(m *testing.M) {
 			"tensorflow" : {
 				"image" : "tensorflow/serving",
 				"defaultImageVersion": "latest",
-				"defaultGPUImageVersion": "latest-gpu"
+				"defaultGPUImageVersion": "latest-gpu",
+				"defaultTimeout": "60"
 			},
 			"sklearn" : {
                 "v1": {

diff --git a/pkg/apis/serving/v1beta1/configmap.go b/pkg/apis/serving/v1beta1/configmap.go
@@ -60,6 +60,8 @@ type PredictorConfig struct {
 	DefaultImageVersion string `json:"defaultImageVersion"`
 	// default predictor docker image version on gpu
 	DefaultGpuImageVersion string `json:"defaultGpuImageVersion"`
+	// Default timeout of predictor for serving a request, in seconds
+	DefaultTimeout int64 `json:"defaultTimeout,string,omitempty"`
 }
 
 // +kubebuilder:object:generate=false