Fix model gen for openvino (#3341)

- check for valid openvino model should include batch dims
triton-inference-server · Sep 8, 2021 · 12c3063 · 12c3063
1 parent 1b7db2f
commit 12c3063
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 34 deletions.
diff --git a/qa/common/gen_qa_dyna_sequence_models.py b/qa/common/gen_qa_dyna_sequence_models.py
@@ -1381,11 +1381,12 @@ def create_libtorch_modelconfig(models_dir, model_version, max_batch, dtype,
 def create_openvino_modelfile(models_dir, model_version, max_batch, dtype,
                               shape):
 
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, shape, shape,
-                                          shape):
+    batch_dim = [] if max_batch == 0 else [max_batch,]
+    if not tu.validate_for_openvino_model(dtype, dtype, dtype,
+                                          batch_dim + shape, batch_dim + shape,
+                                          batch_dim + shape):
         return
 
-    batch_dim = [] if max_batch == 0 else [max_batch,]
     model_name = tu.get_dyna_sequence_model_name(
         "openvino_nobatch" if max_batch == 0 else "openvino", dtype)
     model_version_dir = models_dir + "/" + model_name + "/" + str(model_version)
@@ -1417,8 +1418,10 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype,
 def create_openvino_modelconfig(models_dir, model_version, max_batch, dtype,
                                 shape):
 
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, shape, shape,
-                                          shape):
+    batch_dim = [] if max_batch == 0 else [max_batch,]
+    if not tu.validate_for_openvino_model(dtype, dtype, dtype,
+                                          batch_dim + shape, batch_dim + shape,
+                                          batch_dim + shape):
         return
 
     model_name = tu.get_dyna_sequence_model_name(

diff --git a/qa/common/gen_qa_identity_models.py b/qa/common/gen_qa_identity_models.py
@@ -500,16 +500,17 @@ def create_libtorch_modelconfig(create_savedmodel, models_dir, model_version,
 def create_openvino_modelfile(models_dir, model_version, io_cnt, max_batch,
                               dtype, shape):
 
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, shape, shape,
-                                          shape):
+    batch_dim = [] if max_batch == 0 else [max_batch,]
+    if not tu.validate_for_openvino_model(dtype, dtype, dtype,
+                                          batch_dim + shape, batch_dim + shape,
+                                          batch_dim + shape):
         return
 
     # Create the model
     model_name = tu.get_zero_model_name(
         "openvino_nobatch" if max_batch == 0 else "openvino", io_cnt, dtype)
     model_version_dir = models_dir + "/" + model_name + "/" + str(model_version)
 
-    batch_dim = [] if max_batch == 0 else [max_batch,]
     openvino_inputs = []
     openvino_outputs = []
     for io_num in range(io_cnt):
@@ -535,8 +536,10 @@ def create_openvino_modelfile(models_dir, model_version, io_cnt, max_batch,
 def create_openvino_modelconfig(models_dir, model_version, io_cnt, max_batch,
                                 dtype, shape):
 
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, shape, shape,
-                                          shape):
+    batch_dim = [] if max_batch == 0 else [max_batch,]
+    if not tu.validate_for_openvino_model(dtype, dtype, dtype,
+                                          batch_dim + shape, batch_dim + shape,
+                                          batch_dim + shape):
         return
 
     # Unpack version policy

diff --git a/qa/common/gen_qa_models.py b/qa/common/gen_qa_models.py
@@ -1320,9 +1320,10 @@ def create_openvino_modelfile(models_dir,
                               output1_dtype,
                               swap=False):
 
-    if not tu.validate_for_openvino_model(input_dtype, output0_dtype,
-                                          output1_dtype, input_shape,
-                                          output0_shape, output1_shape):
+    batch_dim = () if max_batch == 0 else (max_batch,)
+    if not tu.validate_for_openvino_model(
+            input_dtype, output0_dtype, output1_dtype, batch_dim + input_shape,
+            batch_dim + output0_shape, batch_dim + output1_shape):
         return
 
     # Create the model
@@ -1331,9 +1332,12 @@ def create_openvino_modelfile(models_dir,
         output0_dtype, output1_dtype)
     model_version_dir = models_dir + "/" + model_name + "/" + str(model_version)
 
-    batch_dim = () if max_batch == 0 else (max_batch,)
-    in0 = ng.parameter(shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT0")
-    in1 = ng.parameter(shape=batch_dim + input_shape, dtype=input_dtype, name="INPUT1")
+    in0 = ng.parameter(shape=batch_dim + input_shape,
+                       dtype=input_dtype,
+                       name="INPUT0")
+    in1 = ng.parameter(shape=batch_dim + input_shape,
+                       dtype=input_dtype,
+                       name="INPUT1")
 
     r0 = ng.add(in0, in1) if not swap else ng.subtract(in0, in1)
     r1 = ng.subtract(in0, in1) if not swap else ng.add(in0, in1)
@@ -1361,9 +1365,10 @@ def create_openvino_modelconfig(models_dir, max_batch, model_version,
                                 input_dtype, output0_dtype, output1_dtype,
                                 output0_label_cnt, version_policy):
 
-    if not tu.validate_for_openvino_model(input_dtype, output0_dtype,
-                                          output1_dtype, input_shape,
-                                          output0_shape, output1_shape):
+    batch_dim = () if max_batch == 0 else (max_batch,)
+    if not tu.validate_for_openvino_model(
+            input_dtype, output0_dtype, output1_dtype, batch_dim + input_shape,
+            batch_dim + output0_shape, batch_dim + output1_shape):
         return
 
     # Unpack version policy

diff --git a/qa/common/gen_qa_reshape_models.py b/qa/common/gen_qa_reshape_models.py
@@ -736,8 +736,10 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype,
                               input_shapes, output_shapes):
 
     assert len(input_shapes) == len(output_shapes)
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, input_shapes[0],
-                                          input_shapes[0], input_shapes[0]):
+    batch_dim = [] if max_batch == 0 else [max_batch,]
+    if not tu.validate_for_openvino_model(
+            dtype, dtype, dtype, batch_dim + input_shapes[0],
+            batch_dim + input_shapes[0], batch_dim + input_shapes[0]):
         return
 
     io_cnt = len(input_shapes)
@@ -747,19 +749,21 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype,
         "openvino_nobatch" if max_batch == 0 else "openvino", io_cnt, dtype)
     model_version_dir = models_dir + "/" + model_name + "/" + str(model_version)
 
-    batch_dim = [] if max_batch == 0 else [max_batch,]
     openvino_inputs = []
     openvino_outputs = []
     for io_num in range(io_cnt):
         in_name = "INPUT{}".format(io_num)
         out_name = "OUTPUT{}".format(io_num)
         openvino_inputs.append(
-            ng.parameter(shape=batch_dim + input_shapes[io_num], dtype=dtype, name=in_name))
+            ng.parameter(shape=batch_dim + input_shapes[io_num],
+                         dtype=dtype,
+                         name=in_name))
 
         openvino_outputs.append(
             ng.reshape(openvino_inputs[io_num],
-                        batch_dim + output_shapes[io_num],
-                        name=out_name, special_zero=False))
+                       batch_dim + output_shapes[io_num],
+                       name=out_name,
+                       special_zero=False))
 
     function = ng.impl.Function(openvino_outputs, openvino_inputs, model_name)
     ie_network = IENetwork(ng.impl.Function.to_capsule(function))
@@ -780,8 +784,10 @@ def create_openvino_modelconfig(models_dir, model_version, max_batch, dtype,
     assert len(input_shapes) == len(input_model_shapes)
     assert len(output_shapes) == len(output_model_shapes)
     assert len(input_shapes) == len(output_shapes)
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, input_shapes[0],
-                                          input_shapes[0], input_shapes[0]):
+    batch_dim = [] if max_batch == 0 else [max_batch,]
+    if not tu.validate_for_openvino_model(
+            dtype, dtype, dtype, batch_dim + input_shapes[0],
+            batch_dim + input_shapes[0], batch_dim + input_shapes[0]):
         return
 
     io_cnt = len(input_shapes)

diff --git a/qa/common/gen_qa_sequence_models.py b/qa/common/gen_qa_sequence_models.py
@@ -1252,17 +1252,18 @@ def create_libtorch_modelconfig(models_dir, model_version, max_batch, dtype,
 def create_openvino_modelfile(models_dir, model_version, max_batch, dtype,
                               shape):
 
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, shape, shape,
-                                          shape):
+    batch_dim = [] if max_batch == 0 else [
+        max_batch,
+    ]
+    if not tu.validate_for_openvino_model(dtype, dtype, dtype,
+                                          batch_dim + shape, batch_dim + shape,
+                                          batch_dim + shape):
         return
 
     model_name = tu.get_sequence_model_name(
         "openvino_nobatch" if max_batch == 0 else "openvino", dtype)
     model_version_dir = models_dir + "/" + model_name + "/" + str(model_version)
 
-    batch_dim = [] if max_batch == 0 else [
-        max_batch,
-    ]
     in0 = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="INPUT")
     start = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="START")
     ready = ng.parameter(shape=batch_dim + shape, dtype=dtype, name="READY")
@@ -1285,8 +1286,12 @@ def create_openvino_modelfile(models_dir, model_version, max_batch, dtype,
 def create_openvino_modelconfig(models_dir, model_version, max_batch, dtype,
                                 shape):
 
-    if not tu.validate_for_openvino_model(dtype, dtype, dtype, shape, shape,
-                                          shape):
+    batch_dim = [] if max_batch == 0 else [
+        max_batch,
+    ]
+    if not tu.validate_for_openvino_model(dtype, dtype, dtype,
+                                          batch_dim + shape, batch_dim + shape,
+                                          batch_dim + shape):
         return
 
     model_name = tu.get_sequence_model_name(