Skip to content
This repository was archived by the owner on Aug 7, 2025. It is now read-only.

Commit 77ca82d

Browse files
author
Jagadeesh J
authored
fix: kserve fastapi migration issues (#2175)
* fix: kserve fastapi migration issues
1 parent 25f3700 commit 77ca82d

File tree

6 files changed

+61
-113
lines changed

6 files changed

+61
-113
lines changed

‎kubernetes/kserve/kf_request_json/v2/mnist/README.md

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,13 @@ The command will create `mnist.mar` file in current directory
1919

2020
Move the mar file to model-store
2121

22-
```
22+
```bash
2323
sudo mv mnist.mar /mnt/models/model-store
2424
```
2525

2626
and use the following config properties (`/mnt/models/config`)
2727

28-
```
28+
```conf
2929
inference_address=http://0.0.0.0:8085
3030
management_address=http://0.0.0.0:8085
3131
metrics_address=http://0.0.0.0:8082
@@ -51,13 +51,13 @@ Move to `kubernetes/kserve/kf_request_json/v2/mnist`
5151

5252
For bytes input, use [tobytes](tobytes.py) utility.
5353

54-
```
54+
```bash
5555
python tobytes.py 0.png
5656
```
5757

5858
For tensor input, use [totensor](totensor.py) utility
5959

60-
```
60+
```bash
6161
python totensor.py 0.png
6262
```
6363

@@ -66,15 +66,15 @@ python totensor.py 0.png
6666

6767
Start TorchServe
6868

69-
```
69+
```bash
7070
torchserve --start --ts-config /mnt/models/config/config.properties --ncs
7171
```
7272

7373
To test locally, clone TorchServe and move to the following folder `kubernetes/kserve/kserve_wrapper`
7474

7575
Start Kserve
7676

77-
```
77+
```bash
7878
python __main__.py
7979
```
8080

@@ -85,12 +85,12 @@ Navigate to `kubernetes/kserve/kf_request_json/v2/mnist`
8585
Run the following command
8686

8787
```bash
88-
curl -v -H "ContentType: application/json" http://localhost:8080/v2/models/mnist/infer -d @./mnist_v2_bytes.json
88+
curl -v -H "Content-Type: application/json" http://localhost:8080/v2/models/mnist/infer -d @./mnist_v2_bytes.json
8989
```
9090

9191
Expected Output
9292

93-
```bash
93+
```json
9494
{"id": "d3b15cad-50a2-4eaf-80ce-8b0a428bd298", "model_name": "mnist", "model_version": "1.0", "outputs": [{"name": "predict", "shape": [1], "datatype": "INT64", "data": [0]}]}
9595
```
9696

@@ -100,8 +100,8 @@ Expected Output
100100

101101
Run the following command
102102

103-
```
104-
curl -v -H "ContentType: application/json" http://localhost:8080/v2/models/mnist/infer -d @./mnist_v2_tensor.json
103+
```bash
104+
curl -v -H "Content-Type: application/json" http://localhost:8080/v2/models/mnist/infer -d @./mnist_v2_tensor.json
105105
```
106106

107107
Expected output
@@ -115,10 +115,11 @@ Expected output
115115
Run the following command
116116

117117
```bash
118-
curl -v -H "ContentType: application/json" http://localhost:8080/v2/models/mnist/explain -d @./mnist_v2_bytes.json
118+
curl -v -H "Content-Type: application/json" http://localhost:8080/v2/models/mnist/explain -d @./mnist_v2_bytes.json
119119
```
120120

121121
Expected output
122-
```bash
122+
123+
```json
123124
{"id": "d3b15cad-50a2-4eaf-80ce-8b0a428bd298", "model_name": "mnist", "model_version": "1.0", "outputs": [{"name": "explain", "shape": [1, 28, 28], "datatype": "FP64", "data": [-0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, 0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, 0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0040547529196303285, -0.000226128774499257, -0.00012734138382422276, 0.005648369544853077, 0.0089047843954152, 0.002638536593970295, 0.002680245911942565, -0.0026578015819202173, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00024465771891337887, 0.0008218450954311162, 0.01528591767842519, 0.007512832335428859, 0.00709498458333515, 0.0034056686436576803, -0.002091925041823873, -0.0007800293875604465, 0.02299587827540853, 0.019004329367380418, -0.0012529559050418735, -0.0014666116646934577, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.005298396405518712, -0.0007901605729004231, 0.0039060659926479398, 0.023174082126728335, 0.01723791770922474, 0.010867034167828598, 0.003001563229273835, 0.00622421771715703, 0.006120712207087491, 0.01673632965122119, 0.005674718948781803, 0.004344134599735745, -0.0012328422311881568, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0006867353833785289, 0.009772899792600862, -0.0038754932221901437, 0.001798693579973005, 0.001307544047675232, -0.0024510981010352315, -0.0008806773488194292, -0.0, -0.0, -0.00014277890760828639, -0.009322313235257151, 0.020608317727589167, 0.004351394518148479, -0.0007875566214137449, -0.0009075897508410689, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.00022247238084657642, -0.0007829029819622099, 0.0026663695200516055, 0.0009733366691924418, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0004323207980879993, 0.023657171939959983, 0.01069484496100618, -0.0023759529165659743, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.002074797197335781, -0.002320101263777886, -0.001289920656543141, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.007629679763806616, 0.01044862710854819, 0.00025032875474040415, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0003770836745884539, -0.005156369309364184, 0.0012477582083019567, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, -4.442513564501309e-05, 0.010248046436803096, 0.0009971133914441863, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, 0.0, 0.0, -0.0, 0.0004501048922351147, -0.00196305355861066, -0.0006664792277975681, 0.0020157403871024866, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.002214456978582924, 0.008361583668963536, 0.0031401942747203444, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0028943545250037983, -0.0031301382844878753, 0.002113252994616467, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0010321050071136991, 0.008905753948020954, 0.0028464383724280478, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0053052889804602885, -0.0019271100770928186, 0.0012090042664300153, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0011945155805738324, 0.005654442809865844, 0.0020132075147173286, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0014689358119857122, 0.0010743412654248086, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0017047980433136346, 0.0029066051664685937, -0.0007805868937027288, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, 5.541726090138969e-05, 0.0014516115182299915, 0.0002827700518397855, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.001440140782635336, 0.002381249982038837, 0.002146825452068144, -0.0, -0.0, 0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.001150052970321427, 0.0002865015237050364, 0.0029798150346815985, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.001775029606380323, 0.000833985914685474, -0.003770739075457816, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0006093176893524411, -0.00046905781658387527, 0.0034053217440919658, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0007450012183962096, 0.001298767353118675, -0.008499247802184222, -6.145165255574976e-05, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, 0.0, 0.0011809726462884672, -0.0018384763902449712, 0.005411106715800028, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.0021392341817010304, 0.0003259163122540385, -0.005276118905978749, -0.0019509840184772497, -9.545685077687876e-07, 0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0007772404694664217, -0.0001517954537059768, 0.006481484678129392, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 8.098064554131295e-05, -0.0024904264199929506, -0.0020718618328775897, -5.3411287747038166e-05, -0.0004556472202791715, 0.0, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0022750984867578, 0.001716405971437602, 0.0003221344811922982, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0015560282437342534, 9.107229584202956e-05, 0.0008772841867241755, 0.0006502979194500701, -0.004128780661881036, 0.0006030386196211547, 0.0, -0.0, 0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, -0.0, 0.0, 0.0, 0.0013959959731925453, 0.0026791526421029673, 0.002399500793142178, -0.00044960969955281656, 0.003101832495190209, 0.007494535809079955, 0.002864118744003058, -0.003052590549800204, 0.003420222341277871, 0.0014924017873988514, -0.0009357389226494119, 0.0007856229438140384, -0.001843397373255761, 1.6031851430693252e-05, 0.0, 0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, -0.000699901824825285, 0.0043822508549258565, -0.003541931476855951, -0.0028896746311921715, -0.0004873454583246359, -0.006087345141728267, 0.000388224886755815, 0.002533641621974457, -0.004352836429303485, -0.0006079421449756437, -0.003810133409713042, -0.0008284413779488711, 0.0, -0.0, 0.0, 0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0010901530854686326, -0.013135007707490608, 0.0004734520308098294, 0.0020504232707536456, -0.006609452262924153, 0.0023647861306777536, 0.004678920703192049, -0.0018122526857900652, 0.0021375383049022263, 0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, -0.0, 0.0, -0.0, -0.0, -0.0, -0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]}]}
124125
```

‎kubernetes/kserve/kf_request_json/v2/mnist/mnist_v2_bytes.json

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
{
22
"inputs": [
33
{
4-
"data": "iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA10lEQVR4nGNgGFhgy6xVdrCszBaLFN/mr28+/QOCr69DMCSnA8WvHti0acu/fx/10OS0X/975CDDw8DA1PDn/1pBVEmLf3+zocy2X/+8USXt/82Ds+/+m4sqeehfOpw97d9VFDmlO++t4JwQNMm6f6sZcEpee2+DR/I4A05J7tt4JJP+IUsu+ncRp6TxO9RAQJY0XvrvMAuypNNHuCTz8n+PzVEcy3DtqgiY1ptx6t8/ewY0yX9ntoDA63//Xs3hQpMMPPsPAv68qmDAAFKXwHIzMzCl6AoAxXp0QujtP+8AAAAASUVORK5CYII=",
4+
"data": ["iVBORw0KGgoAAAANSUhEUgAAABwAAAAcCAAAAABXZoBIAAAA10lEQVR4nGNgGFhgy6xVdrCszBaLFN/mr28+/QOCr69DMCSnA8WvHti0acu/fx/10OS0X/975CDDw8DA1PDn/1pBVEmLf3+zocy2X/+8USXt/82Ds+/+m4sqeehfOpw97d9VFDmlO++t4JwQNMm6f6sZcEpee2+DR/I4A05J7tt4JJP+IUsu+ncRp6TxO9RAQJY0XvrvMAuypNNHuCTz8n+PzVEcy3DtqgiY1ptx6t8/ewY0yX9ntoDA63//Xs3hQpMMPPsPAv68qmDAAFKXwHIzMzCl6AoAxXp0QujtP+8AAAAASUVORK5CYII="],
55
"datatype": "BYTES",
6-
"name": "312a4eb0-0ca7-4803-a101-a6d2c18486fe",
7-
"shape": -1
6+
"name": "e8d5afed-0a56-4deb-ac9c-352663f51b93",
7+
"shape": [-1]
88
}
99
]
10-
}
10+
}

‎kubernetes/kserve/kf_request_json/v2/mnist/tobytes.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1+
import argparse
12
import base64
23
import json
3-
import argparse
44
import uuid
55

66
parser = argparse.ArgumentParser()
@@ -10,11 +10,20 @@
1010
image = open(args.filename, "rb") # open binary file in read mode
1111
image_read = image.read()
1212
image_64_encode = base64.b64encode(image_read)
13-
bytes_array = image_64_encode.decode("utf-8")
13+
bytes_array = list(image_64_encode.decode("utf-8"))
1414
request = {
15-
"inputs": [{"name": str(uuid.uuid4()), "shape": -1, "datatype": "BYTES", "data": bytes_array}]
15+
"inputs": [
16+
{
17+
"name": str(uuid.uuid4()),
18+
"shape": [-1],
19+
"datatype": "BYTES",
20+
"data": bytes_array,
21+
}
22+
]
1623
}
1724

18-
result_file = "{filename}.{ext}".format(filename=str(args.filename).split(".")[0], ext="json")
25+
result_file = "{filename}.{ext}".format(
26+
filename=str(args.filename).split(".")[0], ext="json"
27+
)
1928
with open(result_file, "w") as outfile:
2029
json.dump(request, outfile, indent=4, sort_keys=True)

‎kubernetes/kserve/kserve_wrapper/TorchserveModel.py

Lines changed: 5 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,19 @@
11
""" The torchserve side inference end-points request are handled to
22
return a KServe side response """
3-
import json
43
import logging
54
import pathlib
6-
from typing import Dict
75

86
import kserve
9-
import tornado.web
7+
from kserve.errors import ModelMissingError
108
from kserve.model import Model as Model
11-
from kserve.model import ModelMissingError
129

1310
logging.basicConfig(level=kserve.constants.KSERVE_LOGLEVEL)
1411

12+
PREDICTOR_URL_FORMAT = PREDICTOR_V2_URL_FORMAT = "http://{0}/predictions/{1}"
13+
EXPLAINER_URL_FORMAT = EXPLAINER_V2_URL_FORMAT = "http://{0}/explanations/{1}"
1514
REGISTER_URL_FORMAT = "{0}/models?initial_workers=1&url={1}"
1615
UNREGISTER_URL_FORMAT = "{0}/models/{1}"
1716

18-
PREDICTOR_URL_FORMAT = "http://{0}/v1/models/{1}:predict"
19-
EXPLAINER_URL_FORMAT = "http://{0}/v1/models/{1}:explain"
20-
2117

2218
class TorchserveModel(Model):
2319
"""The torchserve side inference and explain end-points requests are handled to
@@ -49,76 +45,9 @@ def __init__(self, name, inference_address, management_address, model_dir):
4945
self.management_address = management_address
5046
self.model_dir = model_dir
5147

52-
logging.info("kfmodel Predict URL set to %s", self.predictor_host)
48+
logging.info("Predict URL set to %s", self.predictor_host)
5349
self.explainer_host = self.predictor_host
54-
logging.info("kfmodel Explain URL set to %s", self.explainer_host)
55-
56-
async def predict(self, request: Dict) -> Dict:
57-
"""The predict method is called when we hit the inference endpoint and handles
58-
the inference request and response from the Torchserve side and passes it on
59-
to the KServe side.
60-
61-
Args:
62-
request (Dict): Input request from the http client side.
63-
64-
Raises:
65-
NotImplementedError: If the predictor host on the KServe side is not
66-
available.
67-
68-
tornado.web.HTTPError: If there is a bad response from the http client.
69-
70-
Returns:
71-
Dict: The Response from the input from the inference endpoint.
72-
"""
73-
if not self.predictor_host:
74-
raise NotImplementedError
75-
logging.debug("kfmodel predict request is %s", json.dumps(request))
76-
logging.info("PREDICTOR_HOST : %s", self.predictor_host)
77-
headers = {"Content-Type": "application/json; charset=UTF-8"}
78-
response = await self._http_client.fetch(
79-
PREDICTOR_URL_FORMAT.format(self.predictor_host, self.name),
80-
method="POST",
81-
request_timeout=self.timeout,
82-
headers=headers,
83-
body=json.dumps(request),
84-
)
85-
86-
if response.code != 200:
87-
raise tornado.web.HTTPError(status_code=response.code, reason=response.body)
88-
return json.loads(response.body)
89-
90-
async def explain(self, request: Dict) -> Dict:
91-
"""The predict method is called when we hit the explain endpoint and handles the
92-
explain request and response from the Torchserve side and passes it on to the
93-
KServe side.
94-
95-
Args:
96-
request (Dict): Input request from the http client side.
97-
98-
Raises:
99-
NotImplementedError: If the predictor host on the KServe side is not
100-
available.
101-
102-
tornado.web.HTTPError: If there is a bad response from the http client.
103-
104-
Returns:
105-
Dict: The Response from the input from the explain endpoint.
106-
"""
107-
if self.explainer_host is None:
108-
raise NotImplementedError
109-
logging.info("kfmodel explain request is %s", json.dumps(request))
110-
logging.info("EXPLAINER_HOST : %s", self.explainer_host)
111-
headers = {"Content-Type": "application/json; charset=UTF-8"}
112-
response = await self._http_client.fetch(
113-
EXPLAINER_URL_FORMAT.format(self.explainer_host, self.name),
114-
method="POST",
115-
request_timeout=self.timeout,
116-
headers=headers,
117-
body=json.dumps(request),
118-
)
119-
if response.code != 200:
120-
raise tornado.web.HTTPError(status_code=response.code, reason=response.body)
121-
return json.loads(response.body)
50+
logging.info("Explain URL set to %s", self.explainer_host)
12251

12352
def load(self) -> bool:
12453
"""This method validates model availabilty in the model directory

‎kubernetes/kserve/kserve_wrapper/__main__.py

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
DEFAULT_MODEL_NAME = "model"
1313
DEFAULT_INFERENCE_ADDRESS = "http://127.0.0.1:8085"
1414
INFERENCE_PORT = "8085"
15-
DEFAULT_MANAGEMENT_ADDRESS = "http://127.0.0.1:8081"
15+
DEFAULT_MANAGEMENT_ADDRESS = "http://127.0.0.1:8085"
1616

1717
DEFAULT_MODEL_STORE = "/mnt/models/model-store"
1818
CONFIG_PATH = "/mnt/models/config/config.properties"
@@ -31,10 +31,8 @@ def parse_config():
3131
keys = {}
3232

3333
with open(CONFIG_PATH) as f:
34-
3534
for line in f:
3635
if separator in line:
37-
3836
# Find the name and value by splitting the string
3937
name, value = line.split(separator, 1)
4038

@@ -79,13 +77,11 @@ def parse_config():
7977

8078

8179
if __name__ == "__main__":
82-
8380
model_names, inference_address, management_address, model_dir = parse_config()
8481

8582
models = []
8683

8784
for model_name in model_names:
88-
8985
model = TorchserveModel(
9086
model_name, inference_address, management_address, model_dir
9187
)
@@ -100,5 +96,5 @@ def parse_config():
10096
ModelServer(
10197
registered_models=registeredModels,
10298
http_port=8080,
103-
grpc_port=7070,
99+
grpc_port=8081,
104100
).start(models)

0 commit comments

Comments
 (0)