Create model_repository/vllm_model/config.pbtxt
Browse files
model_repository/vllm_model/config.pbtxt
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: "vllm_model"
|
| 2 |
+
backend: "vllm"
|
| 3 |
+
|
| 4 |
+
# The usage of device is deferred to the vLLM engine
|
| 5 |
+
instance_group [
|
| 6 |
+
{
|
| 7 |
+
count: 1
|
| 8 |
+
kind: KIND_MODEL
|
| 9 |
+
}
|
| 10 |
+
]
|