Update dtype param

2024-03-30 22:45:48 -04:00 · 2024-03-30 22:45:48 -04:00 · ffeadbebff
commit ffeadbebff
parent 765124c106
1 changed files with 2 additions and 1 deletions
--- a/vllm/vllm-deployment.yaml
+++ b/vllm/vllm-deployment.yaml
@ -21,7 +21,7 @@ spec:

          resources:
            limits:
-              nvidia.com/gpu: 1
+              nvidia.com/gpu: 2
          env:
            - name: HUGGING_FACE_HUB_TOKEN
              value: ""
@ -38,6 +38,7 @@ spec:
                 "--disable-log-requests",
                 "--trust-remote-code",
                 "--port=8000",
+                 "--dtype=half",
                 "--tensor-parallel-size=1"]
          ports:
            - containerPort: 8000