apiVersion: apps/v1 kind: Deployment metadata: name: vllm-server namespace: vllm-ns spec: replicas: 1 selector: matchLabels: app: vllm-inference-server template: metadata: labels: app: vllm-inference-server spec: runtimeClassName: nvidia containers: - name: vllm-inference-server image: vllm/vllm-openai:v0.3.3 imagePullPolicy: IfNotPresent resources: limits: nvidia.com/gpu: 1 env: - name: HUGGING_FACE_HUB_TOKEN value: "" - name: TRANSFORMERS_CACHE value: /.cache - name: shm-size value: 1g #command: ["/bin/bash", "-c"] #args: #- while true; do sleep 2600; done command: ["python3", "-m", "vllm.entrypoints.openai.api_server"] args: ["--model=mistralai/Mistral-7B-v0.1", "--gpu-memory-utilization=0.95", "--disable-log-requests", "--trust-remote-code", "--port=8000", "--tensor-parallel-size=1"] ports: - containerPort: 8000 name: http securityContext: runAsUser: 1000 volumeMounts: - mountPath: /dev/shm name: dshm - mountPath: /.cache name: cache volumes: - name: cache emptyDir: {} - name: dshm emptyDir: medium: Memory