Tyler Perkins
ffeadbebff
All checks were successful
continuous-integration/drone/push Build is passing
59 lines
1.6 KiB
YAML
59 lines
1.6 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: vllm-server
|
|
namespace: vllm-ns
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: vllm-inference-server
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: vllm-inference-server
|
|
spec:
|
|
runtimeClassName: nvidia
|
|
containers:
|
|
- name: vllm-inference-server
|
|
image: vllm/vllm-openai:v0.3.3
|
|
imagePullPolicy: IfNotPresent
|
|
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
env:
|
|
- name: HUGGING_FACE_HUB_TOKEN
|
|
value: ""
|
|
- name: TRANSFORMERS_CACHE
|
|
value: /.cache
|
|
- name: shm-size
|
|
value: 1g
|
|
#command: ["/bin/bash", "-c"]
|
|
#args:
|
|
#- while true; do sleep 2600; done
|
|
command: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
|
|
args: ["--model=mistralai/Mistral-7B-v0.1",
|
|
"--gpu-memory-utilization=0.95",
|
|
"--disable-log-requests",
|
|
"--trust-remote-code",
|
|
"--port=8000",
|
|
"--dtype=half",
|
|
"--tensor-parallel-size=1"]
|
|
ports:
|
|
- containerPort: 8000
|
|
name: http
|
|
securityContext:
|
|
runAsUser: 1000
|
|
volumeMounts:
|
|
- mountPath: /dev/shm
|
|
name: dshm
|
|
- mountPath: /.cache
|
|
name: cache
|
|
volumes:
|
|
- name: cache
|
|
emptyDir: {}
|
|
- name: dshm
|
|
emptyDir:
|
|
medium: Memory
|