Mirror
/
LocalAI
mirror of https://github.com/mudler/LocalAI.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add tensor_parallel_size setting to vllm setting items (#2085) 

Signed-off-by: Taikono-Himazin <kazu@po.harenet.ne.jp>
This commit is contained in:
Taikono-Himazin 2024-04-20 23:37:02 +09:00 committed by GitHub
parent b319ed58b0
commit 03adc1f60d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/backend.proto
View File

@ -177,6 +177,7 @@ message ModelOptions {
bool EnforceEager = 52;
int32 SwapSpace = 53;
int32 MaxModelLen = 54;
int32 TensorParallelSize = 55;
string MMProj = 41;

2
backend/python/vllm/backend_vllm.py
View File

@ -95,6 +95,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
engine_args.trust_remote_code = request.TrustRemoteCode
if request.EnforceEager:
engine_args.enforce_eager = request.EnforceEager
if request.TensorParallelSize:
engine_args.tensor_parallel_size = request.TensorParallelSize
if request.SwapSpace != 0:
engine_args.swap_space = request.SwapSpace
if request.MaxModelLen != 0:

1
core/backend/options.go
View File

@ -74,6 +74,7 @@ func gRPCModelOpts(c config.BackendConfig) *pb.ModelOptions {
EnforceEager: c.EnforceEager,
SwapSpace: int32(c.SwapSpace),
MaxModelLen: int32(c.MaxModelLen),
TensorParallelSize: int32(c.TensorParallelSize),
MMProj: c.MMProj,
YarnExtFactor: c.YarnExtFactor,
YarnAttnFactor: c.YarnAttnFactor,

1
core/config/backend_config.go
View File

@ -140,6 +140,7 @@ type LLMConfig struct {
EnforceEager bool `yaml:"enforce_eager"` // vLLM
SwapSpace int `yaml:"swap_space"` // vLLM
MaxModelLen int `yaml:"max_model_len"` // vLLM
TensorParallelSize int `yaml:"tensor_parallel_size"` // vLLM
MMProj string `yaml:"mmproj"`
RopeScaling string `yaml:"rope_scaling"`