Mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2024-06-07 19:40:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: set default batch size (#597)

Browse Source
This commit is contained in:
Ettore Di Giacinto 2023-06-14 19:09:27 +02:00 committed by GitHub
parent 1b7990d5d9
commit 10ddd72b58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/prediction.go
View File

@ -44,6 +44,8 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
llamaOpts = append(llamaOpts, llama.SetTensorSplit(c.TensorSplit))
if c.Batch != 0 {
llamaOpts = append(llamaOpts, llama.SetNBatch(c.Batch))
} else {
llamaOpts = append(llamaOpts, llama.SetNBatch(512))
}
return llamaOpts