mirror of
https://github.com/mudler/LocalAI.git
synced 2024-06-07 19:40:48 +00:00
bc8f648a91
The default sampler on some models don't return enough candidates which leads to a false sense of randomness. Tracing back the code it looks that with the temperature sampler there might not be enough candidates to pick from, and since the seed and "randomness" take effect while picking a good candidate this yields to the same results over and over. Fixes https://github.com/mudler/LocalAI/issues/1723 by updating the examples and documentation to use mirostat instead.
31 lines
866 B
YAML
31 lines
866 B
YAML
name: dolphin-mixtral-8x7b
|
|
mmap: true
|
|
parameters:
|
|
model: huggingface://TheBloke/dolphin-2.5-mixtral-8x7b-GGUF/dolphin-2.5-mixtral-8x7b.Q2_K.gguf
|
|
temperature: 0.5
|
|
top_k: 40
|
|
top_p: 0.95
|
|
seed: -1
|
|
mirostat: 2
|
|
mirostat_eta: 1.0
|
|
mirostat_tau: 1.0
|
|
template:
|
|
chat_message: |
|
|
<|im_start|>{{if eq .RoleName "assistant"}}assistant{{else if eq .RoleName "system"}}system{{else if eq .RoleName "user"}}user{{end}}
|
|
{{if .Content}}{{.Content}}{{end}}<|im_end|>
|
|
chat: |
|
|
{{.Input}}
|
|
<|im_start|>assistant
|
|
completion: |
|
|
{{.Input}}
|
|
context_size: 4096
|
|
f16: true
|
|
stopwords:
|
|
- <|im_end|>
|
|
gpu_layers: 90
|
|
|
|
usage: |
|
|
curl http://localhost:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
|
|
"model": "dolphin-mixtral-8x7b",
|
|
"messages": [{"role": "user", "content": "How are you doing?", "temperature": 0.1}]
|
|
}' |