From 295f3030a9f76f7f36705abb760a2cd351d2b4d8 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Wed, 14 Jun 2023 19:33:20 +0200 Subject: [PATCH] feat: add typical_p to model parameters (#598) Signed-off-by: mudler --- api/config.go | 4 ++++ api/openai.go | 2 ++ api/prediction.go | 1 + 3 files changed, 7 insertions(+) diff --git a/api/config.go b/api/config.go index afd5d468..50aee20c 100644 --- a/api/config.go +++ b/api/config.go @@ -236,6 +236,10 @@ func updateConfig(config *Config, input *OpenAIRequest) { config.MirostatTAU = input.MirostatTAU } + if input.TypicalP != 0 { + config.TypicalP = input.TypicalP + } + switch inputs := input.Input.(type) { case string: if inputs != "" { diff --git a/api/openai.go b/api/openai.go index 80172010..b742b404 100644 --- a/api/openai.go +++ b/api/openai.go @@ -133,6 +133,8 @@ type OpenAIRequest struct { // Image (not supported by OpenAI) Mode int `json:"mode"` Step int `json:"step"` + + TypicalP float64 `json:"typical_p" yaml:"typical_p"` } func defaultRequest(modelFile string) OpenAIRequest { diff --git a/api/prediction.go b/api/prediction.go index 7a49a57e..e7a8ca4a 100644 --- a/api/prediction.go +++ b/api/prediction.go @@ -238,6 +238,7 @@ func buildLLamaPredictOptions(c Config, modelPath string) []llama.PredictOption predictOptions = append(predictOptions, llama.SetPredictionMainGPU(c.MainGPU)) predictOptions = append(predictOptions, llama.SetPredictionTensorSplit(c.TensorSplit)) predictOptions = append(predictOptions, llama.SetTailFreeSamplingZ(c.TFZ)) + predictOptions = append(predictOptions, llama.SetTypicalP(c.TypicalP)) return predictOptions }