From 00ccb8d4f110514da040453852c78dd7ae3e9a76 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 29 Jul 2023 10:40:56 +0200 Subject: [PATCH] fix: set default rope freq base to 10000 during model load Signed-off-by: Ettore Di Giacinto --- pkg/grpc/llm/llama/llama.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pkg/grpc/llm/llama/llama.go b/pkg/grpc/llm/llama/llama.go index 1cc2ec7b..18e481ef 100644 --- a/pkg/grpc/llm/llama/llama.go +++ b/pkg/grpc/llm/llama/llama.go @@ -17,9 +17,20 @@ type LLM struct { } func (llm *LLM) Load(opts *pb.ModelOptions) error { + + ropeFreqBase := float32(10000) + ropeFreqScale := float32(1) + + if opts.RopeFreqBase != 0 { + ropeFreqBase = opts.RopeFreqBase + } + if opts.RopeFreqScale != 0 { + ropeFreqScale = opts.RopeFreqScale + } + llamaOpts := []llama.ModelOption{ - llama.WithRopeFreqBase(opts.RopeFreqBase), - llama.WithRopeFreqScale(opts.RopeFreqScale), + llama.WithRopeFreqBase(ropeFreqBase), + llama.WithRopeFreqScale(ropeFreqScale), } if opts.ContextSize != 0 {