Mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2024-06-07 19:40:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(config): pass by config options, respect defaults (#1878)

Browse Source 
This bug had the unpleasant effect that it ignored defaults passed by
the CLI. For instance threads could be changed only via model config
file.
This commit is contained in:
Ettore Di Giacinto 2024-03-22 20:55:11 +01:00 committed by GitHub
parent dd84c29a3d
commit 600152df23
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 33 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
core/config/application_config.go
View File

@ -258,6 +258,21 @@ func WithApiKeys(apiKeys []string) AppOption {
}
}
// ToConfigLoaderOptions returns a slice of ConfigLoader Option.
// Some options defined at the application level are going to be passed as defaults for
// all the configuration for the models.
// This includes for instance the context size or the number of threads.
// If a model doesn't set configs directly to the config model file
// it will use the defaults defined here.
func (o *ApplicationConfig) ToConfigLoaderOptions() []ConfigLoaderOption {
return []ConfigLoaderOption{
LoadOptionContextSize(o.ContextSize),
LoadOptionDebug(o.Debug),
LoadOptionF16(o.F16),
LoadOptionThreads(o.Threads),
}
}
// func WithMetrics(meter *metrics.Metrics) AppOption {
// return func(o *StartupOptions) {
// o.Metrics = meter

25
core/config/backend_config.go
View File

@ -188,7 +188,14 @@ func (c *BackendConfig) FunctionToCall() string {
return c.functionCallNameString
}
func (cfg *BackendConfig) SetDefaults(debug bool, threads, ctx int, f16 bool) {
func (cfg *BackendConfig) SetDefaults(opts ...ConfigLoaderOption) {
lo := &LoadOptions{}
lo.Apply(opts...)
ctx := lo.ctxSize
threads := lo.threads
f16 := lo.f16
debug := lo.debug
defaultTopP := 0.7
defaultTopK := 80
defaultTemp := 0.9
@ -333,9 +340,6 @@ func (lo *LoadOptions) Apply(options ...ConfigLoaderOption) {
// Load a config file for a model
func (cl *BackendConfigLoader) LoadBackendConfigFileByName(modelName, modelPath string, opts ...ConfigLoaderOption) (*BackendConfig, error) {
lo := &LoadOptions{}
lo.Apply(opts...)
// Load a config file if present after the model name
cfg := &BackendConfig{
PredictionOptions: schema.PredictionOptions{
@ -350,7 +354,9 @@ func (cl *BackendConfigLoader) LoadBackendConfigFileByName(modelName, modelPath
// Try loading a model config file
modelConfig := filepath.Join(modelPath, modelName+".yaml")
if _, err := os.Stat(modelConfig); err == nil {
if err := cl.LoadBackendConfig(modelConfig); err != nil {
if err := cl.LoadBackendConfig(
modelConfig, opts...,
); err != nil {
return nil, fmt.Errorf("failed loading model config (%s) %s", modelConfig, err.Error())
}
cfgExisting, exists = cl.GetBackendConfig(modelName)
@ -360,7 +366,7 @@ func (cl *BackendConfigLoader) LoadBackendConfigFileByName(modelName, modelPath
}
}
cfg.SetDefaults(lo.debug, lo.threads, lo.ctxSize, lo.f16)
cfg.SetDefaults(opts...)
return cfg, nil
}
@ -371,9 +377,6 @@ func NewBackendConfigLoader() *BackendConfigLoader {
}
}
func ReadBackendConfigFile(file string, opts ...ConfigLoaderOption) ([]*BackendConfig, error) {
lo := &LoadOptions{}
lo.Apply(opts...)
c := &[]*BackendConfig{}
f, err := os.ReadFile(file)
if err != nil {
@ -384,7 +387,7 @@ func ReadBackendConfigFile(file string, opts ...ConfigLoaderOption) ([]*BackendC
}
for _, cc := range *c {
cc.SetDefaults(lo.debug, lo.threads, lo.ctxSize, lo.f16)
cc.SetDefaults(opts...)
}
return *c, nil
@ -403,7 +406,7 @@ func ReadBackendConfig(file string, opts ...ConfigLoaderOption) (*BackendConfig,
return nil, fmt.Errorf("cannot unmarshal config file: %w", err)
}
c.SetDefaults(lo.debug, lo.threads, lo.ctxSize, lo.f16)
c.SetDefaults(opts...)
return c, nil
}

6
core/startup/startup.go
View File

@ -58,12 +58,14 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
cl := config.NewBackendConfigLoader()
ml := model.NewModelLoader(options.ModelPath)
if err := cl.LoadBackendConfigsFromPath(options.ModelPath); err != nil {
configLoaderOpts := options.ToConfigLoaderOptions()
if err := cl.LoadBackendConfigsFromPath(options.ModelPath, configLoaderOpts...); err != nil {
log.Error().Msgf("error loading config files: %s", err.Error())
}
if options.ConfigFile != "" {
if err := cl.LoadBackendConfigFile(options.ConfigFile); err != nil {
if err := cl.LoadBackendConfigFile(options.ConfigFile, configLoaderOpts...); err != nil {
log.Error().Msgf("error loading config file: %s", err.Error())
}
}