mirror of
https://github.com/mudler/LocalAI.git
synced 2024-06-07 19:40:48 +00:00
fdb45153fe
* feat(llama.cpp): Enable decentralized, distributed inference As https://github.com/mudler/LocalAI/pull/2324 introduced distributed inferencing thanks to @rgerganov implementation in https://github.com/ggerganov/llama.cpp/pull/6829 in upstream llama.cpp, now it is possible to distribute the workload to remote llama.cpp gRPC server. This changeset now uses mudler/edgevpn to establish a secure, distributed network between the nodes using a shared token. The token is generated automatically when starting the server with the `--p2p` flag, and can be used by starting the workers with `local-ai worker p2p-llama-cpp-rpc` by passing the token via environment variable (TOKEN) or with args (--token). As per how mudler/edgevpn works, a network is established between the server and the workers with dht and mdns discovery protocols, the llama.cpp rpc server is automatically started and exposed to the underlying p2p network so the API server can connect on. When the HTTP server is started, it will discover the workers in the network and automatically create the port-forwards to the service locally. Then llama.cpp is configured to use the services. This feature is behind the "p2p" GO_FLAGS Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * go mod tidy Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: add p2p tag Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * better message Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
62 lines
1.9 KiB
Go
62 lines
1.9 KiB
Go
package cli
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"github.com/go-skynet/LocalAI/core/backend"
|
|
cliContext "github.com/go-skynet/LocalAI/core/cli/context"
|
|
"github.com/go-skynet/LocalAI/core/config"
|
|
"github.com/go-skynet/LocalAI/pkg/model"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
type TranscriptCMD struct {
|
|
Filename string `arg:""`
|
|
|
|
Backend string `short:"b" default:"whisper" help:"Backend to run the transcription model"`
|
|
Model string `short:"m" required:"" help:"Model name to run the TTS"`
|
|
Language string `short:"l" help:"Language of the audio file"`
|
|
Threads int `short:"t" default:"1" help:"Number of threads used for parallel computation"`
|
|
ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
|
|
BackendAssetsPath string `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"`
|
|
}
|
|
|
|
func (t *TranscriptCMD) Run(ctx *cliContext.Context) error {
|
|
opts := &config.ApplicationConfig{
|
|
ModelPath: t.ModelsPath,
|
|
Context: context.Background(),
|
|
AssetsDestination: t.BackendAssetsPath,
|
|
}
|
|
|
|
cl := config.NewBackendConfigLoader()
|
|
ml := model.NewModelLoader(opts.ModelPath)
|
|
if err := cl.LoadBackendConfigsFromPath(t.ModelsPath); err != nil {
|
|
return err
|
|
}
|
|
|
|
c, exists := cl.GetBackendConfig(t.Model)
|
|
if !exists {
|
|
return errors.New("model not found")
|
|
}
|
|
|
|
c.Threads = &t.Threads
|
|
|
|
defer func() {
|
|
err := ml.StopAllGRPC()
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("unable to stop all grpc processes")
|
|
}
|
|
}()
|
|
|
|
tr, err := backend.ModelTranscription(t.Filename, t.Language, ml, c, opts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, segment := range tr.Segments {
|
|
fmt.Println(segment.Start.String(), "-", segment.Text)
|
|
}
|
|
return nil
|
|
}
|