mirror of
https://github.com/mudler/LocalAI.git
synced 2024-06-07 19:40:48 +00:00
8ca671761a
Adding subcommands to do certain actions directly from the command line. I'm starting with the models subcommand allowing you to list models from your galleries and install them. This PR partially fixes #816 My intention is to keep adding other subcommands, but I think this is a good start, and I think this already provides value. Also, I added a new dependency to generate the progress bar in the command line, it is not "needed" but I think is a nice to have to have a cooler interface. Here is a screenshot: 
286 lines
8.4 KiB
Go
286 lines
8.4 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"os/signal"
|
|
"path/filepath"
|
|
"strings"
|
|
"syscall"
|
|
|
|
api "github.com/go-skynet/LocalAI/api"
|
|
"github.com/go-skynet/LocalAI/api/options"
|
|
"github.com/go-skynet/LocalAI/internal"
|
|
"github.com/go-skynet/LocalAI/pkg/gallery"
|
|
model "github.com/go-skynet/LocalAI/pkg/model"
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
progressbar "github.com/schollz/progressbar/v3"
|
|
"github.com/urfave/cli/v2"
|
|
)
|
|
|
|
func main() {
|
|
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
|
// clean up process
|
|
go func() {
|
|
c := make(chan os.Signal, 1) // we need to reserve to buffer size 1, so the notifier are not blocked
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
<-c
|
|
os.Exit(1)
|
|
}()
|
|
|
|
path, err := os.Getwd()
|
|
if err != nil {
|
|
log.Error().Msgf("error: %s", err.Error())
|
|
os.Exit(1)
|
|
}
|
|
|
|
app := &cli.App{
|
|
Name: "LocalAI",
|
|
Version: internal.PrintableVersion(),
|
|
Usage: "OpenAI compatible API for running LLaMA/GPT models locally on CPU with consumer grade hardware.",
|
|
Flags: []cli.Flag{
|
|
&cli.BoolFlag{
|
|
Name: "f16",
|
|
EnvVars: []string{"F16"},
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "autoload-galleries",
|
|
EnvVars: []string{"AUTOLOAD_GALLERIES"},
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "debug",
|
|
EnvVars: []string{"DEBUG"},
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "single-active-backend",
|
|
EnvVars: []string{"SINGLE_ACTIVE_BACKEND"},
|
|
Usage: "Allow only one backend to be running.",
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "cors",
|
|
EnvVars: []string{"CORS"},
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "cors-allow-origins",
|
|
EnvVars: []string{"CORS_ALLOW_ORIGINS"},
|
|
},
|
|
&cli.IntFlag{
|
|
Name: "threads",
|
|
Usage: "Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested.",
|
|
EnvVars: []string{"THREADS"},
|
|
Value: 4,
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "models-path",
|
|
Usage: "Path containing models used for inferencing",
|
|
EnvVars: []string{"MODELS_PATH"},
|
|
Value: filepath.Join(path, "models"),
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "galleries",
|
|
Usage: "JSON list of galleries",
|
|
EnvVars: []string{"GALLERIES"},
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "preload-models",
|
|
Usage: "A List of models to apply in JSON at start",
|
|
EnvVars: []string{"PRELOAD_MODELS"},
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "preload-models-config",
|
|
Usage: "A List of models to apply at startup. Path to a YAML config file",
|
|
EnvVars: []string{"PRELOAD_MODELS_CONFIG"},
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "config-file",
|
|
Usage: "Config file",
|
|
EnvVars: []string{"CONFIG_FILE"},
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "address",
|
|
Usage: "Bind address for the API server.",
|
|
EnvVars: []string{"ADDRESS"},
|
|
Value: ":8080",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "image-path",
|
|
Usage: "Image directory",
|
|
EnvVars: []string{"IMAGE_PATH"},
|
|
Value: "/tmp/generated/images",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "audio-path",
|
|
Usage: "audio directory",
|
|
EnvVars: []string{"AUDIO_PATH"},
|
|
Value: "/tmp/generated/audio",
|
|
},
|
|
&cli.StringFlag{
|
|
Name: "backend-assets-path",
|
|
Usage: "Path used to extract libraries that are required by some of the backends in runtime.",
|
|
EnvVars: []string{"BACKEND_ASSETS_PATH"},
|
|
Value: "/tmp/localai/backend_data",
|
|
},
|
|
&cli.StringSliceFlag{
|
|
Name: "external-grpc-backends",
|
|
Usage: "A list of external grpc backends",
|
|
EnvVars: []string{"EXTERNAL_GRPC_BACKENDS"},
|
|
},
|
|
&cli.IntFlag{
|
|
Name: "context-size",
|
|
Usage: "Default context size of the model",
|
|
EnvVars: []string{"CONTEXT_SIZE"},
|
|
Value: 512,
|
|
},
|
|
&cli.IntFlag{
|
|
Name: "upload-limit",
|
|
Usage: "Default upload-limit. MB",
|
|
EnvVars: []string{"UPLOAD_LIMIT"},
|
|
Value: 15,
|
|
},
|
|
&cli.StringSliceFlag{
|
|
Name: "api-keys",
|
|
Usage: "List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys.",
|
|
EnvVars: []string{"API_KEY"},
|
|
},
|
|
&cli.BoolFlag{
|
|
Name: "preload-backend-only",
|
|
Usage: "If set, the api is NOT launched, and only the preloaded models / backends are started. This is intended for multi-node setups.",
|
|
EnvVars: []string{"PRELOAD_BACKEND_ONLY"},
|
|
Value: false,
|
|
},
|
|
},
|
|
Description: `
|
|
LocalAI is a drop-in replacement OpenAI API which runs inference locally.
|
|
|
|
Some of the models compatible are:
|
|
- Vicuna
|
|
- Koala
|
|
- GPT4ALL
|
|
- GPT4ALL-J
|
|
- Cerebras
|
|
- Alpaca
|
|
- StableLM (ggml quantized)
|
|
|
|
For a list of compatible model, check out: https://localai.io/model-compatibility/index.html
|
|
`,
|
|
UsageText: `local-ai [options]`,
|
|
Copyright: "Ettore Di Giacinto",
|
|
Action: func(ctx *cli.Context) error {
|
|
opts := []options.AppOption{
|
|
options.WithConfigFile(ctx.String("config-file")),
|
|
options.WithJSONStringPreload(ctx.String("preload-models")),
|
|
options.WithYAMLConfigPreload(ctx.String("preload-models-config")),
|
|
options.WithModelLoader(model.NewModelLoader(ctx.String("models-path"))),
|
|
options.WithContextSize(ctx.Int("context-size")),
|
|
options.WithDebug(ctx.Bool("debug")),
|
|
options.WithImageDir(ctx.String("image-path")),
|
|
options.WithAudioDir(ctx.String("audio-path")),
|
|
options.WithF16(ctx.Bool("f16")),
|
|
options.WithStringGalleries(ctx.String("galleries")),
|
|
options.WithDisableMessage(false),
|
|
options.WithCors(ctx.Bool("cors")),
|
|
options.WithCorsAllowOrigins(ctx.String("cors-allow-origins")),
|
|
options.WithThreads(ctx.Int("threads")),
|
|
options.WithBackendAssets(backendAssets),
|
|
options.WithBackendAssetsOutput(ctx.String("backend-assets-path")),
|
|
options.WithUploadLimitMB(ctx.Int("upload-limit")),
|
|
options.WithApiKeys(ctx.StringSlice("api-keys")),
|
|
}
|
|
|
|
if ctx.Bool("single-active-backend") {
|
|
opts = append(opts, options.EnableSingleBackend)
|
|
}
|
|
|
|
externalgRPC := ctx.StringSlice("external-grpc-backends")
|
|
// split ":" to get backend name and the uri
|
|
for _, v := range externalgRPC {
|
|
backend := v[:strings.IndexByte(v, ':')]
|
|
uri := v[strings.IndexByte(v, ':')+1:]
|
|
opts = append(opts, options.WithExternalBackend(backend, uri))
|
|
}
|
|
|
|
if ctx.Bool("autoload-galleries") {
|
|
opts = append(opts, options.EnableGalleriesAutoload)
|
|
}
|
|
|
|
if ctx.Bool("preload-backend-only") {
|
|
_, _, err := api.Startup(opts...)
|
|
return err
|
|
}
|
|
|
|
app, err := api.App(opts...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
return app.Listen(ctx.String("address"))
|
|
},
|
|
Commands: []*cli.Command{
|
|
{
|
|
Name: "models",
|
|
Usage: "List or install models",
|
|
Subcommands: []*cli.Command{
|
|
{
|
|
Name: "list",
|
|
Usage: "List the models avaiable in your galleries",
|
|
Action: func(ctx *cli.Context) error {
|
|
var galleries []gallery.Gallery
|
|
if err := json.Unmarshal([]byte(ctx.String("galleries")), &galleries); err != nil {
|
|
log.Error().Msgf("unable to load galleries: %s", err.Error())
|
|
}
|
|
|
|
models, err := gallery.AvailableGalleryModels(galleries, ctx.String("models-path"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, model := range models {
|
|
if model.Installed {
|
|
fmt.Printf(" * %s@%s (installed)\n", model.Gallery.Name, model.Name)
|
|
} else {
|
|
fmt.Printf(" - %s@%s\n", model.Gallery.Name, model.Name)
|
|
}
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
{
|
|
Name: "install",
|
|
Usage: "Install a model from the gallery",
|
|
Action: func(ctx *cli.Context) error {
|
|
modelName := ctx.Args().First()
|
|
|
|
var galleries []gallery.Gallery
|
|
if err := json.Unmarshal([]byte(ctx.String("galleries")), &galleries); err != nil {
|
|
log.Error().Msgf("unable to load galleries: %s", err.Error())
|
|
}
|
|
|
|
progressBar := progressbar.NewOptions(
|
|
1000,
|
|
progressbar.OptionSetDescription(fmt.Sprintf("downloading model %s", modelName)),
|
|
progressbar.OptionShowBytes(false),
|
|
progressbar.OptionClearOnFinish(),
|
|
)
|
|
progressCallback := func(fileName string, current string, total string, percentage float64) {
|
|
progressBar.Set(int(percentage * 10))
|
|
}
|
|
err = gallery.InstallModelFromGallery(galleries, modelName, ctx.String("models-path"), gallery.GalleryModel{}, progressCallback)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
err = app.Run(os.Args)
|
|
if err != nil {
|
|
log.Error().Msgf("error: %s", err.Error())
|
|
os.Exit(1)
|
|
}
|
|
}
|