2023-03-18 22:59:06 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
|
|
|
"os"
|
2023-07-14 23:19:43 +00:00
|
|
|
"os/signal"
|
2023-05-03 21:03:31 +00:00
|
|
|
"path/filepath"
|
2023-07-20 20:10:12 +00:00
|
|
|
"strings"
|
2023-07-14 23:19:43 +00:00
|
|
|
"syscall"
|
2023-05-03 21:03:31 +00:00
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
api "github.com/go-skynet/LocalAI/api"
|
2023-07-14 23:19:43 +00:00
|
|
|
"github.com/go-skynet/LocalAI/api/options"
|
2023-06-26 13:12:43 +00:00
|
|
|
"github.com/go-skynet/LocalAI/internal"
|
2023-04-19 16:43:10 +00:00
|
|
|
model "github.com/go-skynet/LocalAI/pkg/model"
|
2023-04-20 16:33:02 +00:00
|
|
|
"github.com/rs/zerolog"
|
|
|
|
"github.com/rs/zerolog/log"
|
2023-03-18 22:59:06 +00:00
|
|
|
"github.com/urfave/cli/v2"
|
|
|
|
)
|
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
func main() {
|
2023-04-20 16:33:02 +00:00
|
|
|
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
2023-07-14 23:19:43 +00:00
|
|
|
// clean up process
|
|
|
|
go func() {
|
|
|
|
c := make(chan os.Signal, 1) // we need to reserve to buffer size 1, so the notifier are not blocked
|
|
|
|
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
|
|
|
|
<-c
|
|
|
|
os.Exit(1)
|
|
|
|
}()
|
2023-04-20 16:33:02 +00:00
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
path, err := os.Getwd()
|
2023-03-18 22:59:06 +00:00
|
|
|
if err != nil {
|
2023-04-20 16:33:02 +00:00
|
|
|
log.Error().Msgf("error: %s", err.Error())
|
2023-04-19 16:43:10 +00:00
|
|
|
os.Exit(1)
|
2023-03-18 22:59:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
app := &cli.App{
|
2023-06-26 13:12:43 +00:00
|
|
|
Name: "LocalAI",
|
|
|
|
Version: internal.PrintableVersion(),
|
|
|
|
Usage: "OpenAI compatible API for running LLaMA/GPT models locally on CPU with consumer grade hardware.",
|
2023-04-19 16:43:10 +00:00
|
|
|
Flags: []cli.Flag{
|
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "f16",
|
|
|
|
EnvVars: []string{"F16"},
|
|
|
|
},
|
2023-07-20 20:10:12 +00:00
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "autoload-galleries",
|
|
|
|
EnvVars: []string{"AUTOLOAD_GALLERIES"},
|
|
|
|
},
|
2023-04-20 16:33:02 +00:00
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "debug",
|
|
|
|
EnvVars: []string{"DEBUG"},
|
|
|
|
},
|
2023-08-18 23:49:33 +00:00
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "single-active-backend",
|
|
|
|
EnvVars: []string{"SINGLE_ACTIVE_BACKEND"},
|
|
|
|
Usage: "Allow only one backend to be running.",
|
|
|
|
},
|
2023-05-21 12:38:25 +00:00
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "cors",
|
|
|
|
EnvVars: []string{"CORS"},
|
|
|
|
},
|
|
|
|
&cli.StringFlag{
|
|
|
|
Name: "cors-allow-origins",
|
|
|
|
EnvVars: []string{"CORS_ALLOW_ORIGINS"},
|
|
|
|
},
|
2023-04-19 16:43:10 +00:00
|
|
|
&cli.IntFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "threads",
|
|
|
|
Usage: "Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested.",
|
|
|
|
EnvVars: []string{"THREADS"},
|
|
|
|
Value: 4,
|
2023-03-18 22:59:06 +00:00
|
|
|
},
|
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "models-path",
|
|
|
|
Usage: "Path containing models used for inferencing",
|
|
|
|
EnvVars: []string{"MODELS_PATH"},
|
|
|
|
Value: filepath.Join(path, "models"),
|
2023-03-18 22:59:06 +00:00
|
|
|
},
|
2023-06-24 06:18:17 +00:00
|
|
|
&cli.StringFlag{
|
|
|
|
Name: "galleries",
|
|
|
|
Usage: "JSON list of galleries",
|
|
|
|
EnvVars: []string{"GALLERIES"},
|
|
|
|
},
|
2023-05-27 07:26:33 +00:00
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "preload-models",
|
|
|
|
Usage: "A List of models to apply in JSON at start",
|
|
|
|
EnvVars: []string{"PRELOAD_MODELS"},
|
2023-05-27 07:26:33 +00:00
|
|
|
},
|
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "preload-models-config",
|
|
|
|
Usage: "A List of models to apply at startup. Path to a YAML config file",
|
|
|
|
EnvVars: []string{"PRELOAD_MODELS_CONFIG"},
|
2023-05-27 07:26:33 +00:00
|
|
|
},
|
2023-04-27 04:18:18 +00:00
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "config-file",
|
|
|
|
Usage: "Config file",
|
|
|
|
EnvVars: []string{"CONFIG_FILE"},
|
2023-04-27 04:18:18 +00:00
|
|
|
},
|
2023-03-18 22:59:06 +00:00
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "address",
|
|
|
|
Usage: "Bind address for the API server.",
|
|
|
|
EnvVars: []string{"ADDRESS"},
|
|
|
|
Value: ":8080",
|
2023-04-19 16:43:10 +00:00
|
|
|
},
|
2023-05-16 17:32:53 +00:00
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "image-path",
|
|
|
|
Usage: "Image directory",
|
|
|
|
EnvVars: []string{"IMAGE_PATH"},
|
2023-06-22 15:53:10 +00:00
|
|
|
Value: "/tmp/generated/images",
|
|
|
|
},
|
|
|
|
&cli.StringFlag{
|
|
|
|
Name: "audio-path",
|
|
|
|
Usage: "audio directory",
|
|
|
|
EnvVars: []string{"AUDIO_PATH"},
|
|
|
|
Value: "/tmp/generated/audio",
|
2023-05-16 17:32:53 +00:00
|
|
|
},
|
2023-06-01 21:38:52 +00:00
|
|
|
&cli.StringFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "backend-assets-path",
|
|
|
|
Usage: "Path used to extract libraries that are required by some of the backends in runtime.",
|
|
|
|
EnvVars: []string{"BACKEND_ASSETS_PATH"},
|
|
|
|
Value: "/tmp/localai/backend_data",
|
2023-06-01 21:38:52 +00:00
|
|
|
},
|
2023-07-20 20:10:12 +00:00
|
|
|
&cli.StringSliceFlag{
|
|
|
|
Name: "external-grpc-backends",
|
|
|
|
Usage: "A list of external grpc backends",
|
|
|
|
EnvVars: []string{"EXTERNAL_GRPC_BACKENDS"},
|
|
|
|
},
|
2023-04-19 16:43:10 +00:00
|
|
|
&cli.IntFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "context-size",
|
|
|
|
Usage: "Default context size of the model",
|
|
|
|
EnvVars: []string{"CONTEXT_SIZE"},
|
|
|
|
Value: 512,
|
2023-04-19 16:43:10 +00:00
|
|
|
},
|
2023-05-12 08:04:20 +00:00
|
|
|
&cli.IntFlag{
|
2023-06-03 12:25:30 +00:00
|
|
|
Name: "upload-limit",
|
|
|
|
Usage: "Default upload-limit. MB",
|
|
|
|
EnvVars: []string{"UPLOAD_LIMIT"},
|
|
|
|
Value: 15,
|
2023-05-12 08:04:20 +00:00
|
|
|
},
|
2023-08-09 22:06:21 +00:00
|
|
|
&cli.StringSliceFlag{
|
|
|
|
Name: "api-keys",
|
|
|
|
Usage: "List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys.",
|
|
|
|
EnvVars: []string{"API_KEY"},
|
|
|
|
},
|
2023-08-18 19:23:14 +00:00
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "preload-backend-only",
|
|
|
|
Usage: "If set, the api is NOT launched, and only the preloaded models / backends are started. This is intended for multi-node setups.",
|
|
|
|
EnvVars: []string{"PRELOAD_BACKEND_ONLY"},
|
|
|
|
Value: false,
|
|
|
|
},
|
2023-04-19 16:43:10 +00:00
|
|
|
},
|
|
|
|
Description: `
|
|
|
|
LocalAI is a drop-in replacement OpenAI API which runs inference locally.
|
2023-03-18 22:59:06 +00:00
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
Some of the models compatible are:
|
|
|
|
- Vicuna
|
|
|
|
- Koala
|
|
|
|
- GPT4ALL
|
|
|
|
- GPT4ALL-J
|
2023-04-20 22:06:55 +00:00
|
|
|
- Cerebras
|
2023-04-19 16:43:10 +00:00
|
|
|
- Alpaca
|
2023-04-20 22:06:55 +00:00
|
|
|
- StableLM (ggml quantized)
|
2023-03-18 22:59:06 +00:00
|
|
|
|
2023-07-02 09:15:05 +00:00
|
|
|
For a list of compatible model, check out: https://localai.io/model-compatibility/index.html
|
2023-03-18 22:59:06 +00:00
|
|
|
`,
|
2023-04-19 16:43:10 +00:00
|
|
|
UsageText: `local-ai [options]`,
|
2023-07-02 09:15:05 +00:00
|
|
|
Copyright: "Ettore Di Giacinto",
|
2023-03-18 22:59:06 +00:00
|
|
|
Action: func(ctx *cli.Context) error {
|
2023-07-20 20:10:12 +00:00
|
|
|
|
|
|
|
opts := []options.AppOption{
|
2023-07-14 23:19:43 +00:00
|
|
|
options.WithConfigFile(ctx.String("config-file")),
|
|
|
|
options.WithJSONStringPreload(ctx.String("preload-models")),
|
|
|
|
options.WithYAMLConfigPreload(ctx.String("preload-models-config")),
|
|
|
|
options.WithModelLoader(model.NewModelLoader(ctx.String("models-path"))),
|
|
|
|
options.WithContextSize(ctx.Int("context-size")),
|
|
|
|
options.WithDebug(ctx.Bool("debug")),
|
|
|
|
options.WithImageDir(ctx.String("image-path")),
|
|
|
|
options.WithAudioDir(ctx.String("audio-path")),
|
|
|
|
options.WithF16(ctx.Bool("f16")),
|
|
|
|
options.WithStringGalleries(ctx.String("galleries")),
|
|
|
|
options.WithDisableMessage(false),
|
|
|
|
options.WithCors(ctx.Bool("cors")),
|
|
|
|
options.WithCorsAllowOrigins(ctx.String("cors-allow-origins")),
|
|
|
|
options.WithThreads(ctx.Int("threads")),
|
|
|
|
options.WithBackendAssets(backendAssets),
|
|
|
|
options.WithBackendAssetsOutput(ctx.String("backend-assets-path")),
|
2023-07-20 20:10:12 +00:00
|
|
|
options.WithUploadLimitMB(ctx.Int("upload-limit")),
|
2023-08-09 22:06:21 +00:00
|
|
|
options.WithApiKeys(ctx.StringSlice("api-keys")),
|
2023-07-20 20:10:12 +00:00
|
|
|
}
|
|
|
|
|
2023-08-18 23:49:33 +00:00
|
|
|
if ctx.Bool("single-active-backend") {
|
|
|
|
opts = append(opts, options.EnableSingleBackend)
|
|
|
|
}
|
|
|
|
|
2023-07-20 20:10:12 +00:00
|
|
|
externalgRPC := ctx.StringSlice("external-grpc-backends")
|
|
|
|
// split ":" to get backend name and the uri
|
|
|
|
for _, v := range externalgRPC {
|
|
|
|
backend := v[:strings.IndexByte(v, ':')]
|
|
|
|
uri := v[strings.IndexByte(v, ':')+1:]
|
|
|
|
opts = append(opts, options.WithExternalBackend(backend, uri))
|
|
|
|
}
|
|
|
|
|
|
|
|
if ctx.Bool("autoload-galleries") {
|
|
|
|
opts = append(opts, options.EnableGalleriesAutoload)
|
|
|
|
}
|
|
|
|
|
2023-08-18 19:23:14 +00:00
|
|
|
if ctx.Bool("preload-backend-only") {
|
|
|
|
_, _, err := api.Startup(opts...)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-07-20 20:10:12 +00:00
|
|
|
app, err := api.App(opts...)
|
2023-05-30 10:00:30 +00:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return app.Listen(ctx.String("address"))
|
2023-03-18 22:59:06 +00:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
err = app.Run(os.Args)
|
2023-03-18 22:59:06 +00:00
|
|
|
if err != nil {
|
2023-04-20 16:33:02 +00:00
|
|
|
log.Error().Msgf("error: %s", err.Error())
|
2023-03-18 22:59:06 +00:00
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|