mirror of
https://github.com/mudler/LocalAI.git
synced 2024-06-07 19:40:48 +00:00
255748bcba
This PR specifically introduces a `core` folder and moves the following packages over, without any other changes: - `api/backend` - `api/config` - `api/options` - `api/schema` Once this is merged and we confirm there's no regressions, I can migrate over the remaining changes piece by piece to split up application startup, backend services, http, and mqtt as was the goal of the earlier PRs!
93 lines
2.3 KiB
Go
93 lines
2.3 KiB
Go
package base
|
|
|
|
// This is a wrapper to statisfy the GRPC service interface
|
|
// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
|
|
"github.com/go-skynet/LocalAI/core/schema"
|
|
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
|
|
gopsutil "github.com/shirou/gopsutil/v3/process"
|
|
)
|
|
|
|
// Base is a base class for all backends to implement
|
|
// Note: the backends that does not support multiple requests
|
|
// should use SingleThread instead
|
|
type Base struct {
|
|
}
|
|
|
|
func (llm *Base) Locking() bool {
|
|
return false
|
|
}
|
|
|
|
func (llm *Base) Lock() {
|
|
panic("not implemented")
|
|
}
|
|
|
|
func (llm *Base) Unlock() {
|
|
panic("not implemented")
|
|
}
|
|
|
|
func (llm *Base) Busy() bool {
|
|
return false
|
|
}
|
|
|
|
func (llm *Base) Load(opts *pb.ModelOptions) error {
|
|
return fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) Predict(opts *pb.PredictOptions) (string, error) {
|
|
return "", fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) PredictStream(opts *pb.PredictOptions, results chan string) error {
|
|
return fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) Embeddings(opts *pb.PredictOptions) ([]float32, error) {
|
|
return []float32{}, fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) GenerateImage(*pb.GenerateImageRequest) error {
|
|
return fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) AudioTranscription(*pb.TranscriptRequest) (schema.Result, error) {
|
|
return schema.Result{}, fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) TTS(*pb.TTSRequest) error {
|
|
return fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
func (llm *Base) TokenizeString(opts *pb.PredictOptions) (pb.TokenizationResponse, error) {
|
|
return pb.TokenizationResponse{}, fmt.Errorf("unimplemented")
|
|
}
|
|
|
|
// backends may wish to call this to capture the gopsutil info, then enhance with additional memory usage details?
|
|
func (llm *Base) Status() (pb.StatusResponse, error) {
|
|
return pb.StatusResponse{
|
|
Memory: memoryUsage(),
|
|
}, nil
|
|
}
|
|
|
|
func memoryUsage() *pb.MemoryUsageData {
|
|
mud := pb.MemoryUsageData{
|
|
Breakdown: make(map[string]uint64),
|
|
}
|
|
|
|
pid := int32(os.Getpid())
|
|
|
|
backendProcess, err := gopsutil.NewProcess(pid)
|
|
|
|
if err == nil {
|
|
memInfo, err := backendProcess.MemoryInfo()
|
|
if err == nil {
|
|
mud.Total = memInfo.VMS // TEST, but rss seems reasonable first guess. Does include swap, but we might care about that.
|
|
mud.Breakdown["gopsutil-RSS"] = memInfo.RSS
|
|
}
|
|
}
|
|
return &mud
|
|
}
|