2023-04-11 22:02:39 +00:00
|
|
|
package api
|
2023-04-11 21:43:43 +00:00
|
|
|
|
|
|
|
import (
|
|
|
|
"embed"
|
|
|
|
"fmt"
|
|
|
|
"net/http"
|
|
|
|
"strconv"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
|
|
|
|
model "github.com/go-skynet/llama-cli/pkg/model"
|
|
|
|
|
|
|
|
llama "github.com/go-skynet/go-llama.cpp"
|
|
|
|
"github.com/gofiber/fiber/v2"
|
|
|
|
"github.com/gofiber/fiber/v2/middleware/cors"
|
|
|
|
"github.com/gofiber/fiber/v2/middleware/filesystem"
|
|
|
|
"github.com/gofiber/fiber/v2/middleware/recover"
|
|
|
|
)
|
|
|
|
|
|
|
|
type OpenAIResponse struct {
|
|
|
|
Created int `json:"created,omitempty"`
|
|
|
|
Object string `json:"chat.completion,omitempty"`
|
|
|
|
ID string `json:"id,omitempty"`
|
|
|
|
Model string `json:"model,omitempty"`
|
|
|
|
Choices []Choice `json:"choices,omitempty"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type Choice struct {
|
|
|
|
Index int `json:"index,omitempty"`
|
|
|
|
FinishReason string `json:"finish_reason,omitempty"`
|
|
|
|
Message Message `json:"message,omitempty"`
|
|
|
|
Text string `json:"text,omitempty"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type Message struct {
|
|
|
|
Role string `json:"role,omitempty"`
|
|
|
|
Content string `json:"content,omitempty"`
|
|
|
|
}
|
|
|
|
|
|
|
|
type OpenAIModel struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Object string `json:"object"`
|
|
|
|
}
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
type OpenAIRequest struct {
|
|
|
|
Model string `json:"model"`
|
|
|
|
|
|
|
|
// Prompt is read only by completion API calls
|
|
|
|
Prompt string `json:"prompt"`
|
2023-04-13 13:20:51 +00:00
|
|
|
|
|
|
|
// Messages is read only by chat/completion API calls
|
2023-04-11 22:02:39 +00:00
|
|
|
Messages []Message `json:"messages"`
|
|
|
|
|
|
|
|
// Common options between all the API calls
|
|
|
|
TopP float64 `json:"top_p"`
|
|
|
|
TopK int `json:"top_k"`
|
|
|
|
Temperature float64 `json:"temperature"`
|
|
|
|
Maxtokens int `json:"max_tokens"`
|
|
|
|
}
|
|
|
|
|
2023-04-11 21:43:43 +00:00
|
|
|
//go:embed index.html
|
|
|
|
var indexHTML embed.FS
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
func openAIEndpoint(chat bool, defaultModel *llama.LLama, loader *model.ModelLoader, threads int, defaultMutex *sync.Mutex, mutexMap *sync.Mutex, mutexes map[string]*sync.Mutex) func(c *fiber.Ctx) error {
|
2023-04-11 21:43:43 +00:00
|
|
|
return func(c *fiber.Ctx) error {
|
|
|
|
var err error
|
|
|
|
var model *llama.LLama
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
input := new(OpenAIRequest)
|
2023-04-11 21:43:43 +00:00
|
|
|
// Get input data from the request body
|
|
|
|
if err := c.BodyParser(input); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if input.Model == "" {
|
|
|
|
if defaultModel == nil {
|
|
|
|
return fmt.Errorf("no default model loaded, and no model specified")
|
|
|
|
}
|
|
|
|
model = defaultModel
|
|
|
|
} else {
|
|
|
|
model, err = loader.LoadModel(input.Model)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// This is still needed, see: https://github.com/ggerganov/llama.cpp/discussions/784
|
|
|
|
if input.Model != "" {
|
|
|
|
mutexMap.Lock()
|
|
|
|
l, ok := mutexes[input.Model]
|
|
|
|
if !ok {
|
|
|
|
m := &sync.Mutex{}
|
|
|
|
mutexes[input.Model] = m
|
|
|
|
l = m
|
|
|
|
}
|
|
|
|
mutexMap.Unlock()
|
|
|
|
l.Lock()
|
|
|
|
defer l.Unlock()
|
|
|
|
} else {
|
|
|
|
defaultMutex.Lock()
|
|
|
|
defer defaultMutex.Unlock()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the parameters for the language model prediction
|
2023-04-11 22:02:39 +00:00
|
|
|
topP := input.TopP
|
|
|
|
if topP == 0 {
|
|
|
|
topP = 0.7
|
2023-04-11 21:43:43 +00:00
|
|
|
}
|
2023-04-11 22:02:39 +00:00
|
|
|
topK := input.TopK
|
|
|
|
if topK == 0 {
|
|
|
|
topK = 80
|
2023-04-11 21:43:43 +00:00
|
|
|
}
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
temperature := input.Temperature
|
|
|
|
if temperature == 0 {
|
|
|
|
temperature = 0.9
|
2023-04-11 21:43:43 +00:00
|
|
|
}
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
tokens := input.Maxtokens
|
|
|
|
if tokens == 0 {
|
|
|
|
tokens = 512
|
2023-04-11 21:43:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
predInput := input.Prompt
|
2023-04-11 22:02:39 +00:00
|
|
|
if chat {
|
|
|
|
mess := []string{}
|
|
|
|
for _, i := range input.Messages {
|
|
|
|
mess = append(mess, i.Content)
|
2023-04-11 21:43:43 +00:00
|
|
|
}
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
predInput = strings.Join(mess, "\n")
|
2023-04-11 21:43:43 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// A model can have a "file.bin.tmpl" file associated with a prompt template prefix
|
|
|
|
templatedInput, err := loader.TemplatePrefix(input.Model, struct {
|
|
|
|
Input string
|
|
|
|
}{Input: predInput})
|
|
|
|
if err == nil {
|
|
|
|
predInput = templatedInput
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate the prediction using the language model
|
|
|
|
prediction, err := model.Predict(
|
|
|
|
predInput,
|
|
|
|
llama.SetTemperature(temperature),
|
|
|
|
llama.SetTopP(topP),
|
|
|
|
llama.SetTopK(topK),
|
|
|
|
llama.SetTokens(tokens),
|
|
|
|
llama.SetThreads(threads),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-04-11 22:02:39 +00:00
|
|
|
if chat {
|
|
|
|
// Return the chat prediction in the response body
|
|
|
|
return c.JSON(OpenAIResponse{
|
|
|
|
Model: input.Model,
|
|
|
|
Choices: []Choice{{Message: Message{Role: "assistant", Content: prediction}}},
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-04-11 21:43:43 +00:00
|
|
|
// Return the prediction in the response body
|
|
|
|
return c.JSON(OpenAIResponse{
|
|
|
|
Model: input.Model,
|
2023-04-11 22:02:39 +00:00
|
|
|
Choices: []Choice{{Text: prediction}},
|
2023-04-11 21:43:43 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func Start(defaultModel *llama.LLama, loader *model.ModelLoader, listenAddr string, threads int) error {
|
|
|
|
app := fiber.New()
|
|
|
|
|
|
|
|
// Default middleware config
|
|
|
|
app.Use(recover.New())
|
|
|
|
app.Use(cors.New())
|
|
|
|
|
|
|
|
// This is still needed, see: https://github.com/ggerganov/llama.cpp/discussions/784
|
|
|
|
var mutex = &sync.Mutex{}
|
|
|
|
mu := map[string]*sync.Mutex{}
|
|
|
|
var mumutex = &sync.Mutex{}
|
|
|
|
|
|
|
|
// openAI compatible API endpoint
|
2023-04-11 22:02:39 +00:00
|
|
|
app.Post("/v1/chat/completions", openAIEndpoint(true, defaultModel, loader, threads, mutex, mumutex, mu))
|
|
|
|
app.Post("/v1/completions", openAIEndpoint(false, defaultModel, loader, threads, mutex, mumutex, mu))
|
2023-04-11 21:43:43 +00:00
|
|
|
app.Get("/v1/models", func(c *fiber.Ctx) error {
|
|
|
|
models, err := loader.ListModels()
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
dataModels := []OpenAIModel{}
|
|
|
|
for _, m := range models {
|
|
|
|
dataModels = append(dataModels, OpenAIModel{ID: m, Object: "model"})
|
|
|
|
}
|
|
|
|
return c.JSON(struct {
|
|
|
|
Object string `json:"object"`
|
|
|
|
Data []OpenAIModel `json:"data"`
|
|
|
|
}{
|
|
|
|
Object: "list",
|
|
|
|
Data: dataModels,
|
|
|
|
})
|
|
|
|
})
|
|
|
|
|
|
|
|
app.Use("/", filesystem.New(filesystem.Config{
|
|
|
|
Root: http.FS(indexHTML),
|
|
|
|
NotFoundFile: "index.html",
|
|
|
|
}))
|
|
|
|
|
|
|
|
/*
|
|
|
|
curl --location --request POST 'http://localhost:8080/predict' --header 'Content-Type: application/json' --data-raw '{
|
|
|
|
"text": "What is an alpaca?",
|
|
|
|
"topP": 0.8,
|
|
|
|
"topK": 50,
|
|
|
|
"temperature": 0.7,
|
|
|
|
"tokens": 100
|
|
|
|
}'
|
|
|
|
*/
|
|
|
|
// Endpoint to generate the prediction
|
|
|
|
app.Post("/predict", func(c *fiber.Ctx) error {
|
|
|
|
mutex.Lock()
|
|
|
|
defer mutex.Unlock()
|
|
|
|
// Get input data from the request body
|
|
|
|
input := new(struct {
|
|
|
|
Text string `json:"text"`
|
|
|
|
})
|
|
|
|
if err := c.BodyParser(input); err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Set the parameters for the language model prediction
|
|
|
|
topP, err := strconv.ParseFloat(c.Query("topP", "0.9"), 64) // Default value of topP is 0.9
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
topK, err := strconv.Atoi(c.Query("topK", "40")) // Default value of topK is 40
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
temperature, err := strconv.ParseFloat(c.Query("temperature", "0.5"), 64) // Default value of temperature is 0.5
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
tokens, err := strconv.Atoi(c.Query("tokens", "128")) // Default value of tokens is 128
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Generate the prediction using the language model
|
|
|
|
prediction, err := defaultModel.Predict(
|
|
|
|
input.Text,
|
|
|
|
llama.SetTemperature(temperature),
|
|
|
|
llama.SetTopP(topP),
|
|
|
|
llama.SetTopK(topK),
|
|
|
|
llama.SetTokens(tokens),
|
|
|
|
llama.SetThreads(threads),
|
|
|
|
)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// Return the prediction in the response body
|
|
|
|
return c.JSON(struct {
|
|
|
|
Prediction string `json:"prediction"`
|
|
|
|
}{
|
|
|
|
Prediction: prediction,
|
|
|
|
})
|
|
|
|
})
|
|
|
|
|
|
|
|
// Start the server
|
|
|
|
app.Listen(listenAddr)
|
|
|
|
return nil
|
|
|
|
}
|