From d38e9090df32dfd239b36d3ae284b5d8ec87b7a5 Mon Sep 17 00:00:00 2001 From: Dave Date: Fri, 7 Jun 2024 05:22:28 -0400 Subject: [PATCH 1/3] experiment: `-j4` for `build-linux:` (#2514) experiment: set -j4 to see if things go faster, while we wait for a proper fix from mudler Signed-off-by: Dave Lee --- .github/workflows/release.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 157abd82..781a66e0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -102,7 +102,7 @@ jobs: export PATH=/usr/local/cuda/bin:$PATH export PATH=/opt/rocm/bin:$PATH source /opt/intel/oneapi/setvars.sh - GO_TAGS=p2p make dist + GO_TAGS=p2p make -j4 dist - uses: actions/upload-artifact@v4 with: name: LocalAI-linux From 0d62594099f1c86e2764207aeede0856459e232f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Serta=C3=A7=20=C3=96zercan?= <852750+sozercan@users.noreply.github.com> Date: Fri, 7 Jun 2024 18:20:31 +0300 Subject: [PATCH 2/3] fix: fix chat webui response parsing (#2515) fix: fix chat webui Signed-off-by: Sertac Ozercan --- core/http/static/chat.js | 108 ++++++++++++++++++++++++++----------- core/http/views/index.html | 14 ++--- 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/core/http/static/chat.js b/core/http/static/chat.js index 7427ddee..ef15f838 100644 --- a/core/http/static/chat.js +++ b/core/http/static/chat.js @@ -38,7 +38,7 @@ function submitSystemPrompt(event) { localStorage.setItem("system_prompt", document.getElementById("systemPrompt").value); document.getElementById("systemPrompt").blur(); } - + var image = ""; function submitPrompt(event) { @@ -54,15 +54,15 @@ function submitPrompt(event) { } function readInputImage() { - + if (!this.files || !this.files[0]) return; - + const FR = new FileReader(); - + FR.addEventListener("load", function(evt) { image = evt.target.result; - }); - + }); + FR.readAsDataURL(this.files[0]); } @@ -155,7 +155,7 @@ function readInputImage() { stream: true, }), }); - + if (!response.ok) { Alpine.store("chat").add( "assistant", @@ -163,11 +163,11 @@ function readInputImage() { ); return; } - + const reader = response.body ?.pipeThrough(new TextDecoderStream()) .getReader(); - + if (!reader) { Alpine.store("chat").add( "assistant", @@ -175,30 +175,74 @@ function readInputImage() { ); return; } - - while (true) { - const { value, done } = await reader.read(); - if (done) break; - let dataDone = false; - const arr = value.split("\n"); - arr.forEach((data) => { - if (data.length === 0) return; - if (data.startsWith(":")) return; - if (data === "data: [DONE]") { - dataDone = true; - return; + + // Function to add content to the chat and handle DOM updates efficiently + const addToChat = (token) => { + const chatStore = Alpine.store("chat"); + chatStore.add("assistant", token); + // Efficiently scroll into view without triggering multiple reflows + const messages = document.getElementById('messages'); + messages.scrollTop = messages.scrollHeight; + }; + + let buffer = ""; + let contentBuffer = []; + + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + + buffer += value; + + let lines = buffer.split("\n"); + buffer = lines.pop(); // Retain any incomplete line in the buffer + + lines.forEach((line) => { + if (line.length === 0 || line.startsWith(":")) return; + if (line === "data: [DONE]") { + return; + } + + if (line.startsWith("data: ")) { + try { + const jsonData = JSON.parse(line.substring(6)); + const token = jsonData.choices[0].delta.content; + + if (token) { + contentBuffer.push(token); + } + } catch (error) { + console.error("Failed to parse line:", line, error); + } + } + }); + + // Efficiently update the chat in batch + if (contentBuffer.length > 0) { + addToChat(contentBuffer.join("")); + contentBuffer = []; } - const token = JSON.parse(data.substring(6)).choices[0].delta.content; - if (!token) { - return; - } - hljs.highlightAll(); - Alpine.store("chat").add("assistant", token); - document.getElementById('messages').scrollIntoView(false) - }); + } + + // Final content flush if any data remains + if (contentBuffer.length > 0) { + addToChat(contentBuffer.join("")); + } + + // Highlight all code blocks once at the end hljs.highlightAll(); - if (dataDone) break; + } catch (error) { + console.error("An error occurred while reading the stream:", error); + Alpine.store("chat").add( + "assistant", + `Error: Failed to process stream`, + ); + } finally { + // Perform any cleanup if necessary + reader.releaseLock(); } + // Remove class "loader" from the element with "loader" id //document.getElementById("loader").classList.remove("loader"); document.getElementById("loader").style.display = "none"; @@ -209,7 +253,7 @@ function readInputImage() { // set focus to the input document.getElementById("input").focus(); } - + document.getElementById("key").addEventListener("submit", submitKey); document.getElementById("system_prompt").addEventListener("submit", submitSystemPrompt); @@ -230,7 +274,7 @@ function readInputImage() { } else { document.getElementById("systemPrompt").value = null; } - + marked.setOptions({ highlight: function (code) { return hljs.highlightAuto(code).value; diff --git a/core/http/views/index.html b/core/http/views/index.html index 7630cc7b..e2cbfe03 100644 --- a/core/http/views/index.html +++ b/core/http/views/index.html @@ -4,22 +4,22 @@
- + {{template "views/partials/navbar" .}} - +

Welcome to your LocalAI instance!

The FOSS alternative to OpenAI, Claude, ...

Documentation - +
{{template "views/partials/inprogress" .}} - + {{ if eq (len .ModelsConfig) 0 }}

Ouch! seems you don't have any models installed!

..install something from the 🖼️ Gallery or check the Getting started documentation

@@ -70,9 +70,9 @@ {{ end }} - - {{ end }} From 3b7a78addaa03025f3e06cc0b25955eabc6193c0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Fri, 7 Jun 2024 17:20:42 +0200 Subject: [PATCH 3/3] fix(stream): do not break channel consumption (#2517) Signed-off-by: Ettore Di Giacinto --- core/http/endpoints/openai/chat.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index f8a928eb..227bb7d2 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -437,7 +437,6 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup if err != nil { log.Debug().Msgf("Sending chunk failed: %v", err) input.Cancel() - break } w.Flush() }