Merge branch 'master' into rf-service-wiring-1c

.github/workflows/release.yaml

@@ -102,7 +102,7 @@ jobs:
           export PATH=/usr/local/cuda/bin:$PATH
           export PATH=/opt/rocm/bin:$PATH
           source /opt/intel/oneapi/setvars.sh
-          GO_TAGS=p2p make dist
+          GO_TAGS=p2p make -j4 dist
       - uses: actions/upload-artifact@v4
         with:
           name: LocalAI-linux

core/http/endpoints/openai/chat.go

@@ -440,7 +440,6 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup
 					if err != nil {
 						log.Debug().Msgf("Sending chunk failed: %v", err)
 						input.Cancel()
-						break
 					}
 					w.Flush()
 				}

core/http/static/chat.js

@@ -176,29 +176,73 @@ function readInputImage() {
       return;
     }
 
+    // Function to add content to the chat and handle DOM updates efficiently
+    const addToChat = (token) => {
+      const chatStore = Alpine.store("chat");
+      chatStore.add("assistant", token);
+      // Efficiently scroll into view without triggering multiple reflows
+      const messages = document.getElementById('messages');
+      messages.scrollTop = messages.scrollHeight;
+    };
+
+    let buffer = "";
+    let contentBuffer = [];
+
+    try {
       while (true) {
         const { value, done } = await reader.read();
         if (done) break;
-      let dataDone = false;
-      const arr = value.split("\n");
-      arr.forEach((data) => {
-        if (data.length === 0) return;
-        if (data.startsWith(":")) return;
-        if (data === "data: [DONE]") {
-          dataDone = true;
+
+        buffer += value;
+
+        let lines = buffer.split("\n");
+        buffer = lines.pop(); // Retain any incomplete line in the buffer
+
+        lines.forEach((line) => {
+          if (line.length === 0 || line.startsWith(":")) return;
+          if (line === "data: [DONE]") {
             return;
           }
-        const token = JSON.parse(data.substring(6)).choices[0].delta.content;
-        if (!token) {
-          return;
+
+          if (line.startsWith("data: ")) {
+            try {
+              const jsonData = JSON.parse(line.substring(6));
+              const token = jsonData.choices[0].delta.content;
+
+              if (token) {
+                contentBuffer.push(token);
+              }
+            } catch (error) {
+              console.error("Failed to parse line:", line, error);
+            }
           }
-        hljs.highlightAll();
-        Alpine.store("chat").add("assistant", token);
-        document.getElementById('messages').scrollIntoView(false)
         });
-      hljs.highlightAll();
-      if (dataDone) break;
+
+        // Efficiently update the chat in batch
+        if (contentBuffer.length > 0) {
+          addToChat(contentBuffer.join(""));
+          contentBuffer = [];
         }
+      }
+
+      // Final content flush if any data remains
+      if (contentBuffer.length > 0) {
+        addToChat(contentBuffer.join(""));
+      }
+
+      // Highlight all code blocks once at the end
+      hljs.highlightAll();
+    } catch (error) {
+      console.error("An error occurred while reading the stream:", error);
+      Alpine.store("chat").add(
+        "assistant",
+        `<span class='error'>Error: Failed to process stream</span>`,
+      );
+    } finally {
+      // Perform any cleanup if necessary
+      reader.releaseLock();
+    }
+
     // Remove class "loader" from the element with "loader" id
     //document.getElementById("loader").classList.remove("loader");
     document.getElementById("loader").style.display = "none";