Merge 8931ca3cf4 into e28ba4b807

Signed-off-by: Michael Mior <michael.mior@gmail.com>
Signed-off-by: Michael Mior <mmior@mail.rit.edu>

.dockerignore

@@ -6,6 +6,11 @@ examples/chatbot-ui/models
 examples/rwkv/models
 examples/**/models
 Dockerfile*
+__pycache__
 
 # SonarQube
-.scannerwork
+.scannerwork
+
+# backend virtual environments
+**/venv
+backend/python/**/source

.github/workflows/test-extra.yml

@@ -25,22 +25,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
           pip install --user grpcio-tools==1.63.0
           
-          sudo rm -rfv /usr/bin/conda || true
-
       - name: Test transformers
         run: |
-           export PATH=$PATH:/opt/conda/bin
            make --jobs=5 --output-sync=target -C backend/python/transformers
            make --jobs=5 --output-sync=target -C backend/python/transformers test
 
@@ -55,22 +47,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
           pip install --user grpcio-tools==1.63.0
           
-          sudo rm -rfv /usr/bin/conda || true
-
       - name: Test sentencetransformers
         run: |
-           export PATH=$PATH:/opt/conda/bin
            make --jobs=5 --output-sync=target -C backend/python/sentencetransformers
            make --jobs=5 --output-sync=target -C backend/python/sentencetransformers test
 
@@ -86,22 +70,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
           pip install --user grpcio-tools==1.63.0
-          
-          sudo rm -rfv /usr/bin/conda || true
 
       - name: Test rerankers
         run: |
-           export PATH=$PATH:/opt/conda/bin
            make --jobs=5 --output-sync=target -C backend/python/rerankers
            make --jobs=5 --output-sync=target -C backend/python/rerankers test
 
@@ -115,25 +91,16 @@ jobs:
       - name: Dependencies
         run: |
           sudo apt-get update
-          sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          sudo apt-get install -y build-essential ffmpeg
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           pip install --user grpcio-tools==1.63.0
-          
-          sudo rm -rfv /usr/bin/conda || true
-
       - name: Test diffusers
         run: |
-           export PATH=$PATH:/opt/conda/bin
-           make --jobs=5 --output-sync=target -C backend/python/diffusers
-           make --jobs=5 --output-sync=target -C backend/python/diffusers test
+          make --jobs=5 --output-sync=target -C backend/python/diffusers
+          make --jobs=5 --output-sync=target -C backend/python/diffusers test
 
   tests-parler-tts:
     runs-on: ubuntu-latest
@@ -146,22 +113,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
           pip install --user grpcio-tools==1.63.0
-          
-          sudo rm -rfv /usr/bin/conda || true
 
       - name: Test parler-tts
         run: |
-           export PATH=$PATH:/opt/conda/bin
            make --jobs=5 --output-sync=target -C backend/python/parler-tts
            make --jobs=5 --output-sync=target -C backend/python/parler-tts test
 
@@ -176,22 +135,14 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
           pip install --user grpcio-tools==1.63.0
-          
-          sudo rm -rfv /usr/bin/conda || true
 
       - name: Test transformers-musicgen
         run: |
-           export PATH=$PATH:/opt/conda/bin
            make --jobs=5 --output-sync=target -C backend/python/transformers-musicgen
            make --jobs=5 --output-sync=target -C backend/python/transformers-musicgen test
 
@@ -208,22 +159,14 @@ jobs:
   #       run: |
   #         sudo apt-get update
   #         sudo apt-get install build-essential ffmpeg
-  #         curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-  #            sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-  #             gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-  #            sudo apt-get update && \
-  #            sudo apt-get install -y conda
+  #         # Install UV
+  #         curl -LsSf https://astral.sh/uv/install.sh | sh
   #         sudo apt-get install -y ca-certificates cmake curl patch python3-pip
   #         sudo apt-get install -y libopencv-dev
   #         pip install --user grpcio-tools==1.63.0
-          
-  #         sudo rm -rfv /usr/bin/conda || true
 
   #     - name: Test petals
   #       run: |
-  #          export PATH=$PATH:/opt/conda/bin
   #          make --jobs=5 --output-sync=target -C backend/python/petals
   #          make --jobs=5 --output-sync=target -C backend/python/petals test
 
@@ -280,22 +223,14 @@ jobs:
   #       run: |
   #         sudo apt-get update
   #         sudo apt-get install build-essential ffmpeg
-  #         curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-  #            sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-  #             gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-  #            sudo apt-get update && \
-  #            sudo apt-get install -y conda
+  #         # Install UV
+  #         curl -LsSf https://astral.sh/uv/install.sh | sh
   #         sudo apt-get install -y ca-certificates cmake curl patch python3-pip
   #         sudo apt-get install -y libopencv-dev
   #         pip install --user grpcio-tools==1.63.0
-          
-  #         sudo rm -rfv /usr/bin/conda || true
 
   #     - name: Test bark
   #       run: |
-  #          export PATH=$PATH:/opt/conda/bin
   #          make --jobs=5 --output-sync=target -C backend/python/bark
   #          make --jobs=5 --output-sync=target -C backend/python/bark test
 
@@ -313,20 +248,13 @@ jobs:
   #       run: |
   #         sudo apt-get update
   #         sudo apt-get install build-essential ffmpeg
-  #         curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-  #            sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-  #             gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-  #            sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-  #            sudo apt-get update && \
-  #            sudo apt-get install -y conda
+  #         # Install UV
+  #         curl -LsSf https://astral.sh/uv/install.sh | sh
   #         sudo apt-get install -y ca-certificates cmake curl patch python3-pip
   #         sudo apt-get install -y libopencv-dev
   #         pip install --user grpcio-tools==1.63.0
-  #         sudo rm -rfv /usr/bin/conda || true
   #     - name: Test vllm
   #       run: |
-  #          export PATH=$PATH:/opt/conda/bin
   #          make --jobs=5 --output-sync=target -C backend/python/vllm
   #          make --jobs=5 --output-sync=target -C backend/python/vllm test
   tests-vallex:
@@ -340,20 +268,13 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake curl patch python3-pip
           sudo apt-get install -y libopencv-dev
           pip install --user grpcio-tools==1.63.0
-          sudo rm -rfv /usr/bin/conda || true
       - name: Test vall-e-x
         run: |
-           export PATH=$PATH:/opt/conda/bin
            make --jobs=5 --output-sync=target -C backend/python/vall-e-x
            make --jobs=5 --output-sync=target -C backend/python/vall-e-x test
 
@@ -368,19 +289,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install build-essential ffmpeg
-          curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-             sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-              gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
-             sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
-             sudo apt-get update && \
-             sudo apt-get install -y conda
           sudo apt-get install -y ca-certificates cmake curl patch espeak espeak-ng python3-pip
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           pip install --user grpcio-tools==1.63.0
-          sudo rm -rfv /usr/bin/conda || true
-
       - name: Test coqui
         run: |
-           export PATH=$PATH:/opt/conda/bin
-           make --jobs=5 --output-sync=target -C backend/python/coqui
-           make --jobs=5 --output-sync=target -C backend/python/coqui test
+          make --jobs=5 --output-sync=target -C backend/python/coqui
+          make --jobs=5 --output-sync=target -C backend/python/coqui test

.github/workflows/test.yml

@@ -78,6 +78,8 @@ jobs:
              sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
              sudo apt-get update && \
              sudo apt-get install -y conda
+          # Install UV
+          curl -LsSf https://astral.sh/uv/install.sh | sh
           sudo apt-get install -y ca-certificates cmake patch python3-pip unzip
           sudo apt-get install -y libopencv-dev
 

.gitignore

@@ -46,4 +46,7 @@ prepare
 *pb2_grpc.py
 
 # SonarQube
-.scannerwork
+.scannerwork
+
+# backend virtual environments
+**/venv

Dockerfile

@@ -76,26 +76,15 @@ RUN test -n "$TARGETARCH" \
 # The requirements-extras target is for any builds with IMAGE_TYPE=extras. It should not be placed in this target unless every IMAGE_TYPE=extras build will use it
 FROM requirements-core AS requirements-extras
 
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends gpg && \
-    curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
-    install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
-    gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
-    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list && \
-    echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-        conda && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 ENV PATH="/root/.cargo/bin:${PATH}"
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         espeak-ng \
-        espeak && \
+        espeak \
+        python3-venv && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
@@ -246,6 +235,7 @@ ARG FFMPEG
 ARG BUILD_TYPE
 ARG TARGETARCH
 ARG IMAGE_TYPE=extras
+ARG EXTRA_BACKENDS
 ARG MAKEFLAGS
 
 ENV BUILD_TYPE=${BUILD_TYPE}
@@ -257,7 +247,6 @@ ARG CUDA_MAJOR_VERSION=11
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 ENV NVIDIA_REQUIRE_CUDA="cuda>=${CUDA_MAJOR_VERSION}.0"
 ENV NVIDIA_VISIBLE_DEVICES=all
-ENV PIP_CACHE_PURGE=true
 
 # Add FFmpeg
 RUN if [ "${FFMPEG}" = "true" ]; then \
@@ -290,51 +279,58 @@ COPY --from=builder /build/sources/go-piper/piper-phonemize/pi/lib/* /usr/lib/
 # do not let stablediffusion rebuild (requires an older version of absl)
 COPY --from=builder /build/backend-assets/grpc/stablediffusion ./backend-assets/grpc/stablediffusion
 
-## Duplicated from Makefile to avoid having a big layer that's hard to push
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/autogptq \
+# Change the shell to bash so we can use [[ tests below
+SHELL ["/bin/bash", "-c"]
+# We try to strike a balance between individual layer size (as that affects total push time) and total image size
+# Splitting the backends into more groups with fewer items results in a larger image, but a smaller size for the largest layer
+# Splitting the backends into fewer groups with more items results in a smaller image, but a larger size for the largest layer
+
+RUN if [[ ( "${EXTRA_BACKENDS}" =~ "coqui" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/coqui \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "parler-tts" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/parler-tts \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "diffusers" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/diffusers \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "transformers-musicgen" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/transformers-musicgen \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "exllama1" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/exllama \
     ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/bark \
+
+RUN if [[ ( "${EXTRA_BACKENDS}" =~ "vall-e-x" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/vall-e-x \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "petals" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/petals \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "sentencetransformers" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/sentencetransformers \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "exllama2" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/exllama2 \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "transformers" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/transformers \
     ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/diffusers \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/vllm \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/mamba \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/sentencetransformers \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/rerankers \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/transformers \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/vall-e-x \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/exllama \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/exllama2 \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/petals \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/transformers-musicgen \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/parler-tts \
-    ; fi
-RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-    make -C backend/python/coqui \
+
+RUN if [[ ( "${EXTRA_BACKENDS}" =~ "vllm" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/vllm \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "autogptq" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/autogptq \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "bark" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/bark \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "rerankers" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/rerankers \
+    ; fi && \
+    if [[ ( "${EXTRA_BACKENDS}" =~ "mamba" || -z "${EXTRA_BACKENDS}" ) && "$IMAGE_TYPE" == "extras" ]]; then \
+        make -C backend/python/mamba \
     ; fi
 
 # Make sure the models directory exists

Makefile

@@ -5,7 +5,7 @@ BINARY_NAME=local-ai
 
 # llama.cpp versions
 GOLLAMA_STABLE_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be
-CPPLLAMA_VERSION?=628b299106d1e9476fdecb3cbe546bf5c60f1b89
+CPPLLAMA_VERSION?=858f6b73f6e57a62523d16a955d565254be889b4
 
 # gpt4all version
 GPT4ALL_REPO?=https://github.com/nomic-ai/gpt4all
@@ -574,20 +574,20 @@ vllm-protogen-clean:
 # Note: it is duplicated in the Dockerfile
 prepare-extra-conda-environments: protogen-python
 	$(MAKE) -C backend/python/autogptq
-	$(MAKE) -C backend/python/bark
-	$(MAKE) -C backend/python/coqui
-	$(MAKE) -C backend/python/diffusers
-	$(MAKE) -C backend/python/vllm
-	$(MAKE) -C backend/python/mamba
-	$(MAKE) -C backend/python/sentencetransformers
-	$(MAKE) -C backend/python/rerankers
-	$(MAKE) -C backend/python/transformers
-	$(MAKE) -C backend/python/transformers-musicgen
-	$(MAKE) -C backend/python/parler-tts
-	$(MAKE) -C backend/python/vall-e-x
-	$(MAKE) -C backend/python/exllama
-	$(MAKE) -C backend/python/petals
-	$(MAKE) -C backend/python/exllama2
+#	$(MAKE) -C backend/python/bark
+#	$(MAKE) -C backend/python/coqui
+#	$(MAKE) -C backend/python/diffusers
+#	$(MAKE) -C backend/python/vllm
+#	$(MAKE) -C backend/python/mamba
+#	$(MAKE) -C backend/python/sentencetransformers
+#	$(MAKE) -C backend/python/rerankers
+#	$(MAKE) -C backend/python/transformers
+#	$(MAKE) -C backend/python/transformers-musicgen
+#	$(MAKE) -C backend/python/parler-tts
+#	$(MAKE) -C backend/python/vall-e-x
+#	$(MAKE) -C backend/python/exllama
+#	$(MAKE) -C backend/python/petals
+#	$(MAKE) -C backend/python/exllama2
 
 prepare-test-extra: protogen-python
 	$(MAKE) -C backend/python/transformers

README.md

@@ -114,6 +114,7 @@ Model galleries
 Other:
 - Helm chart https://github.com/go-skynet/helm-charts
 - VSCode extension https://github.com/badgooooor/localai-vscode-plugin
+- Terminal utility https://github.com/djcopley/ShellOracle
 - Local Smart assistant https://github.com/mudler/LocalAGI
 - Home Assistant https://github.com/sammcj/homeassistant-localai / https://github.com/drndos/hass-openai-custom-conversation
 - Discord bot https://github.com/mudler/LocalAGI/tree/main/examples/discord

backend/cpp/llama/Makefile

@@ -62,8 +62,8 @@ grpc-server: llama.cpp llama.cpp/examples/grpc-server
 	@echo "Building grpc-server with $(BUILD_TYPE) build type and $(CMAKE_ARGS)"
 ifneq (,$(findstring sycl,$(BUILD_TYPE)))
 	bash -c "source $(ONEAPI_VARS); \
-	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release"	
+	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && $(MAKE)"
 else
-	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && cmake --build . --config Release
+	cd llama.cpp && mkdir -p build && cd build && cmake .. $(CMAKE_ARGS) && $(MAKE)
 endif
 	cp llama.cpp/build/bin/grpc-server .

backend/python/autogptq/Makefile

@@ -1,6 +1,6 @@
 .PHONY: autogptq
 autogptq: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash install.sh
 
 .PHONY: protogen
 protogen: backend_pb2_grpc.py backend_pb2.py
@@ -10,4 +10,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/autogptq/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/autogptq/requirements.txt

@@ -0,0 +1,7 @@
+accelerate
+auto-gptq==0.7.1
+grpcio==1.63.0
+protobuf
+torch
+certifi
+transformers==4.38.2

backend/python/autogptq/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the autogptq server with conda
+## A bash script wrapper that runs the autogptq server
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/autogptq.py $@
+python $MY_DIR/autogptq.py $@

backend/python/bark/Makefile

@@ -1,6 +1,6 @@
 .PHONY: ttsbark
 ttsbark: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -22,4 +22,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/bark/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/bark/requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+bark==0.1.5
+grpcio==1.63.0
+protobuf
+certifi
+transformers==4.38.2

backend/python/bark/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the ttsbark server with conda
+## A bash script wrapper that runs the ttsbark server
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/ttsbark.py $@
+python $MY_DIR/ttsbark.py $@

backend/python/bark/test.sh

@@ -1,11 +1,9 @@
 #!/bin/bash
 ##
-## A bash script wrapper that runs the bark server with conda
+## A bash script wrapper that runs the bark tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test.py
+python -m unittest $MY_DIR/test.py

backend/python/coqui/Makefile

@@ -1,6 +1,6 @@
 .PHONY: coqui
 coqui: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -22,4 +22,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/coqui/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/coqui/requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+TTS==0.22.0
+grpcio==1.63.0
+protobuf
+certifi
+transformers==4.38.2

backend/python/coqui/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the ttsbark server with conda
+## A bash script wrapper that runs the coqui server
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/coqui_server.py $@
+python $MY_DIR/coqui_server.py $@

backend/python/coqui/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the bark server with conda
+## A bash script wrapper that runs the coqui tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test.py
+python -m unittest $MY_DIR/test.py

backend/python/diffusers/Makefile

@@ -13,8 +13,7 @@ endif
 
 .PHONY: diffusers
 diffusers: protogen
-	@echo "Installing $(CONDA_ENV_PATH)..."
-	bash install.sh $(CONDA_ENV_PATH)
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -33,4 +32,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/diffusers/diffusers-rocm.yml

@@ -1,65 +0,0 @@
-name: diffusers
-channels:
-  - defaults
-dependencies:
-  - _libgcc_mutex=0.1=main
-  - _openmp_mutex=5.1=1_gnu
-  - bzip2=1.0.8=h7b6447c_0
-  - ca-certificates=2023.08.22=h06a4308_0
-  - ld_impl_linux-64=2.38=h1181459_1
-  - libffi=3.4.4=h6a678d5_0
-  - libgcc-ng=11.2.0=h1234567_1
-  - libgomp=11.2.0=h1234567_1
-  - libstdcxx-ng=11.2.0=h1234567_1
-  - libuuid=1.41.5=h5eee18b_0
-  - ncurses=6.4=h6a678d5_0
-  - openssl=3.0.11=h7f8727e_2
-  - pip=23.2.1=py311h06a4308_0
-  - python=3.11.5=h955ad1f_0
-  - readline=8.2=h5eee18b_0
-  - setuptools=68.0.0=py311h06a4308_0
-  - sqlite=3.41.2=h5eee18b_0
-  - tk=8.6.12=h1ccaba5_0
-  - tzdata=2023c=h04d1e81_0
-  - wheel=0.41.2=py311h06a4308_0
-  - xz=5.4.2=h5eee18b_0
-  - zlib=1.2.13=h5eee18b_0
-  - pip:
-      - --pre
-      - --extra-index-url https://download.pytorch.org/whl/nightly/
-      - accelerate>=0.11.0
-      - certifi==2023.7.22
-      - charset-normalizer==3.3.0
-      - compel==2.0.2
-      - diffusers==0.24.0
-      - filelock==3.12.4
-      - fsspec==2023.9.2
-      - grpcio==1.63.0
-      - huggingface-hub>=0.19.4
-      - idna==3.4
-      - importlib-metadata==6.8.0
-      - jinja2==3.1.2
-      - markupsafe==2.1.3
-      - mpmath==1.3.0
-      - networkx==3.1
-      - numpy==1.26.0
-      - omegaconf
-      - packaging==23.2
-      - pillow==10.0.1
-      - protobuf==4.24.4
-      - psutil==5.9.5
-      - pyparsing==3.1.1
-      - pyyaml==6.0.1
-      - regex==2023.10.3
-      - requests==2.31.0
-      - safetensors==0.4.0
-      - sympy==1.12
-      - tqdm==4.66.1
-      - transformers>=4.25.1
-      - triton==2.1.0
-      - typing-extensions==4.8.0
-      - urllib3==2.0.6
-      - zipp==3.17.0
-      - torch
-      - opencv-python
-prefix: /opt/conda/envs/diffusers

backend/python/diffusers/diffusers.yml

@@ -1,75 +0,0 @@
-name: diffusers
-channels:
-  - defaults
-dependencies:
-  - _libgcc_mutex=0.1=main
-  - _openmp_mutex=5.1=1_gnu
-  - bzip2=1.0.8=h7b6447c_0
-  - ca-certificates=2023.08.22=h06a4308_0
-  - ld_impl_linux-64=2.38=h1181459_1
-  - libffi=3.4.4=h6a678d5_0
-  - libgcc-ng=11.2.0=h1234567_1
-  - libgomp=11.2.0=h1234567_1
-  - libstdcxx-ng=11.2.0=h1234567_1
-  - libuuid=1.41.5=h5eee18b_0
-  - ncurses=6.4=h6a678d5_0
-  - openssl=3.0.11=h7f8727e_2
-  - pip=23.2.1=py311h06a4308_0
-  - python=3.11.5=h955ad1f_0
-  - readline=8.2=h5eee18b_0
-  - setuptools=68.0.0=py311h06a4308_0
-  - sqlite=3.41.2=h5eee18b_0
-  - tk=8.6.12=h1ccaba5_0
-  - tzdata=2023c=h04d1e81_0
-  - wheel=0.41.2=py311h06a4308_0
-  - xz=5.4.2=h5eee18b_0
-  - zlib=1.2.13=h5eee18b_0
-  - pip:
-      - accelerate>=0.11.0
-      - certifi==2023.7.22
-      - charset-normalizer==3.3.0
-      - compel==2.0.2
-      - diffusers==0.24.0
-      - filelock==3.12.4
-      - fsspec==2023.9.2
-      - grpcio==1.63.0
-      - huggingface-hub>=0.19.4
-      - idna==3.4
-      - importlib-metadata==6.8.0
-      - jinja2==3.1.2
-      - markupsafe==2.1.3
-      - mpmath==1.3.0
-      - networkx==3.1
-      - numpy==1.26.0
-      - nvidia-cublas-cu12==12.1.3.1
-      - nvidia-cuda-cupti-cu12==12.1.105
-      - nvidia-cuda-nvrtc-cu12==12.1.105
-      - nvidia-cuda-runtime-cu12==12.1.105
-      - nvidia-cudnn-cu12==8.9.2.26
-      - nvidia-cufft-cu12==11.0.2.54
-      - nvidia-curand-cu12==10.3.2.106
-      - nvidia-cusolver-cu12==11.4.5.107
-      - nvidia-cusparse-cu12==12.1.0.106
-      - nvidia-nccl-cu12==2.18.1
-      - nvidia-nvjitlink-cu12==12.2.140
-      - nvidia-nvtx-cu12==12.1.105
-      - omegaconf
-      - packaging==23.2
-      - pillow==10.0.1
-      - protobuf==4.24.4
-      - psutil==5.9.5
-      - pyparsing==3.1.1
-      - pyyaml==6.0.1
-      - regex==2023.10.3
-      - requests==2.31.0
-      - safetensors==0.4.0
-      - sympy==1.12
-      - torch==2.1.0
-      - tqdm==4.66.1
-      - transformers>=4.25.1
-      - triton==2.1.0
-      - typing-extensions==4.8.0
-      - urllib3==2.0.6
-      - zipp==3.17.0
-      - opencv-python
-prefix: /opt/conda/envs/diffusers

backend/python/diffusers/install.sh

@@ -1,50 +1,25 @@
 #!/bin/bash
 set -ex
 
-SKIP_CONDA=${SKIP_CONDA:-0}
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Check if environment exist
-conda_env_exists(){
-    ! conda list --name "${@}" >/dev/null 2>/dev/null
-}
+python -m venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
 
-if [ $SKIP_CONDA -eq 1 ]; then
-    echo "Skipping conda environment installation"
-else
-    export PATH=$PATH:/opt/conda/bin
-    if conda_env_exists "diffusers" ; then
-        echo "Creating virtual environment..."
-        conda env create --name diffusers --file $1
-        echo "Virtual environment created."
-    else 
-        echo "Virtual environment already exists."
-    fi
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
 fi
 
 if [ -d "/opt/intel" ]; then
     # Intel GPU: If the directory exists, we assume we are using the Intel image
     # https://github.com/intel/intel-extension-for-pytorch/issues/538
-    pip install torch==2.1.0a0 \
-                torchvision==0.16.0a0 \
-                torchaudio==2.1.0a0 \
-                intel-extension-for-pytorch==2.1.10+xpu \
-                --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/
-    
-    pip install google-api-python-client \
-                grpcio==1.63.0 \
-                grpcio-tools==1.63.0 \
-                diffusers==0.24.0 \
-                transformers>=4.25.1 \
-                accelerate \
-                compel==2.0.2 \
-                Pillow
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
 fi
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then
-    if [ $SKIP_CONDA -ne 1 ]; then
-        # Activate conda environment
-        source activate diffusers
-    fi
-
     pip cache purge
 fi

backend/python/diffusers/requirements-intel.txt

@@ -0,0 +1,3 @@
+intel-extension-for-pytorch
+torchaudio
+torchvision

backend/python/diffusers/requirements.txt

@@ -0,0 +1,10 @@
+accelerate
+compel
+diffusers
+grpcio==1.63.0
+opencv-python
+pillow
+protobuf
+torch
+transformers==4.38.2
+certifi

backend/python/diffusers/run.sh

@@ -1,19 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the diffusers server with conda
+## A bash script wrapper that runs the GRPC backend
 
-if [ -d "/opt/intel" ]; then
-    # Assumes we are using the Intel oneAPI container image
-    # https://github.com/intel/intel-extension-for-pytorch/issues/538
-    export XPU=1
-else
-    export PATH=$PATH:/opt/conda/bin
-    # Activate conda environment
-    source activate diffusers
-fi
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python $DIR/backend_diffusers.py $@
+python $MY_DIR/backend_diffusers.py $@

backend/python/diffusers/test.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the diffusers server with conda
+## A bash script wrapper that runs the tests
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate diffusers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python -m unittest $DIR/test.py
+python -m unittest $MY_DIR/test.py

backend/python/exllama/.gitignore

@@ -0,0 +1 @@
+source

backend/python/exllama/Makefile

@@ -18,4 +18,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	$(RM) -r venv source

backend/python/exllama/install.sh

@@ -1,31 +1,26 @@
 #!/bin/bash
 set -ex
 
-export PATH=$PATH:/opt/conda/bin
-
 if [ "$BUILD_TYPE" != "cublas" ]; then
     echo "[exllama] Attention!!! Nvidia GPU is required - skipping installation"
     exit 0
 fi
 
-# Check if environment exist
-conda_env_exists(){
-    ! conda list --name "${@}" >/dev/null 2>/dev/null
-}
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-if conda_env_exists "exllama" ; then
-    echo "Creating virtual environment..."
-    conda env create --name exllama --file $1
-    echo "Virtual environment created."
-else
-    echo "Virtual environment already exists."
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
 fi
 
-source activate exllama
+git clone https://github.com/turboderp/exllama $MY_DIR/source
+uv pip install --requirement ${MY_DIR}/source/requirements.txt
 
-git clone https://github.com/turboderp/exllama $CONDA_PREFIX/exllama && pushd $CONDA_PREFIX/exllama && pip install -r requirements.txt && popd
-
-cp -rfv $CONDA_PREFIX/exllama/* ./
+cp -rfv ./*py $MY_DIR/source/
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then
     pip cache purge

backend/python/exllama/requirements.txt

@@ -0,0 +1,5 @@
+grpcio==1.63.0
+protobuf
+torch
+transformers==4.38.2
+certifi

backend/python/exllama/run.sh

@@ -1,15 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the exllama server with conda
-export PATH=$PATH:/opt/conda/bin
+## A bash script wrapper that runs the exllama server with uv
 
-# Activate conda environment
-source activate exllama
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-cd $DIR
-
-python $DIR/exllama.py $@
+python $MY_DIR/source/exllama.py $@

backend/python/exllama2/.gitignore

@@ -0,0 +1 @@
+source

backend/python/exllama2/Makefile

@@ -1,6 +1,5 @@
 .PHONY: exllama2
 exllama2: protogen
-	$(MAKE) -C ../common-env/transformers
 	bash install.sh
 
 .PHONY: run
@@ -17,4 +16,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	$(RM) -r venv source

backend/python/exllama2/exllama2.yml

@@ -1,57 +0,0 @@
-name: exllama2
-channels:
-  - defaults
-dependencies:
-  - _libgcc_mutex=0.1=main
-  - _openmp_mutex=5.1=1_gnu
-  - bzip2=1.0.8=h7b6447c_0
-  - ca-certificates=2023.08.22=h06a4308_0
-  - ld_impl_linux-64=2.38=h1181459_1
-  - libffi=3.4.4=h6a678d5_0
-  - libgcc-ng=11.2.0=h1234567_1
-  - libgomp=11.2.0=h1234567_1
-  - libstdcxx-ng=11.2.0=h1234567_1
-  - libuuid=1.41.5=h5eee18b_0
-  - ncurses=6.4=h6a678d5_0
-  - openssl=3.0.11=h7f8727e_2
-  - pip=23.2.1=py311h06a4308_0
-  - python=3.11.5=h955ad1f_0
-  - readline=8.2=h5eee18b_0
-  - setuptools=68.0.0=py311h06a4308_0
-  - sqlite=3.41.2=h5eee18b_0
-  - tk=8.6.12=h1ccaba5_0
-  - tzdata=2023c=h04d1e81_0
-  - wheel=0.41.2=py311h06a4308_0
-  - xz=5.4.2=h5eee18b_0
-  - zlib=1.2.13=h5eee18b_0
-  - pip:
-      - filelock==3.12.4
-      - fsspec==2023.9.2
-      - grpcio==1.63.0
-      - markupsafe==2.1.3
-      - mpmath==1.3.0
-      - networkx==3.1
-      - protobuf==4.24.4
-      - nvidia-cublas-cu12==12.1.3.1
-      - nvidia-cuda-cupti-cu12==12.1.105
-      - nvidia-cuda-nvrtc-cu12==12.1.105
-      - nvidia-cuda-runtime-cu12==12.1.105
-      - nvidia-cudnn-cu12==8.9.2.26
-      - nvidia-cufft-cu12==11.0.2.54
-      - nvidia-curand-cu12==10.3.2.106
-      - nvidia-cusolver-cu12==11.4.5.107
-      - nvidia-cusparse-cu12==12.1.0.106
-      - nvidia-nccl-cu12==2.18.1
-      - nvidia-nvjitlink-cu12==12.2.140
-      - nvidia-nvtx-cu12==12.1.105
-      - pandas
-      - numpy
-      - ninja
-      - fastparquet
-      - torch>=2.1.0
-      - safetensors>=0.3.2
-      - sentencepiece>=0.1.97
-      - pygments
-      - websockets
-      - regex
-prefix: /opt/conda/envs/exllama2

backend/python/exllama2/install.sh

@@ -2,30 +2,30 @@
 set -e
 ##
 ## A bash script installs the required dependencies of VALL-E-X and prepares the environment
-export SHA=c0ddebaaaf8ffd1b3529c2bb654e650bce2f790f
+EXLLAMA2_VERSION=c0ddebaaaf8ffd1b3529c2bb654e650bce2f790f
 
 if [ "$BUILD_TYPE" != "cublas" ]; then
     echo "[exllamav2] Attention!!! Nvidia GPU is required - skipping installation"
     exit 0
 fi
 
-export PATH=$PATH:/opt/conda/bin
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-echo $CONDA_PREFIX
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
 
-git clone https://github.com/turboderp/exllamav2 $CONDA_PREFIX/exllamav2
+uv pip install --requirement ${MY_DIR}/requirements.txt
 
-pushd $CONDA_PREFIX/exllamav2
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
 
-git checkout -b build $SHA
+git clone https://github.com/turboderp/exllamav2 source
+pushd source && git checkout -b build ${EXLLAMA2_VERSION} && popd
 
-# TODO: this needs to be pinned within the conda environments
-pip install -r requirements.txt
+uv pip install --requirement ${MY_DIR}/source/requirements.txt
 
-popd
-
-cp -rfv $CONDA_PREFIX/exllamav2/* ./  
+cp -rfv ./*py $MY_DIR/source/  
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then
     pip cache purge

backend/python/exllama2/requirements.txt

@@ -0,0 +1,4 @@
+accelerate
+grpcio==1.63.0
+protobuf
+certifi

backend/python/exllama2/run.sh

@@ -1,16 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the exllama server with conda
+## A bash script wrapper that runs the exllama2 server
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-cd $DIR
-
-python $DIR/exllama2_backend.py $@
+python $MY_DIR/source/exllama2_backend.py $@

backend/python/mamba/Makefile

@@ -1,6 +1,5 @@
 .PHONY: mamba
 mamba: protogen
-	$(MAKE) -C ../common-env/transformers
 	bash install.sh
 
 .PHONY: run
@@ -23,4 +22,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	$(RM) -r venv

backend/python/mamba/install.sh

@@ -8,14 +8,29 @@ if [ "$BUILD_TYPE" != "cublas" ]; then
     exit 0
 fi
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
 
-echo $CONDA_PREFIX
+# mabma does not specify it's build dependencies per PEP517, so we need to disable build isolation
+# this also means that we need to install the basic build dependencies into the venv ourselves
+# https://github.com/Dao-AILab/causal-conv1d/issues/24
+uv pip install --requirement ${MY_DIR}/requirements-install.txt
 
-pip install causal-conv1d==1.0.0 mamba-ssm==1.0.1
+uv pip install --no-build-isolation --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then
     pip cache purge

backend/python/mamba/requirements-install.txt

@@ -0,0 +1,7 @@
+# mabma does not specify it's build dependencies per PEP517, so we need to disable build isolation
+# this also means that we need to install the basic build dependencies into the venv ourselves
+# https://github.com/Dao-AILab/causal-conv1d/issues/24
+packaging
+setuptools
+wheel
+torch

backend/python/mamba/requirements.txt

@@ -0,0 +1,7 @@
+accelerate
+causal-conv1d==1.2.0.post2
+mamba-ssm==1.2.0.post1
+grpcio==1.63.0
+torch==2.1.2
+certifi
+transformers==4.38.2

backend/python/mamba/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the diffusers server with conda
+## A bash script wrapper that runs the GRPC server
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/backend_mamba.py $@
+python $MY_DIR/backend_mamba.py $@

backend/python/mamba/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that sets up and runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_backend_mamba.py
+python -m unittest $MY_DIR/test_backend_mamba.py

backend/python/parler-tts/Makefile

@@ -36,4 +36,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	$(RM) -r venv

backend/python/parler-tts/install.sh

@@ -1,39 +1,29 @@
 #!/bin/bash
 set -ex
 
-SKIP_CONDA=${SKIP_CONDA:-0}
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Check if environment exist
-conda_env_exists(){
-    ! conda list --name "${@}" >/dev/null 2>/dev/null
-}
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
 
-if [ $SKIP_CONDA -eq 1 ]; then
-    echo "Skipping conda environment installation"
-else
-    export PATH=$PATH:/opt/conda/bin
-    if conda_env_exists "parler" ; then
-        echo "Creating virtual environment..."
-        conda env create --name parler --file $1
-        echo "Virtual environment created."
-    else 
-        echo "Virtual environment already exists."
-    fi
+uv pip install --requirement ${MY_DIR}/requirements.txt
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
 fi
 
-if [ $SKIP_CONDA -ne 1 ]; then
-    # Activate conda environment
-    source activate parler
-    # https://github.com/descriptinc/audiotools/issues/101
-    # incompatible protobuf versions.
-    curl -L https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -o $CONDA_PREFIX/lib/python3.11/site-packages/google/protobuf/internal/builder.py
+# https://github.com/descriptinc/audiotools/issues/101
+# incompatible protobuf versions.
+PYDIR=$(ls $MY_DIR/venv/lib)
+curl -L https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -o $MY_DIR/venv/lib/$PYDIR/site-packages/google/protobuf/internal/builder.py
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
 fi
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then
-    if [ $SKIP_CONDA -ne 1 ]; then
-        # Activate conda environment
-        source activate parler
-    fi
-
     pip cache purge
 fi

backend/python/parler-tts/requirements.txt

@@ -0,0 +1,7 @@
+accelerate
+grpcio==1.63.0
+protobuf
+torch
+git+https://github.com/huggingface/parler-tts.git@10016fb0300c0dc31a0fb70e26f3affee7b62f16
+certifi
+transformers==4.38.2

backend/python/parler-tts/run.sh

@@ -1,16 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the parler-tts server with conda
+## A bash script wrapper that runs the GRPC backend
 
-echo "Launching gRPC server for parler-tts"
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-export PATH=$PATH:/opt/conda/bin
+source $MY_DIR/venv/bin/activate
 
-# Activate conda environment
-source activate parler
-
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/parler_tts_server.py $@
+python $MY_DIR/parler_tts_server.py $@

backend/python/parler-tts/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that sets up and runs the tests
 
-# Activate conda environment
-source activate parler
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_parler.py
+python -m unittest $MY_DIR/test_parler.py

backend/python/petals/Makefile

@@ -24,4 +24,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/petals/install.sh

@@ -1,5 +1,25 @@
 #!/bin/bash
+set -ex
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-conda env create --name petals --file $1
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/petals/requirements.txt

@@ -0,0 +1,3 @@
+git+https://github.com/bigscience-workshop/petals
+certifi
+transformers==4.38.2

backend/python/petals/run.sh

@@ -1,23 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the exllama server with conda
+## A bash script wrapper that runs the GRPC backend
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-CONDA_ENV=petals
+source $MY_DIR/venv/bin/activate
 
-# Activate conda environment
-# if source is available use it, or use conda
-#
-if [ -f /opt/conda/bin/activate ]; then
-    source activate $CONDA_ENV
-else
-    eval "$(conda shell.bash hook)"
-    conda activate $CONDA_ENV
-fi
-
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/backend_petals.py $@
+python $MY_DIR/backend_petals.py $@

backend/python/petals/test.sh

@@ -1,20 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-CONDA_ENV=petals
-# Activate conda environment
-# if source is available use it, or use conda
-#
-if [ -f /opt/conda/bin/activate ]; then
-    source activate $CONDA_ENV
-else
-    eval "$(conda shell.bash hook)"
-    conda activate $CONDA_ENV
-fi
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_petals.py
+python -m unittest $MY_DIR/test_petals.py

backend/python/rerankers/Makefile

@@ -1,7 +1,6 @@
 .PHONY: rerankers
 rerankers: protogen
-	$(MAKE) -C ../common-env/transformers
-
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -24,4 +23,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/rerankers/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/rerankers/requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+rerankers[transformers]
+grpcio==1.63.0
+protobuf
+certifi
+transformers==4.38.2

backend/python/rerankers/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the reranker server with conda
+## A bash script wrapper that runs the GRPC backend
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/reranker.py $@
+python $MY_DIR/reranker.py $@

backend/python/rerankers/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the reranker server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_reranker.py
+python -m unittest $MY_DIR/test_reranker.py

backend/python/sentencetransformers/Makefile

@@ -1,6 +1,6 @@
 .PHONY: sentencetransformers
 sentencetransformers: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash ./install.sh
 
 
 .PHONY: run
@@ -24,4 +24,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/sentencetransformers/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/sentencetransformers/requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+sentence-transformers==2.5.1
+transformers==4.38.2
+grpcio==1.63.0
+protobuf
+certifi

backend/python/sentencetransformers/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the sentencetransformers server with conda
+## A bash script wrapper that runs the GRPC backend
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/sentencetransformers.py $@
+python $MY_DIR/sentencetransformers.py $@

backend/python/sentencetransformers/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the sentencetransformers server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_sentencetransformers.py
+python -m unittest $MY_DIR/test_sentencetransformers.py

backend/python/transformers-musicgen/Makefile

@@ -1,6 +1,6 @@
 .PHONY: transformers-musicgen
 transformers-musicgen: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -22,4 +22,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/transformers-musicgen/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/transformers-musicgen/requirements.txt

@@ -0,0 +1,7 @@
+accelerate
+transformers==4.38.2
+grpcio==1.63.0
+protobuf
+torch
+scipy==1.13.0
+certifi

backend/python/transformers-musicgen/run.sh

@@ -1,16 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the transformers-musicgen server with conda
+## A bash script wrapper that runs the GRPC backend
 
-echo "Launching gRPC server for transformers-musicgen"
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-export PATH=$PATH:/opt/conda/bin
+source $MY_DIR/venv/bin/activate
 
-# Activate conda environment
-source activate transformers
-
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/transformers_server.py $@
+python $MY_DIR/transformers_server.py $@

backend/python/transformers-musicgen/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_transformers.py
+python -m unittest $MY_DIR/test_transformers.py

backend/python/transformers/Makefile

@@ -1,6 +1,6 @@
 .PHONY: transformers
 transformers: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -23,4 +23,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/transformers/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/transformers/requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+transformers==4.38.2
+grpcio==1.63.0
+protobuf
+torch
+certifi

backend/python/transformers/run.sh

@@ -1,20 +1,17 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that runs the GRPC backend
 
 
 if [ -d "/opt/intel" ]; then
     # Assumes we are using the Intel oneAPI container image
     # https://github.com/intel/intel-extension-for-pytorch/issues/538
     export XPU=1
-else
-    export PATH=$PATH:/opt/conda/bin
-    # Activate conda environment
-    source activate transformers
 fi
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-python $DIR/transformers_server.py $@
+source $MY_DIR/venv/bin/activate
+
+python $MY_DIR/transformers_server.py $@

backend/python/transformers/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_transformers_server.py
+python -m unittest $MY_DIR/test_transformers_server.py

backend/python/transformers/transformers_server.py

@@ -150,11 +150,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     devices = Core().available_devices
                     if "GPU" in " ".join(devices):
                         device_map="AUTO:GPU"
-
+                # While working on a fine tuned model, inference may give an inaccuracy and performance drop on GPU if winograd convolutions are selected. 
+                # https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html
+                if "CPU" or "NPU" in device_map:
+                    if "-CPU" or "-NPU" not in device_map:
+                        ovconfig={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"}
+                else:
+                    ovconfig={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT","GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"}
                 self.model = OVModelForCausalLM.from_pretrained(model_name, 
                                                                 compile=True,
                                                                 trust_remote_code=request.TrustRemoteCode,
-                                                                ov_config={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT","GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"}, 
+                                                                ov_config=ovconfig,
                                                                 device=device_map)
                 self.OV = True
             elif request.Type == "OVModelForFeatureExtraction":
@@ -168,11 +174,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     devices = Core().available_devices
                     if "GPU" in " ".join(devices):
                         device_map="AUTO:GPU"
-
+                # While working on a fine tuned model, inference may give an inaccuracy and performance drop on GPU if winograd convolutions are selected. 
+                # https://docs.openvino.ai/2024/openvino-workflow/running-inference/inference-devices-and-modes/gpu-device.html
+                if "CPU" or "NPU" in device_map:
+                    if "-CPU" or "-NPU" not in device_map:
+                        ovconfig={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"}
+                else:
+                    ovconfig={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT","GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"}
                 self.model = OVModelForFeatureExtraction.from_pretrained(model_name, 
                                                                 compile=True,
                                                                 trust_remote_code=request.TrustRemoteCode,
-                                                                ov_config={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT", "GPU_DISABLE_WINOGRAD_CONVOLUTION": "YES"}, 
+                                                                ov_config=ovconfig, 
                                                                 export=True,
                                                                 device=device_map)
                 self.OV = True
@@ -234,8 +246,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
         # Pool to get sentence embeddings; i.e. generate one 1024 vector for the entire sentence
         sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
-        print("Calculated embeddings for: " + request.Embeddings, file=sys.stderr)
-        print("Embeddings:", sentence_embeddings, file=sys.stderr)
+#        print("Calculated embeddings for: " + request.Embeddings, file=sys.stderr)
+#        print("Embeddings:", sentence_embeddings, file=sys.stderr)
         return backend_pb2.EmbeddingResult(embeddings=sentence_embeddings[0])
 
     async def _predict(self, request, context, streaming=False): 

backend/python/vall-e-x/.gitignore

@@ -0,0 +1 @@
+source

backend/python/vall-e-x/Makefile

@@ -4,7 +4,6 @@ endif
 
 .PHONY: ttsvalle
 ttsvalle: protogen
-	$(MAKE) -C ../common-env/transformers
 	bash install.sh
 
 .PHONY: run
@@ -27,4 +26,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf source venv

backend/python/vall-e-x/install.sh

@@ -1,21 +1,34 @@
 #!/bin/bash
+set -ex
 
 ##
 ## A bash script installs the required dependencies of VALL-E-X and prepares the environment
-export SHA=3faaf8ccadb154d63b38070caf518ce9309ea0f4
+export VALL_E_X_VERSION=3faaf8ccadb154d63b38070caf518ce9309ea0f4
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-SKIP_CONDA=${SKIP_CONDA:-0}
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
 
-if [ $SKIP_CONDA -ne 1 ]; then
-    source activate transformers
-else
-    export PATH=$PATH:/opt/conda/bin
-    CONDA_PREFIX=$PWD
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
 fi
 
-git clone https://github.com/Plachtaa/VALL-E-X.git $CONDA_PREFIX/vall-e-x && pushd $CONDA_PREFIX/vall-e-x && git checkout -b build $SHA && popd
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
 
-cp -rfv $CONDA_PREFIX/vall-e-x/* ./
+git clone https://github.com/Plachtaa/VALL-E-X.git $MY_DIR/source
+pushd $MY_DIR/source && git checkout -b build $VALL_E_X_VERSION && popd
+
+uv pip install --requirement ${MY_DIR}/source/requirements.txt
+
+cp -rfv ./*py $MY_DIR/source/  
 
 if [ "$PIP_CACHE_PURGE" = true ] ; then
     pip cache purge

backend/python/vall-e-x/requirements.txt

@@ -0,0 +1,4 @@
+accelerate
+grpcio==1.63.0
+protobuf
+certifi

backend/python/vall-e-x/run.sh

@@ -1,15 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the ttsvalle server with conda
-export PATH=$PATH:/opt/conda/bin
+## A bash script wrapper that runs the GRPC backend
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-cd $DIR
-
-python $DIR/ttsvalle.py $@
+pushd $MY_DIR/source && python ttsvalle.py $@

backend/python/vall-e-x/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the ttsvalle server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test.py
+pushd $MY_DIR/source && python -m unittest $MY_DIR/test.py

backend/python/vllm/Makefile

@@ -1,6 +1,6 @@
 .PHONY: vllm
 vllm: protogen
-	$(MAKE) -C ../common-env/transformers
+	bash install.sh
 
 .PHONY: run
 run: protogen
@@ -22,4 +22,8 @@ protogen-clean:
 	$(RM) backend_pb2_grpc.py backend_pb2.py
 
 backend_pb2_grpc.py backend_pb2.py:
-	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+	python3 -m grpc_tools.protoc -I../.. --python_out=. --grpc_python_out=. backend.proto
+
+.PHONY: clean
+clean: protogen-clean
+	rm -rf venv

backend/python/vllm/install.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -ex
+
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+
+uv venv ${MY_DIR}/venv
+source ${MY_DIR}/venv/bin/activate
+
+uv pip install --requirement ${MY_DIR}/requirements.txt
+
+if [ -f "requirements-${BUILD_TYPE}.txt" ]; then
+    uv pip install --requirement ${MY_DIR}/requirements-${BUILD_TYPE}.txt
+fi
+
+if [ -d "/opt/intel" ]; then
+    # Intel GPU: If the directory exists, we assume we are using the Intel image
+    # https://github.com/intel/intel-extension-for-pytorch/issues/538
+    if [ -f "requirements-intel.txt" ]; then
+        uv pip install --index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ --requirement ${MY_DIR}/requirements-intel.txt
+    fi
+fi
+
+if [ "$PIP_CACHE_PURGE" = true ] ; then
+    pip cache purge
+fi

backend/python/vllm/requirements.txt

@@ -0,0 +1,6 @@
+accelerate
+vllm
+grpcio==1.63.0
+protobuf
+certifi
+transformers==4.38.2

backend/python/vllm/run.sh

@@ -1,14 +1,10 @@
 #!/bin/bash
 
 ##
-## A bash script wrapper that runs the diffusers server with conda
+## A bash script wrapper that runs the GRPC backend
 
-export PATH=$PATH:/opt/conda/bin
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# Activate conda environment
-source activate transformers
+source $MY_DIR/venv/bin/activate
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-python $DIR/backend_vllm.py $@
+python $MY_DIR/backend_vllm.py $@

backend/python/vllm/test.sh

@@ -1,11 +1,10 @@
 #!/bin/bash
+
 ##
-## A bash script wrapper that runs the transformers server with conda
+## A bash script wrapper that runs the tests
 
-# Activate conda environment
-source activate transformers
+MY_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
 
-# get the directory where the bash script is located
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source $MY_DIR/venv/bin/activate
 
-python -m unittest $DIR/test_backend_vllm.py
+python -m unittest $MY_DIR/test_backend_vllm.py

core/config/backend_config_loader.go

@@ -182,6 +182,12 @@ func (cl *BackendConfigLoader) GetAllBackendConfigs() []BackendConfig {
 	return res
 }
 
+func (cl *BackendConfigLoader) RemoveBackendConfig(m string) {
+	cl.Lock()
+	defer cl.Unlock()
+	delete(cl.configs, m)
+}
+
 func (cl *BackendConfigLoader) ListBackendConfigs() []string {
 	cl.Lock()
 	defer cl.Unlock()

core/http/app.go

@@ -20,6 +20,7 @@ import (
 	"github.com/gofiber/contrib/fiberzerolog"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/cors"
+	"github.com/gofiber/fiber/v2/middleware/favicon"
 	"github.com/gofiber/fiber/v2/middleware/filesystem"
 	"github.com/gofiber/fiber/v2/middleware/recover"
 
@@ -182,8 +183,16 @@ func App(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *confi
 	}
 	routes.RegisterJINARoutes(app, cl, ml, appConfig, auth)
 
+	httpFS := http.FS(embedDirStatic)
+
+	app.Use(favicon.New(favicon.Config{
+		URL:        "/favicon.ico",
+		FileSystem: httpFS,
+		File:       "static/favicon.ico",
+	}))
+
 	app.Use("/static", filesystem.New(filesystem.Config{
-		Root:       http.FS(embedDirStatic),
+		Root:       httpFS,
 		PathPrefix: "static",
 		Browse:     true,
 	}))

core/http/elements/gallery.go

@@ -2,6 +2,7 @@ package elements
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/chasefleming/elem-go"
 	"github.com/chasefleming/elem-go/attrs"
@@ -13,7 +14,12 @@ const (
 	NoImage = "https://upload.wikimedia.org/wikipedia/commons/6/65/No-Image-Placeholder.svg"
 )
 
-func DoneProgress(uid, text string) string {
+func DoneProgress(galleryID, text string, showDelete bool) string {
+	// Split by @ and grab the name
+	if strings.Contains(galleryID, "@") {
+		galleryID = strings.Split(galleryID, "@")[1]
+	}
+
 	return elem.Div(
 		attrs.Props{},
 		elem.H3(
@@ -25,10 +31,11 @@ func DoneProgress(uid, text string) string {
 			},
 			elem.Text(text),
 		),
+		elem.If(showDelete, deleteButton(galleryID), reInstallButton(galleryID)),
 	).Render()
 }
 
-func ErrorProgress(err string) string {
+func ErrorProgress(err, galleryName string) string {
 	return elem.Div(
 		attrs.Props{},
 		elem.H3(
@@ -38,8 +45,9 @@ func ErrorProgress(err string) string {
 				"tabindex":  "-1",
 				"autofocus": "",
 			},
-			elem.Text("Error"+err),
+			elem.Text("Error "+err),
 		),
+		installButton(galleryName),
 	).Render()
 }
 
@@ -67,7 +75,7 @@ func StartProgressBar(uid, progress, text string) string {
 	return elem.Div(attrs.Props{
 		"hx-trigger": "done",
 		"hx-get":     "/browse/job/" + uid,
-		"hx-swap":    "outerHTML",
+		"hx-swap":    "innerHTML",
 		"hx-target":  "this",
 	},
 		elem.H3(
@@ -99,7 +107,119 @@ func cardSpan(text, icon string) elem.Node {
 		elem.I(attrs.Props{
 			"class": icon + " pr-2",
 		}),
+
 		elem.Text(text),
+
+		//elem.Text(text),
+	)
+}
+
+func searchableElement(text, icon string) elem.Node {
+	return elem.Form(
+		attrs.Props{},
+		elem.Input(
+			attrs.Props{
+				"type":  "hidden",
+				"name":  "search",
+				"value": text,
+			},
+		),
+		elem.Span(
+			attrs.Props{
+				"class": "inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2",
+			},
+
+			elem.A(
+				attrs.Props{
+					//	"name":      "search",
+					//	"value":     text,
+					//"class":     "inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2",
+					"href":      "#!",
+					"hx-post":   "/browse/search/models",
+					"hx-target": "#search-results",
+					// TODO: this doesn't work
+					//	"hx-vals":      `{ \"search\": \"` + text + `\" }`,
+					"hx-indicator": ".htmx-indicator",
+				},
+				elem.I(attrs.Props{
+					"class": icon + " pr-2",
+				}),
+				elem.Text(text),
+			),
+		),
+
+		//elem.Text(text),
+	)
+}
+
+func link(text, url string) elem.Node {
+	return elem.A(
+		attrs.Props{
+			"class":  "inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2",
+			"href":   url,
+			"target": "_blank",
+		},
+		elem.I(attrs.Props{
+			"class": "fas fa-link pr-2",
+		}),
+		elem.Text(text),
+	)
+}
+func installButton(galleryName string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-right inline-block rounded bg-primary px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-swap":               "outerHTML",
+			// post the Model ID as param
+			"hx-post": "/browse/install/model/" + galleryName,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-download pr-2",
+			},
+		),
+		elem.Text("Install"),
+	)
+}
+
+func reInstallButton(galleryName string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"class":                 "float-right inline-block rounded bg-primary ml-2 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-swap":               "outerHTML",
+			// post the Model ID as param
+			"hx-post": "/browse/install/model/" + galleryName,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-arrow-rotate-right pr-2",
+			},
+		),
+		elem.Text("Reinstall"),
+	)
+}
+
+func deleteButton(modelName string) elem.Node {
+	return elem.Button(
+		attrs.Props{
+			"data-twe-ripple-init":  "",
+			"data-twe-ripple-color": "light",
+			"hx-confirm":            "Are you sure you wish to delete the model?",
+			"class":                 "float-right inline-block rounded bg-red-800 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-red-accent-300 hover:shadow-red-2 focus:bg-red-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-red-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
+			"hx-swap":               "outerHTML",
+			// post the Model ID as param
+			"hx-post": "/browse/delete/model/" + modelName,
+		},
+		elem.I(
+			attrs.Props{
+				"class": "fa-solid fa-cancel pr-2",
+			},
+		),
+		elem.Text("Delete"),
 	)
 }
 
@@ -114,43 +234,6 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
 	// 		elem.Text(s),
 	// 	)
 	// }
-	deleteButton := func(m *gallery.GalleryModel) elem.Node {
-		return elem.Button(
-			attrs.Props{
-				"data-twe-ripple-init":  "",
-				"data-twe-ripple-color": "light",
-				"class":                 "float-right inline-block rounded bg-red-800 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-red-accent-300 hover:shadow-red-2 focus:bg-red-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-red-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
-				"hx-swap":               "outerHTML",
-				// post the Model ID as param
-				"hx-post": "/browse/delete/model/" + m.Name,
-			},
-			elem.I(
-				attrs.Props{
-					"class": "fa-solid fa-cancel pr-2",
-				},
-			),
-			elem.Text("Delete"),
-		)
-	}
-
-	installButton := func(m *gallery.GalleryModel) elem.Node {
-		return elem.Button(
-			attrs.Props{
-				"data-twe-ripple-init":  "",
-				"data-twe-ripple-color": "light",
-				"class":                 "float-right inline-block rounded bg-primary px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-primary-accent-300 hover:shadow-primary-2 focus:bg-primary-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-primary-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong",
-				"hx-swap":               "outerHTML",
-				// post the Model ID as param
-				"hx-post": "/browse/install/model/" + fmt.Sprintf("%s@%s", m.Gallery.Name, m.Name),
-			},
-			elem.I(
-				attrs.Props{
-					"class": "fa-solid fa-download pr-2",
-				},
-			),
-			elem.Text("Install"),
-		)
-	}
 
 	descriptionDiv := func(m *gallery.GalleryModel) elem.Node {
 
@@ -187,25 +270,26 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
 			)
 		}
 
+		tagsNodes := []elem.Node{}
 		for _, tag := range m.Tags {
-			nodes = append(nodes,
-				cardSpan(tag, "fas fa-tag"),
+			tagsNodes = append(tagsNodes,
+				searchableElement(tag, "fas fa-tag"),
 			)
 		}
 
+		nodes = append(nodes,
+			elem.Div(
+				attrs.Props{
+					"class": "flex flex-row flex-wrap content-center",
+				},
+				tagsNodes...,
+			),
+		)
+
 		for i, url := range m.URLs {
 			nodes = append(nodes,
-				elem.A(
-					attrs.Props{
-						"class":  "inline-block bg-gray-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2",
-						"href":   url,
-						"target": "_blank",
-					},
-					elem.I(attrs.Props{
-						"class": "fas fa-link pr-2",
-					}),
-					elem.Text("Link #"+fmt.Sprintf("%d", i+1)),
-				))
+				link("Link #"+fmt.Sprintf("%d", i+1), url),
+			)
 		}
 
 		return elem.Div(
@@ -224,12 +308,12 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
 					elem.Raw(StartProgressBar(installing.Get(galleryID), "0", "Installing")),
 				), // Otherwise, show install button (if not installed) or display "Installed"
 				elem.If(m.Installed,
-					//elem.Node(elem.Div(
-					//		attrs.Props{},
-					//	span("Installed"), deleteButton(m),
-					//	)),
-					deleteButton(m),
-					installButton(m),
+					elem.Node(elem.Div(
+						attrs.Props{},
+						reInstallButton(m.ID()),
+						deleteButton(m.Name),
+					)),
+					installButton(m.ID()),
 				),
 			),
 		)
@@ -243,11 +327,19 @@ func ListModels(models []*gallery.GalleryModel, installing *xsync.SyncedMap[stri
 			m.Icon = NoImage
 		}
 
+		divProperties := attrs.Props{
+			"class": "flex justify-center items-center",
+		}
+
+		_, trustRemoteCodeExists := m.Overrides["trust_remote_code"]
+		if trustRemoteCodeExists {
+			// should this be checking for trust_remote_code: false? I don't think we ever use that value.
+			divProperties["class"] = divProperties["class"] + " remote-code"
+		}
+
 		elems = append(elems,
 
-			elem.Div(attrs.Props{
-				"class": "flex justify-center items-center",
-			},
+			elem.Div(divProperties,
 				elem.A(attrs.Props{
 					"href": "#!",
 					//		"class": "justify-center items-center",

core/http/endpoints/localai/gallery.go

@@ -61,11 +61,11 @@ func (mgs *ModelGalleryEndpointService) ApplyModelGalleryEndpoint() func(c *fibe
 			return err
 		}
 		mgs.galleryApplier.C <- gallery.GalleryOp{
-			Req:         input.GalleryModel,
-			Id:          uuid.String(),
-			GalleryName: input.ID,
-			Galleries:   mgs.galleries,
-			ConfigURL:   input.ConfigURL,
+			Req:              input.GalleryModel,
+			Id:               uuid.String(),
+			GalleryModelName: input.ID,
+			Galleries:        mgs.galleries,
+			ConfigURL:        input.ConfigURL,
 		}
 		return c.JSON(struct {
 			ID        string `json:"uuid"`
@@ -79,8 +79,8 @@ func (mgs *ModelGalleryEndpointService) DeleteModelGalleryEndpoint() func(c *fib
 		modelName := c.Params("name")
 
 		mgs.galleryApplier.C <- gallery.GalleryOp{
-			Delete:      true,
-			GalleryName: modelName,
+			Delete:           true,
+			GalleryModelName: modelName,
 		}
 
 		uuid, err := uuid.NewUUID()

core/http/endpoints/localai/welcome.go

@@ -3,6 +3,7 @@ package localai
 import (
 	"github.com/go-skynet/LocalAI/core/config"
 	"github.com/go-skynet/LocalAI/internal"
+	"github.com/go-skynet/LocalAI/pkg/gallery"
 	"github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/gofiber/fiber/v2"
 )
@@ -13,11 +14,22 @@ func WelcomeEndpoint(appConfig *config.ApplicationConfig,
 		models, _ := ml.ListModels()
 		backendConfigs := cl.GetAllBackendConfigs()
 
+		galleryConfigs := map[string]*gallery.Config{}
+		for _, m := range backendConfigs {
+
+			cfg, err := gallery.GetLocalModelConfiguration(ml.ModelPath, m.Name)
+			if err != nil {
+				continue
+			}
+			galleryConfigs[m.Name] = cfg
+		}
+
 		summary := fiber.Map{
 			"Title":             "LocalAI API - " + internal.PrintableVersion(),
 			"Version":           internal.PrintableVersion(),
 			"Models":            models,
 			"ModelsConfig":      backendConfigs,
+			"GalleryConfig":     galleryConfigs,
 			"ApplicationConfig": appConfig,
 		}
 

core/http/routes/ui.go

@@ -3,6 +3,7 @@ package routes
 import (
 	"fmt"
 	"html/template"
+	"sort"
 	"strings"
 
 	"github.com/go-skynet/LocalAI/core/config"
@@ -34,11 +35,24 @@ func RegisterUIRoutes(app *fiber.App,
 	app.Get("/browse", auth, func(c *fiber.Ctx) error {
 		models, _ := gallery.AvailableGalleryModels(appConfig.Galleries, appConfig.ModelPath)
 
+		// Get all available tags
+		allTags := map[string]struct{}{}
+		tags := []string{}
+		for _, m := range models {
+			for _, t := range m.Tags {
+				allTags[t] = struct{}{}
+			}
+		}
+		for t := range allTags {
+			tags = append(tags, t)
+		}
+		sort.Strings(tags)
 		summary := fiber.Map{
 			"Title":        "LocalAI - Models",
 			"Version":      internal.PrintableVersion(),
 			"Models":       template.HTML(elements.ListModels(models, installingModels)),
 			"Repositories": appConfig.Galleries,
+			"AllTags":      tags,
 			//	"ApplicationConfig": appConfig,
 		}
 
@@ -92,9 +106,9 @@ func RegisterUIRoutes(app *fiber.App,
 		installingModels.Set(galleryID, uid)
 
 		op := gallery.GalleryOp{
-			Id:          uid,
-			GalleryName: galleryID,
-			Galleries:   appConfig.Galleries,
+			Id:               uid,
+			GalleryModelName: galleryID,
+			Galleries:        appConfig.Galleries,
 		}
 		go func() {
 			galleryService.C <- op
@@ -118,12 +132,13 @@ func RegisterUIRoutes(app *fiber.App,
 		installingModels.Set(galleryID, uid)
 
 		op := gallery.GalleryOp{
-			Id:          uid,
-			Delete:      true,
-			GalleryName: galleryID,
+			Id:               uid,
+			Delete:           true,
+			GalleryModelName: galleryID,
 		}
 		go func() {
 			galleryService.C <- op
+			cl.RemoveBackendConfig(galleryID)
 		}()
 
 		return c.SendString(elements.StartProgressBar(uid, "0", "Deletion"))
@@ -146,7 +161,7 @@ func RegisterUIRoutes(app *fiber.App,
 			return c.SendString(elements.ProgressBar("100"))
 		}
 		if status.Error != nil {
-			return c.SendString(elements.ErrorProgress(status.Error.Error()))
+			return c.SendString(elements.ErrorProgress(status.Error.Error(), status.GalleryModelName))
 		}
 
 		return c.SendString(elements.ProgressBar(fmt.Sprint(status.Progress)))
@@ -158,18 +173,22 @@ func RegisterUIRoutes(app *fiber.App,
 
 		status := galleryService.GetStatus(c.Params("uid"))
 
+		galleryID := ""
 		for _, k := range installingModels.Keys() {
 			if installingModels.Get(k) == c.Params("uid") {
+				galleryID = k
 				installingModels.Delete(k)
 			}
 		}
 
+		showDelete := true
 		displayText := "Installation completed"
 		if status.Deletion {
+			showDelete = false
 			displayText = "Deletion completed"
 		}
 
-		return c.SendString(elements.DoneProgress(c.Params("uid"), displayText))
+		return c.SendString(elements.DoneProgress(galleryID, displayText, showDelete))
 	})
 
 	// Show the Chat page
@@ -191,7 +210,8 @@ func RegisterUIRoutes(app *fiber.App,
 		backendConfigs := cl.GetAllBackendConfigs()
 
 		if len(backendConfigs) == 0 {
-			return c.SendString("No models available")
+			// If no model is available redirect to the index which suggests how to install models
+			return c.Redirect("/")
 		}
 
 		summary := fiber.Map{
@@ -224,7 +244,8 @@ func RegisterUIRoutes(app *fiber.App,
 		backendConfigs := cl.GetAllBackendConfigs()
 
 		if len(backendConfigs) == 0 {
-			return c.SendString("No models available")
+			// If no model is available redirect to the index which suggests how to install models
+			return c.Redirect("/")
 		}
 
 		summary := fiber.Map{
@@ -257,7 +278,8 @@ func RegisterUIRoutes(app *fiber.App,
 		backendConfigs := cl.GetAllBackendConfigs()
 
 		if len(backendConfigs) == 0 {
-			return c.SendString("No models available")
+			// If no model is available redirect to the index which suggests how to install models
+			return c.Redirect("/")
 		}
 
 		summary := fiber.Map{

core/http/static/general.css

@@ -72,6 +72,16 @@ body {
     margin: 0.5rem;
 }
 
+.remote-code { /* Attempt to make this stand out */
+    outline-style: solid;
+    outline-color: red;
+    outline-width: 0.33rem;
+}
+
+.remote-code::after {
+    content: "\0026A0 Trust Remote Code Required \0026A0"
+}
+
 ul {
     list-style-type: disc; /* Adds bullet points */
     padding-left: 1.25rem; /* Indents the list from the left margin */

core/http/views/index.html

@@ -16,16 +16,31 @@
             <p class="mt-4 text-lg">The FOSS alternative to OpenAI, Claude, ...</p>
             <a href="https://localai.io" target="_blank" class="mt-4 inline-block bg-blue-500 text-white py-2 px-4 rounded-lg shadow transition duration-300 ease-in-out hover:bg-blue-700 hover:shadow-lg">
                 <i class="fas fa-book-reader pr-2"></i>Documentation
-            </a>
+            </a>    
         </div>
 
         <div class="models mt-12">
+            {{ if eq (len .ModelsConfig) 0 }}
+            <h2 class="text-center text-3xl font-semibold text-gray-100"> <i class="text-yellow-200 ml-2 fa-solid fa-triangle-exclamation animate-pulse"></i> Ouch! seems you don't have any models installed!</h2>
+            <p class="text-center mt-4 text-xl">..install something from the <a class="text-gray-400 hover:text-white ml-1 px-3 py-2 rounded" href="/browse">🖼️ Gallery</a> or check the <a href="https://localai.io/basics/getting_started/" class="text-gray-400 hover:text-white ml-1 px-3 py-2 rounded"> <i class="fa-solid fa-book"></i> Getting started documentation </a></p>
+            {{ else }}
             <h2 class="text-center text-3xl font-semibold text-gray-100">Installed models</h2>
             <p class="text-center mt-4 text-xl">We have {{len .ModelsConfig}} pre-loaded models available.</p>
             <ul class="mt-8 space-y-4">
+                {{$galleryConfig:=.GalleryConfig}}
                 {{ range .ModelsConfig }}
+                {{ $cfg:= index $galleryConfig .Name}}
                 <li class="bg-gray-800 border border-gray-700 p-4 rounded-lg">
                     <div class="flex justify-between items-center">
+            
+                        <img {{ if $cfg.Icon }}
+                            src="{{$cfg.Icon}}"
+                            {{ else }}
+                            src="https://upload.wikimedia.org/wikipedia/commons/6/65/No-Image-Placeholder.svg"
+                            {{ end }}
+                            class="rounded-t-lg max-h-24 max-w-24 object-cover mt-3"
+                            >
+                     
                         <p class="font-bold text-white flex items-center"><i class="fas fa-brain pr-2"></i>{{.Name}}</p>
                         {{ if .Backend }}
                         <!-- Badge for Backend -->
@@ -37,11 +52,16 @@
                             auto
                         </span>
                         {{ end }}
+
+                        <button 
+                            class="float-right inline-block rounded bg-red-800 px-6 pb-2.5 mb-3 pt-2.5 text-xs font-medium uppercase leading-normal text-white shadow-primary-3 transition duration-150 ease-in-out hover:bg-red-accent-300 hover:shadow-red-2 focus:bg-red-accent-300 focus:shadow-primary-2 focus:outline-none focus:ring-0 active:bg-red-600 active:shadow-primary-2 dark:shadow-black/30 dark:hover:shadow-dark-strong dark:focus:shadow-dark-strong dark:active:shadow-dark-strong" 
+                            data-twe-ripple-color="light" data-twe-ripple-init="" hx-confirm="Are you sure you wish to delete the model?" hx-post="/browse/delete/model/{{.Name}}" hx-swap="outerHTML"><i class="fa-solid fa-cancel pr-2"></i>Delete</button>
                     </div>
                     <!-- Additional details can go here -->
                 </li>
                 {{ end }}
             </ul>
+            {{ end }}
         </div>
     </div>
 

core/http/views/models.html

@@ -13,8 +13,56 @@
                 🖼️ Available models from <i>{{ len .Repositories }}</i> repositories     <a href="https://localai.io/models/" target="_blank" >
                     <i class="fas fa-circle-info pr-2"></i>
                 </a></h2> 
-            
-        
+
+            <div class="text-center font-semibold text-gray-100">
+                <h2>Filter by type:</h2>
+                <button  hx-post="/browse/search/models"
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "tts"}'
+                hx-indicator=".htmx-indicator" >TTS</button> 
+                <button  hx-post="/browse/search/models" 
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "stablediffusion"}'
+                hx-indicator=".htmx-indicator" >Image generation</button> 
+                <button  hx-post="/browse/search/models" \
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "llm"}'
+                hx-indicator=".htmx-indicator" >Text generation</button> 
+                <button  hx-post="/browse/search/models" 
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "multimodal"}'
+                hx-indicator=".htmx-indicator" >Multimodal</button> 
+                <button  hx-post="/browse/search/models" 
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "embedding"}'
+                hx-indicator=".htmx-indicator" >Embeddings</button>
+                <button  hx-post="/browse/search/models"
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "rerank"}'
+                hx-indicator=".htmx-indicator" >Rerankers</button> 
+                <button  
+                    hx-post="/browse/search/models"
+                    class="text-white-500 inline-block bg-blue-200 rounded-full px-3 py-1 text-sm font-semibold text-gray-700 mr-2 mb-2 hover:bg-gray-300 hover:shadow-gray-2"
+                    hx-target="#search-results" 
+                    hx-vals='{"search": "whisper"}'
+                hx-indicator=".htmx-indicator" >Audio transcription</button> 
+            </div>
+
+            <div class="text-center text-xs font-semibold text-gray-100">
+                Filter by tags:
+                {{ range .AllTags }}
+                    <button  hx-post="/browse/search/models" class="text-blue-500" hx-target="#search-results" 
+                    hx-vals='{"search": "{{.}}"}'
+                    hx-indicator=".htmx-indicator" >{{.}}</button> 
+                {{ end }}
+            </div>
+
             <span class="htmx-indicator loader"></span>
             <input class="form-control appearance-none block w-full px-3 py-2 text-base font-normal text-gray-300 pb-2 mb-5 bg-gray-800 bg-clip-padding border border-solid border-gray-600 rounded transition ease-in-out m-0 focus:text-gray-300 focus:bg-gray-900 focus:border-blue-500 focus:outline-none" type="search" 
                 name="search" placeholder="Begin Typing To Search models..."