mirror of
https://github.com/mudler/LocalAI.git
synced 2024-06-07 19:40:48 +00:00
1981154f49
* fix: initial work towards not committing generated files to the repository Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * feat: improve build docs Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: remove unused folder from .dockerignore and .gitignore Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: attempt to fix extra backend tests Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: attempt to fix other tests Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: more test fixes Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: fix apple tests Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: more extras tests fixes Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: add GOBIN to PATH in docker build Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: extra tests and Dockerfile corrections Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: remove build dependency checks Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: add golang protobuf compilers to tests-linux action Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: ensure protogen is run for extra backend installs Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: use newer protobuf Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: more missing protoc binaries Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: missing dependencies during docker build Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: don't install grpc compilers in the final stage if they aren't needed Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: python-grpc-tools in 22.04 repos is too old Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: add a couple of extra build dependencies to Makefile Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> * fix: unbreak container rebuild functionality Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com> --------- Signed-off-by: Chris Jowett <421501+cryptk@users.noreply.github.com>
327 lines
17 KiB
YAML
327 lines
17 KiB
YAML
---
|
|
name: 'Tests extras backends'
|
|
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches:
|
|
- master
|
|
tags:
|
|
- '*'
|
|
|
|
concurrency:
|
|
group: ci-tests-extra-${{ github.head_ref || github.ref }}-${{ github.repository }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
tests-transformers:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
sudo apt-get install -y libopencv-dev
|
|
pip install --user grpcio-tools
|
|
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
|
|
- name: Test transformers
|
|
run: |
|
|
export PATH=$PATH:/opt/conda/bin
|
|
make --jobs=5 --output-sync=target -C backend/python/transformers
|
|
make --jobs=5 --output-sync=target -C backend/python/transformers test
|
|
|
|
tests-sentencetransformers:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
sudo apt-get install -y libopencv-dev
|
|
pip install --user grpcio-tools
|
|
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
|
|
- name: Test sentencetransformers
|
|
run: |
|
|
export PATH=$PATH:/opt/conda/bin
|
|
make --jobs=5 --output-sync=target -C backend/python/sentencetransformers
|
|
make --jobs=5 --output-sync=target -C backend/python/sentencetransformers test
|
|
|
|
tests-diffusers:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
sudo apt-get install -y libopencv-dev
|
|
pip install --user grpcio-tools
|
|
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
|
|
- name: Test diffusers
|
|
run: |
|
|
export PATH=$PATH:/opt/conda/bin
|
|
make --jobs=5 --output-sync=target -C backend/python/diffusers
|
|
make --jobs=5 --output-sync=target -C backend/python/diffusers test
|
|
|
|
|
|
tests-transformers-musicgen:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
sudo apt-get install -y libopencv-dev
|
|
pip install --user grpcio-tools
|
|
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
|
|
- name: Test transformers-musicgen
|
|
run: |
|
|
export PATH=$PATH:/opt/conda/bin
|
|
make --jobs=5 --output-sync=target -C backend/python/transformers-musicgen
|
|
make --jobs=5 --output-sync=target -C backend/python/transformers-musicgen test
|
|
|
|
|
|
|
|
# tests-petals:
|
|
# runs-on: ubuntu-latest
|
|
# steps:
|
|
# - name: Clone
|
|
# uses: actions/checkout@v4
|
|
# with:
|
|
# submodules: true
|
|
# - name: Dependencies
|
|
# run: |
|
|
# sudo apt-get update
|
|
# sudo apt-get install build-essential ffmpeg
|
|
# curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
# sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
# gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
# sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
# sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
# sudo apt-get update && \
|
|
# sudo apt-get install -y conda
|
|
# sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
# sudo apt-get install -y libopencv-dev
|
|
# pip install --user grpcio-tools
|
|
|
|
# sudo rm -rfv /usr/bin/conda || true
|
|
|
|
# - name: Test petals
|
|
# run: |
|
|
# export PATH=$PATH:/opt/conda/bin
|
|
# make --jobs=5 --output-sync=target -C backend/python/petals
|
|
# make --jobs=5 --output-sync=target -C backend/python/petals test
|
|
|
|
|
|
|
|
# tests-bark:
|
|
# runs-on: ubuntu-latest
|
|
# steps:
|
|
# - name: Release space from worker
|
|
# run: |
|
|
# echo "Listing top largest packages"
|
|
# pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
|
|
# head -n 30 <<< "${pkgs}"
|
|
# echo
|
|
# df -h
|
|
# echo
|
|
# sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true
|
|
# sudo apt-get remove --auto-remove android-sdk-platform-tools || true
|
|
# sudo apt-get purge --auto-remove android-sdk-platform-tools || true
|
|
# sudo rm -rf /usr/local/lib/android
|
|
# sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true
|
|
# sudo rm -rf /usr/share/dotnet
|
|
# sudo apt-get remove -y '^mono-.*' || true
|
|
# sudo apt-get remove -y '^ghc-.*' || true
|
|
# sudo apt-get remove -y '.*jdk.*|.*jre.*' || true
|
|
# sudo apt-get remove -y 'php.*' || true
|
|
# sudo apt-get remove -y hhvm powershell firefox monodoc-manual msbuild || true
|
|
# sudo apt-get remove -y '^google-.*' || true
|
|
# sudo apt-get remove -y azure-cli || true
|
|
# sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true
|
|
# sudo apt-get remove -y '^gfortran-.*' || true
|
|
# sudo apt-get remove -y microsoft-edge-stable || true
|
|
# sudo apt-get remove -y firefox || true
|
|
# sudo apt-get remove -y powershell || true
|
|
# sudo apt-get remove -y r-base-core || true
|
|
# sudo apt-get autoremove -y
|
|
# sudo apt-get clean
|
|
# echo
|
|
# echo "Listing top largest packages"
|
|
# pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
|
|
# head -n 30 <<< "${pkgs}"
|
|
# echo
|
|
# sudo rm -rfv build || true
|
|
# sudo rm -rf /usr/share/dotnet || true
|
|
# sudo rm -rf /opt/ghc || true
|
|
# sudo rm -rf "/usr/local/share/boost" || true
|
|
# sudo rm -rf "$AGENT_TOOLSDIRECTORY" || true
|
|
# df -h
|
|
# - name: Clone
|
|
# uses: actions/checkout@v4
|
|
# with:
|
|
# submodules: true
|
|
# - name: Dependencies
|
|
# run: |
|
|
# sudo apt-get update
|
|
# sudo apt-get install build-essential ffmpeg
|
|
# curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
# sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
# gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
# sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
# sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
# sudo apt-get update && \
|
|
# sudo apt-get install -y conda
|
|
# sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
# sudo apt-get install -y libopencv-dev
|
|
# pip install --user grpcio-tools
|
|
|
|
# sudo rm -rfv /usr/bin/conda || true
|
|
|
|
# - name: Test bark
|
|
# run: |
|
|
# export PATH=$PATH:/opt/conda/bin
|
|
# make --jobs=5 --output-sync=target -C backend/python/bark
|
|
# make --jobs=5 --output-sync=target -C backend/python/bark test
|
|
|
|
|
|
# Below tests needs GPU. Commented out for now
|
|
# TODO: Re-enable as soon as we have GPU nodes
|
|
# tests-vllm:
|
|
# runs-on: ubuntu-latest
|
|
# steps:
|
|
# - name: Clone
|
|
# uses: actions/checkout@v4
|
|
# with:
|
|
# submodules: true
|
|
# - name: Dependencies
|
|
# run: |
|
|
# sudo apt-get update
|
|
# sudo apt-get install build-essential ffmpeg
|
|
# curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
# sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
# gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
# sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
# sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
# sudo apt-get update && \
|
|
# sudo apt-get install -y conda
|
|
# sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
# sudo apt-get install -y libopencv-dev
|
|
# pip install --user grpcio-tools
|
|
# sudo rm -rfv /usr/bin/conda || true
|
|
# - name: Test vllm
|
|
# run: |
|
|
# export PATH=$PATH:/opt/conda/bin
|
|
# make --jobs=5 --output-sync=target -C backend/python/vllm
|
|
# make --jobs=5 --output-sync=target -C backend/python/vllm test
|
|
tests-vallex:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch python3-pip
|
|
sudo apt-get install -y libopencv-dev
|
|
pip install --user grpcio-tools
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
- name: Test vall-e-x
|
|
run: |
|
|
export PATH=$PATH:/opt/conda/bin
|
|
make --jobs=5 --output-sync=target -C backend/python/vall-e-x
|
|
make --jobs=5 --output-sync=target -C backend/python/vall-e-x test
|
|
|
|
tests-coqui:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch espeak espeak-ng python3-pip
|
|
pip install --user grpcio-tools
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
|
|
- name: Test coqui
|
|
run: |
|
|
export PATH=$PATH:/opt/conda/bin
|
|
make --jobs=5 --output-sync=target -C backend/python/coqui
|
|
make --jobs=5 --output-sync=target -C backend/python/coqui test
|