mirror of
https://github.com/mudler/LocalAI.git
synced 2024-12-23 14:32:25 +00:00
93702e39d4
* feat(build): adjust number of parallel make jobs * fix: update make on MacOS from brew to support --output-sync argument * fix: cache grpc with version as part of key to improve validity of cache hits * fix: use gmake for tests-apple to use the updated GNU make version * fix: actually use the new make version for tests-apple * feat: parallelize tests-extra * feat: attempt to cache grpc build for docker images * fix: don't quote GRPC version * fix: don't cache go modules, we have limited cache space, better used elsewhere * fix: release with the same version of go that we test with * fix: don't fail on exporting cache layers * fix: remove deprecated BUILD_GRPC docker arg from Makefile
200 lines
8.4 KiB
YAML
200 lines
8.4 KiB
YAML
---
|
|
name: 'tests'
|
|
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches:
|
|
- master
|
|
tags:
|
|
- '*'
|
|
|
|
env:
|
|
GRPC_VERSION: v1.58.0
|
|
|
|
concurrency:
|
|
group: ci-tests-${{ github.head_ref || github.ref }}-${{ github.repository }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
tests-linux:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
go-version: ['1.21.x']
|
|
steps:
|
|
- name: Release space from worker
|
|
run: |
|
|
echo "Listing top largest packages"
|
|
pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
|
|
head -n 30 <<< "${pkgs}"
|
|
echo
|
|
df -h
|
|
echo
|
|
sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true
|
|
sudo apt-get remove --auto-remove android-sdk-platform-tools || true
|
|
sudo apt-get purge --auto-remove android-sdk-platform-tools || true
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo apt-get remove -y '^mono-.*' || true
|
|
sudo apt-get remove -y '^ghc-.*' || true
|
|
sudo apt-get remove -y '.*jdk.*|.*jre.*' || true
|
|
sudo apt-get remove -y 'php.*' || true
|
|
sudo apt-get remove -y hhvm powershell firefox monodoc-manual msbuild || true
|
|
sudo apt-get remove -y '^google-.*' || true
|
|
sudo apt-get remove -y azure-cli || true
|
|
sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true
|
|
sudo apt-get remove -y '^gfortran-.*' || true
|
|
sudo apt-get autoremove -y
|
|
sudo apt-get clean
|
|
echo
|
|
echo "Listing top largest packages"
|
|
pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
|
|
head -n 30 <<< "${pkgs}"
|
|
echo
|
|
sudo rm -rfv build || true
|
|
df -h
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Setup Go ${{ matrix.go-version }}
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: ${{ matrix.go-version }}
|
|
cache: false
|
|
# You can test your matrix by printing the current Go version
|
|
- name: Display Go version
|
|
run: go version
|
|
- name: Dependencies
|
|
run: |
|
|
sudo apt-get update
|
|
sudo apt-get install build-essential ffmpeg
|
|
curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg && \
|
|
sudo install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
|
|
gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list' && \
|
|
sudo /bin/bash -c 'echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" | tee -a /etc/apt/sources.list.d/conda.list' && \
|
|
sudo apt-get update && \
|
|
sudo apt-get install -y conda
|
|
sudo apt-get install -y ca-certificates cmake curl patch
|
|
sudo apt-get install -y libopencv-dev
|
|
|
|
sudo rm -rfv /usr/bin/conda || true
|
|
PATH=$PATH:/opt/conda/bin make -C backend/python/sentencetransformers
|
|
|
|
# Pre-build piper before we start tests in order to have shared libraries in place
|
|
make sources/go-piper && \
|
|
GO_TAGS="tts" make -C sources/go-piper piper.o && \
|
|
sudo cp -rfv sources/go-piper/piper-phonemize/pi/lib/. /usr/lib/ && \
|
|
# Pre-build stable diffusion before we install a newer version of abseil (not compatible with stablediffusion-ncn)
|
|
GO_TAGS="stablediffusion tts" GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build
|
|
- name: Cache grpc
|
|
id: cache-grpc
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: grpc
|
|
key: ${{ runner.os }}-grpc-${{ env.GRPC_VERSION }}
|
|
- name: Build grpc
|
|
if: steps.cache-grpc.outputs.cache-hit != 'true'
|
|
run: |
|
|
git clone --recurse-submodules -b ${{ env.GRPC_VERSION }} --depth 1 --jobs 5 --shallow-submodules https://github.com/grpc/grpc && \
|
|
cd grpc && mkdir -p cmake/build && cd cmake/build && cmake -DgRPC_INSTALL=ON \
|
|
-DgRPC_BUILD_TESTS=OFF \
|
|
../.. && sudo make --jobs 5
|
|
- name: Install gRPC
|
|
run: |
|
|
cd grpc && cd cmake/build && sudo make --jobs 5 install
|
|
- name: Test
|
|
run: |
|
|
GO_TAGS="stablediffusion tts" make --jobs 5 --output-sync=target test
|
|
- name: Setup tmate session if tests fail
|
|
if: ${{ failure() }}
|
|
uses: mxschmitt/action-tmate@v3
|
|
timeout-minutes: 5
|
|
|
|
tests-aio-container:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Release space from worker
|
|
run: |
|
|
echo "Listing top largest packages"
|
|
pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
|
|
head -n 30 <<< "${pkgs}"
|
|
echo
|
|
df -h
|
|
echo
|
|
sudo apt-get remove -y '^llvm-.*|^libllvm.*' || true
|
|
sudo apt-get remove --auto-remove android-sdk-platform-tools || true
|
|
sudo apt-get purge --auto-remove android-sdk-platform-tools || true
|
|
sudo rm -rf /usr/local/lib/android
|
|
sudo apt-get remove -y '^dotnet-.*|^aspnetcore-.*' || true
|
|
sudo rm -rf /usr/share/dotnet
|
|
sudo apt-get remove -y '^mono-.*' || true
|
|
sudo apt-get remove -y '^ghc-.*' || true
|
|
sudo apt-get remove -y '.*jdk.*|.*jre.*' || true
|
|
sudo apt-get remove -y 'php.*' || true
|
|
sudo apt-get remove -y hhvm powershell firefox monodoc-manual msbuild || true
|
|
sudo apt-get remove -y '^google-.*' || true
|
|
sudo apt-get remove -y azure-cli || true
|
|
sudo apt-get remove -y '^mongo.*-.*|^postgresql-.*|^mysql-.*|^mssql-.*' || true
|
|
sudo apt-get remove -y '^gfortran-.*' || true
|
|
sudo apt-get autoremove -y
|
|
sudo apt-get clean
|
|
echo
|
|
echo "Listing top largest packages"
|
|
pkgs=$(dpkg-query -Wf '${Installed-Size}\t${Package}\t${Status}\n' | awk '$NF == "installed"{print $1 "\t" $2}' | sort -nr)
|
|
head -n 30 <<< "${pkgs}"
|
|
echo
|
|
sudo rm -rfv build || true
|
|
df -h
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Build images
|
|
run: |
|
|
docker build --build-arg FFMPEG=true --build-arg IMAGE_TYPE=core --build-arg MAKEFLAGS="--jobs=5 --output-sync=target" -t local-ai:tests -f Dockerfile .
|
|
BASE_IMAGE=local-ai:tests DOCKER_AIO_IMAGE=local-ai-aio:test make docker-aio
|
|
- name: Test
|
|
run: |
|
|
LOCALAI_MODELS_DIR=$PWD/models LOCALAI_IMAGE_TAG=test LOCALAI_IMAGE=local-ai-aio \
|
|
make run-e2e-aio
|
|
- name: Setup tmate session if tests fail
|
|
if: ${{ failure() }}
|
|
uses: mxschmitt/action-tmate@v3
|
|
timeout-minutes: 5
|
|
|
|
tests-apple:
|
|
runs-on: macOS-14
|
|
strategy:
|
|
matrix:
|
|
go-version: ['1.21.x']
|
|
steps:
|
|
- name: Clone
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
- name: Setup Go ${{ matrix.go-version }}
|
|
uses: actions/setup-go@v4
|
|
with:
|
|
go-version: ${{ matrix.go-version }}
|
|
cache: false
|
|
# You can test your matrix by printing the current Go version
|
|
- name: Display Go version
|
|
run: go version
|
|
- name: Dependencies
|
|
run: |
|
|
brew install protobuf grpc make
|
|
- name: Test
|
|
run: |
|
|
export C_INCLUDE_PATH=/usr/local/include
|
|
export CPLUS_INCLUDE_PATH=/usr/local/include
|
|
# Used to run the newer GNUMake version from brew that supports --output-sync
|
|
export PATH="/opt/homebrew/opt/make/libexec/gnubin:$PATH"
|
|
BUILD_TYPE="GITHUB_CI_HAS_BROKEN_METAL" CMAKE_ARGS="-DLLAMA_F16C=OFF -DLLAMA_AVX512=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF" make --jobs 4 --output-sync=target test
|
|
- name: Setup tmate session if tests fail
|
|
if: ${{ failure() }}
|
|
uses: mxschmitt/action-tmate@v3
|
|
timeout-minutes: 5 |