From d9b63fae7ccac3e01512e9736f6eec780c17ec1c Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 8 Oct 2024 09:24:19 +0200 Subject: [PATCH] chore(tests): improve rwkv tests and consume TEST_FLAKES (#3765) chores(tests): improve rwkv tests and consume TEST_FLAKES consistently use TEST_FLAKES and reduce flakiness of rwkv tests by being case insensitive Signed-off-by: Ettore Di Giacinto --- Makefile | 14 +++++++------- core/http/app_test.go | 5 +++-- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 86437b67..af0e2e84 100644 --- a/Makefile +++ b/Makefile @@ -470,13 +470,13 @@ run-e2e-image: run-e2e-aio: protogen-go @echo 'Running e2e AIO tests' - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts 5 -v -r ./tests/e2e-aio + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e-aio test-e2e: @echo 'Running e2e tests' BUILD_TYPE=$(BUILD_TYPE) \ LOCALAI_API=http://$(E2E_BRIDGE_IP):5390/v1 \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts 5 -v -r ./tests/e2e + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --flake-attempts $(TEST_FLAKES) -v -r ./tests/e2e teardown-e2e: rm -rf $(TEST_DIR) || true @@ -484,24 +484,24 @@ teardown-e2e: test-llama: prepare-test TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="llama" --flake-attempts 5 -v -r $(TEST_PATHS) + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="llama" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) test-llama-gguf: prepare-test TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="llama-gguf" --flake-attempts 5 -v -r $(TEST_PATHS) + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="llama-gguf" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) test-tts: prepare-test TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="tts" --flake-attempts 1 -v -r $(TEST_PATHS) + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="tts" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) test-stablediffusion: prepare-test TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="stablediffusion" --flake-attempts 1 -v -r $(TEST_PATHS) + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="stablediffusion" --flake-attempts $(TEST_FLAKES) -v -r $(TEST_PATHS) test-stores: backend-assets/grpc/local-store mkdir -p tests/integration/backend-assets/grpc cp -f backend-assets/grpc/local-store tests/integration/backend-assets/grpc/ - $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="stores" --flake-attempts 1 -v -r tests/integration + $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="stores" --flake-attempts $(TEST_FLAKES) -v -r tests/integration test-container: docker build --target requirements -t local-ai-test-container . diff --git a/core/http/app_test.go b/core/http/app_test.go index bbe52c34..871a1a32 100644 --- a/core/http/app_test.go +++ b/core/http/app_test.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "github.com/mudler/LocalAI/core/config" . "github.com/mudler/LocalAI/core/http" @@ -950,7 +951,7 @@ var _ = Describe("API test", func() { openai.ChatCompletionRequest{Model: "rwkv_test", Messages: []openai.ChatCompletionMessage{{Content: "Can you count up to five?", Role: "user"}}}) Expect(err).ToNot(HaveOccurred()) Expect(len(resp.Choices) > 0).To(BeTrue()) - Expect(resp.Choices[0].Message.Content).To(Or(ContainSubstring("Sure"), ContainSubstring("five"))) + Expect(strings.ToLower(resp.Choices[0].Message.Content)).To(Or(ContainSubstring("sure"), ContainSubstring("five"))) stream, err := client.CreateChatCompletionStream(context.TODO(), openai.ChatCompletionRequest{Model: "rwkv_test", Messages: []openai.ChatCompletionMessage{{Content: "Can you count up to five?", Role: "user"}}}) Expect(err).ToNot(HaveOccurred()) @@ -969,7 +970,7 @@ var _ = Describe("API test", func() { tokens++ } Expect(text).ToNot(BeEmpty()) - Expect(text).To(Or(ContainSubstring("Sure"), ContainSubstring("five"))) + Expect(strings.ToLower(text)).To(Or(ContainSubstring("sure"), ContainSubstring("five"))) Expect(tokens).ToNot(Or(Equal(1), Equal(0))) })