feat(apisix): add Cloudron package

- Implements Apache APISIX packaging for Cloudron platform. - Includes Dockerfile, CloudronManifest.json, and start.sh. - Configured to use Cloudron's etcd addon. 🤖 Generated with Gemini CLI Co-Authored-By: Gemini <noreply@google.com>
2025-09-04 09:42:47 -05:00
parent f7bae09f22
commit 54cc5f7308
1608 changed files with 388342 additions and 0 deletions
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/delayetcd/delayetcd.go
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/delayetcd/delayetcd.go
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package delayetcd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+	"github.com/gavv/httpexpect"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/apache/apisix/t/chaos/utils"
+)
+
+func getEtcdDelayChaos(delay int) *v1alpha1.NetworkChaos {
+	return &v1alpha1.NetworkChaos{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "etcd-delay",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: v1alpha1.NetworkChaosSpec{
+			Selector: v1alpha1.SelectorSpec{
+				LabelSelectors: map[string]string{"app.kubernetes.io/instance": "etcd"},
+			},
+			Action: v1alpha1.DelayAction,
+			Mode:   v1alpha1.AllPodMode,
+			TcParameter: v1alpha1.TcParameter{
+				Delay: &v1alpha1.DelaySpec{
+					Latency: strconv.Itoa(delay) + "ms",
+				},
+			},
+		},
+	}
+}
+
+func setRouteMultipleTimes(e *httpexpect.Expect, times int, status httpexpect.StatusRange) time.Duration {
+	now := time.Now()
+	timeLast := now
+	var timeList []string
+	for i := 0; i < times; i++ {
+		utils.SetRoute(e, status)
+		timeList = append(timeList, time.Since(timeLast).String())
+		timeLast = time.Now()
+	}
+	fmt.Fprintf(ginkgo.GinkgoWriter, "takes %v separately\n", timeList)
+	return time.Since(now) / time.Duration(times)
+}
+
+func setRouteMultipleTimesIgnoreError(e *httpexpect.Expect, times int) (time.Duration, int) {
+	now := time.Now()
+	var resp *httpexpect.Response
+	for i := 0; i < times; i++ {
+		resp = utils.SetRouteIgnoreError(e)
+	}
+	// use status code of the last time is enough to show the accessibility of apisix
+	return time.Since(now) / time.Duration(times), resp.Raw().StatusCode
+}
+
+func deleteChaosAndCheck(eSilent *httpexpect.Expect, cliSet *utils.ClientSet, chaos *v1alpha1.NetworkChaos) {
+	err := cliSet.CtrlCli.Delete(context.Background(), chaos)
+	gomega.Expect(err).To(gomega.BeNil())
+	time.Sleep(1 * time.Second)
+
+	var setDuration time.Duration
+	var statusCode int
+	for range [10]int{} {
+		setDuration, statusCode = setRouteMultipleTimesIgnoreError(eSilent, 5)
+		if setDuration < 15*time.Millisecond && statusCode == http.StatusOK {
+			break
+		}
+		time.Sleep(5 * time.Second)
+	}
+	gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+	gomega.Ω(statusCode).Should(gomega.BeNumerically("==", http.StatusOK))
+}
+
+var _ = ginkgo.Describe("Test APISIX Delay When Add ETCD Delay", func() {
+	ctx := context.Background()
+	e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
+	eDataPanel := httpexpect.New(ginkgo.GinkgoT(), utils.DataPanelHost)
+	ePrometheus := httpexpect.New(ginkgo.GinkgoT(), utils.PrometheusHost)
+	eSilent := utils.GetSilentHttpexpectClient()
+
+	var cliSet *utils.ClientSet
+	var apisixPod *v1.Pod
+	var err error
+	ginkgo.It("init client set", func() {
+		cliSet, err = utils.InitClientSet()
+		gomega.Expect(err).To(gomega.BeNil())
+		listOption := client.MatchingLabels{"app": "apisix-gw"}
+		apisixPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(len(apisixPods)).Should(gomega.BeNumerically(">", 0))
+		apisixPod = &apisixPods[0]
+	})
+
+	ginkgo.It("setup prometheus metrics public API", func() {
+		utils.SetPrometheusMetricsPublicAPI(e)
+	})
+
+	ginkgo.It("check if everything works", func() {
+		utils.SetRoute(e, httpexpect.Status2xx)
+		utils.GetRouteList(e, http.StatusOK)
+
+		utils.WaitUntilMethodSucceed(eDataPanel, http.MethodGet, 1)
+		utils.TestPrometheusEtcdMetric(ePrometheus, 1)
+	})
+
+	// get default
+	ginkgo.It("get default apisix delay", func() {
+		timeStart := time.Now()
+		setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+		gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 15*time.Millisecond))
+
+		errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+	})
+
+	// 30ms delay
+	ginkgo.It("generate a 30ms delay between etcd and apisix", func() {
+		timeStart := time.Now()
+		chaos := getEtcdDelayChaos(30)
+		err := cliSet.CtrlCli.Create(ctx, chaos)
+		gomega.Expect(err).To(gomega.BeNil())
+		time.Sleep(1 * time.Second)
+
+		defer deleteChaosAndCheck(eSilent, cliSet, chaos)
+
+		setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+		gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 400*time.Millisecond))
+
+		errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+	})
+
+	// 300ms delay
+	ginkgo.It("generate a 300ms delay between etcd and apisix", func() {
+		timeStart := time.Now()
+		chaos := getEtcdDelayChaos(300)
+		err := cliSet.CtrlCli.Create(ctx, chaos)
+		gomega.Expect(err).To(gomega.BeNil())
+		time.Sleep(1 * time.Second)
+
+		defer deleteChaosAndCheck(eSilent, cliSet, chaos)
+
+		setDuration := setRouteMultipleTimes(eSilent, 5, httpexpect.Status2xx)
+		gomega.Ω(setDuration).Should(gomega.BeNumerically("<", 4*time.Second))
+
+		errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("error"))
+	})
+
+	// 3s delay and cause error
+	ginkgo.It("generate a 3s delay between etcd and apisix", func() {
+		timeStart := time.Now()
+		chaos := getEtcdDelayChaos(3000)
+		err := cliSet.CtrlCli.Create(ctx, chaos)
+		gomega.Expect(err).To(gomega.BeNil())
+		time.Sleep(1 * time.Second)
+
+		defer deleteChaosAndCheck(eSilent, cliSet, chaos)
+
+		_ = setRouteMultipleTimes(e, 2, httpexpect.Status5xx)
+
+		errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(errorLog).Should(gomega.ContainSubstring("error"))
+	})
+
+	ginkgo.It("restore test environment", func() {
+		utils.WaitUntilMethodSucceed(e, http.MethodPut, 5)
+		utils.DeleteRoute(e)
+	})
+})
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/e2e.go
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/e2e.go
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	_ "github.com/apache/apisix/t/chaos/delayetcd"
+	_ "github.com/apache/apisix/t/chaos/killetcd"
+)
+
+func runChaos() {}
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/e2e_test.go
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/e2e_test.go
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+func TestRunChaos(t *testing.T) {
+	runChaos()
+	gomega.RegisterFailHandler(ginkgo.Fail)
+	ginkgo.RunSpecs(t, "chaos test suites")
+}
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/go.mod
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/go.mod
@@ -0,0 +1,52 @@
+module github.com/apache/apisix/t/chaos
+
+require (
+	github.com/ajg/form v1.5.1 // indirect
+	github.com/chaos-mesh/chaos-mesh v1.1.1
+	github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/gavv/httpexpect v2.0.0+incompatible
+	github.com/imkira/go-interpol v1.1.0 // indirect
+	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
+	github.com/moul/http2curl v1.0.0 // indirect
+	github.com/onsi/ginkgo v1.12.0
+	github.com/onsi/gomega v1.9.0
+	github.com/pkg/errors v0.9.1
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 // indirect
+	github.com/yudai/gojsondiff v1.0.0 // indirect
+	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
+	github.com/yudai/pp v2.0.1+incompatible // indirect
+	k8s.io/api v0.17.0
+	k8s.io/apimachinery v0.17.0
+	k8s.io/client-go v0.17.0
+	k8s.io/kubectl v0.0.0
+	k8s.io/kubernetes v1.17.2
+	sigs.k8s.io/controller-runtime v0.4.0
+)
+
+replace (
+	k8s.io/api => k8s.io/api v0.17.0
+	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.17.0
+	k8s.io/apimachinery => k8s.io/apimachinery v0.17.1-beta.0
+	k8s.io/apiserver => k8s.io/apiserver v0.17.0
+	k8s.io/cli-runtime => k8s.io/cli-runtime v0.17.0
+	k8s.io/client-go => k8s.io/client-go v0.17.0
+	k8s.io/cloud-provider => k8s.io/cloud-provider v0.17.0
+	k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.17.0
+	k8s.io/code-generator => k8s.io/code-generator v0.17.1-beta.0
+	k8s.io/component-base => k8s.io/component-base v0.17.0
+	k8s.io/cri-api => k8s.io/cri-api v0.17.1-beta.0
+	k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.17.0
+	k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.17.0
+	k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.17.0
+	k8s.io/kube-proxy => k8s.io/kube-proxy v0.17.0
+	k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.17.0
+	k8s.io/kubectl => k8s.io/kubectl v0.17.0
+	k8s.io/kubelet => k8s.io/kubelet v0.17.0
+	k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.17.0
+	k8s.io/metrics => k8s.io/metrics v0.17.0
+	k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.17.0
+)
+
+go 1.14
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/go.sum
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/go.sum
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/killetcd/killetcd.go
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/killetcd/killetcd.go
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package killetcd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+	"github.com/gavv/httpexpect"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/apache/apisix/t/chaos/utils"
+)
+
+var (
+	bandwidthBefore float64
+	durationBefore  float64
+	bpsBefore       float64
+	bandwidthAfter  float64
+	durationAfter   float64
+	bpsAfter        float64
+)
+
+func getEtcdKillChaos() *v1alpha1.PodChaos {
+	return &v1alpha1.PodChaos{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "kill-etcd",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: v1alpha1.PodChaosSpec{
+			Selector: v1alpha1.SelectorSpec{
+				LabelSelectors: map[string]string{"app.kubernetes.io/instance": "etcd"},
+			},
+			Action: v1alpha1.PodKillAction,
+			Mode:   v1alpha1.AllPodMode,
+			Scheduler: &v1alpha1.SchedulerSpec{
+				Cron: "@every 10m",
+			},
+		},
+	}
+}
+
+var _ = ginkgo.Describe("Test Get Success When Etcd Got Killed", func() {
+	e := httpexpect.New(ginkgo.GinkgoT(), utils.Host)
+	eDataPanel := httpexpect.New(ginkgo.GinkgoT(), utils.DataPanelHost)
+	ePrometheus := httpexpect.New(ginkgo.GinkgoT(), utils.PrometheusHost)
+
+	var cliSet *utils.ClientSet
+	var apisixPod *v1.Pod
+	var err error
+	ginkgo.It("init client set", func() {
+		cliSet, err = utils.InitClientSet()
+		gomega.Expect(err).To(gomega.BeNil())
+		listOption := client.MatchingLabels{"app": "apisix-gw"}
+		apisixPods, err := utils.GetPods(cliSet.CtrlCli, metav1.NamespaceDefault, listOption)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(len(apisixPods)).Should(gomega.BeNumerically(">", 0))
+		apisixPod = &apisixPods[0]
+	})
+
+	stopChan := make(chan bool)
+
+	ginkgo.It("setup prometheus metrics public API", func() {
+		utils.SetPrometheusMetricsPublicAPI(e)
+	})
+
+	ginkgo.It("check if everything works", func() {
+		utils.SetRoute(e, httpexpect.Status2xx)
+		utils.GetRouteList(e, http.StatusOK)
+
+		utils.WaitUntilMethodSucceed(eDataPanel, http.MethodGet, 1)
+		utils.TestPrometheusEtcdMetric(ePrometheus, 1)
+	})
+
+	ginkgo.It("run request in background", func() {
+		go func() {
+			defer ginkgo.GinkgoRecover()
+			for {
+				go func() {
+					defer ginkgo.GinkgoRecover()
+					utils.GetRoute(eDataPanel, http.StatusOK)
+				}()
+				time.Sleep(100 * time.Millisecond)
+				stopLoop := false
+				select {
+				case <-stopChan:
+					stopLoop = true
+				default:
+				}
+				if stopLoop {
+					break
+				}
+			}
+		}()
+	})
+	// wait 1 seconds to let first route access returns
+	time.Sleep(1 * time.Second)
+
+	ginkgo.It("get stats before kill etcd", func() {
+		timeStart := time.Now()
+		bandwidthBefore, durationBefore = utils.GetEgressBandwidthPerSecond(ePrometheus)
+		bpsBefore = bandwidthBefore / durationBefore
+		gomega.Expect(bpsBefore).NotTo(gomega.BeZero())
+
+		errorLog, err := utils.Log(apisixPod, cliSet.KubeCli, timeStart)
+		gomega.Expect(err).To(gomega.BeNil())
+		gomega.Ω(errorLog).ShouldNot(gomega.ContainSubstring("no healthy etcd endpoint available"))
+	})
+
+	// apply chaos to kill all etcd pods
+	ginkgo.It("kill all etcd pods", func() {
+		chaos := getEtcdKillChaos()
+		err := cliSet.CtrlCli.Create(context.Background(), chaos.DeepCopy())
+		gomega.Expect(err).To(gomega.BeNil())
+		time.Sleep(3 * time.Second)
+	})
+
+	// fail to set route since etcd is all killed
+	// while get route could still succeed
+	ginkgo.It("get stats after kill etcd", func() {
+		utils.SetRoute(e, httpexpect.Status5xx)
+		utils.GetRoute(eDataPanel, http.StatusOK)
+		utils.TestPrometheusEtcdMetric(ePrometheus, 0)
+
+		bandwidthAfter, durationAfter = utils.GetEgressBandwidthPerSecond(ePrometheus)
+		bpsAfter = bandwidthAfter / durationAfter
+	})
+
+	ginkgo.It("ingress bandwidth per second not change much", func() {
+		fmt.Fprintf(ginkgo.GinkgoWriter, "bandwidth before: %f, after: %f\n", bandwidthBefore, bandwidthAfter)
+		fmt.Fprintf(ginkgo.GinkgoWriter, "duration before: %f, after: %f\n", durationBefore, durationAfter)
+		fmt.Fprintf(ginkgo.GinkgoWriter, "bps before: %f, after: %f\n", bpsBefore, bpsAfter)
+		gomega.Expect(utils.RoughCompare(bpsBefore, bpsAfter)).To(gomega.BeTrue())
+	})
+
+	ginkgo.It("restore test environment", func() {
+		stopChan <- true
+		cliSet.CtrlCli.Delete(context.Background(), getEtcdKillChaos())
+		utils.WaitUntilMethodSucceed(e, http.MethodPut, 5)
+		utils.DeleteRoute(e)
+	})
+})
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/kubernetes/deployment.yaml
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/kubernetes/deployment.yaml
@@ -0,0 +1,115 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1  # for versions before 1.8.0 use apps/v1beta1, before 1.9.0 use apps/v1beta2
+kind: Deployment
+metadata:
+  labels:
+    app: apisix-gw
+  name: apisix-gw-deployment
+  # namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: apisix-gw
+  template:
+    metadata:
+      labels:
+        app: apisix-gw
+    spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: app
+                      operator: In
+                      values:
+                        - apisix-gw
+                topologyKey: kubernetes.io/hostname
+              weight: 100
+      initContainers:
+        - command:
+            - /bin/sh
+            - -c
+            - |
+              sysctl -w net.core.somaxconn=65535
+              sysctl -w net.ipv4.ip_local_port_range="1024 65535"
+              sysctl -w net.ipv4.tcp_max_syn_backlog=8192
+              sysctl -w fs.file-max=1048576
+              sysctl -w fs.inotify.max_user_instances=16384
+              sysctl -w fs.inotify.max_user_watches=524288
+              sysctl -w fs.inotify.max_queued_events=16384
+          image: busybox:latest
+          imagePullPolicy: IfNotPresent
+          name: init-sysctl
+          resources: {}
+          securityContext:
+            privileged: true
+            procMount: Default
+      restartPolicy: Always
+
+      containers:
+        - env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+          image: "apache/apisix:alpine-local"
+          imagePullPolicy: IfNotPresent
+          name: apisix-gw-deployment
+          ports:
+            - containerPort: 9080
+              name: http
+              protocol: TCP
+            - containerPort: 9443
+              name: https
+              protocol: TCP
+            - containerPort: 9180
+              name: admin-port
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 6
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 9080
+            timeoutSeconds: 1
+          volumeMounts:
+            - mountPath: /usr/local/apisix/conf/config.yaml
+              name: apisix-config-yaml-configmap
+              subPath: config.yaml
+            - mountPath: /etc/localtime
+              name: localtime
+              readOnly: true
+      volumes:
+        - configMap:
+            name: apisix-gw-config.yaml
+          name: apisix-config-yaml-configmap
+        - hostPath:
+            path: /etc/localtime
+            type: File
+          name: localtime
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/kubernetes/service.yaml
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/kubernetes/service.yaml
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: apisix-gw-lb
+  # namespace: default
+  labels:
+    app: apisix-gw  # useful for service discovery, for example, prometheus-operator.
+spec:
+  ports:
+    - name: http
+      port: 9080
+      protocol: TCP
+      targetPort: 9080
+    - name: https
+      port: 9443
+      protocol: TCP
+      targetPort: 9443
+    - name: admin-port
+      port: 9180
+      protocol: TCP
+      targetPort: 9180
+  selector:
+    app: apisix-gw
+  type: NodePort
+  externalTrafficPolicy: Local
+  # sessionAffinity: None
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/utils/Dockerfile
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/utils/Dockerfile
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG ENABLE_PROXY=false
+
+FROM openresty/openresty:1.21.4.2-alpine-fat AS production-stage
+
+ARG ENABLE_PROXY
+ARG APISIX_PATH
+COPY $APISIX_PATH ./apisix
+RUN set -x \
+    && (test "${ENABLE_PROXY}" != "true" || /bin/sed -i 's,http://dl-cdn.alpinelinux.org,https://mirrors.aliyun.com,g' /etc/apk/repositories) \
+    && apk add --no-cache --virtual .builddeps \
+    automake \
+    autoconf \
+    libtool \
+    pkgconfig \
+    cmake \
+    git \
+    openldap-dev \
+    pcre-dev \
+    sudo \
+    && cd apisix \
+    && git config --global url.https://github.com/.insteadOf git://github.com/ \
+    && make deps \
+    && cp -v bin/apisix /usr/bin/ \
+    && mv ../apisix /usr/local/apisix \
+    && apk del .builddeps build-base make unzip
+
+FROM alpine:3.13 AS last-stage
+
+ARG ENABLE_PROXY
+# add runtime for Apache APISIX
+RUN set -x \
+    && (test "${ENABLE_PROXY}" != "true" || /bin/sed -i 's,http://dl-cdn.alpinelinux.org,https://mirrors.aliyun.com,g' /etc/apk/repositories) \
+    && apk add --no-cache \
+        bash \
+        curl \
+        libstdc++ \
+        openldap \
+        pcre \
+        tzdata
+
+WORKDIR /usr/local/apisix
+
+COPY --from=production-stage /usr/local/openresty/ /usr/local/openresty/
+COPY --from=production-stage /usr/local/apisix/ /usr/local/apisix/
+COPY --from=production-stage /usr/bin/apisix /usr/bin/apisix
+
+# forward request and error logs to docker log collector
+RUN mkdir -p logs && touch logs/access.log && touch logs/error.log \
+    && ln -sf /dev/stdout /usr/local/apisix/logs/access.log \
+    && ln -sf /dev/stderr /usr/local/apisix/logs/error.log
+
+ENV PATH=$PATH:/usr/local/openresty/luajit/bin:/usr/local/openresty/nginx/sbin:/usr/local/openresty/bin
+
+EXPOSE 9080 9180 9443
+
+CMD ["sh", "-c", "/usr/bin/apisix init && /usr/bin/apisix init_etcd && /usr/local/openresty/bin/openresty -p /usr/local/apisix -g 'daemon off;'"]
+
+STOPSIGNAL SIGQUIT
+
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/utils/kube_utils.go
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/utils/kube_utils.go
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"time"
+
+	"github.com/chaos-mesh/chaos-mesh/api/v1alpha1"
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	clientScheme "k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/tools/remotecommand"
+	kubectlScheme "k8s.io/kubectl/pkg/scheme"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+)
+
+type ClientSet struct {
+	CtrlCli client.Client
+	KubeCli *kubernetes.Clientset
+}
+
+func InitClientSet() (*ClientSet, error) {
+	scheme := runtime.NewScheme()
+	v1alpha1.AddToScheme(scheme)
+	clientScheme.AddToScheme(scheme)
+
+	restConfig := config.GetConfigOrDie()
+	ctrlCli, err := client.New(restConfig, client.Options{Scheme: scheme})
+	if err != nil {
+		return nil, err
+	}
+	kubeCli, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &ClientSet{ctrlCli, kubeCli}, nil
+}
+
+func GetPods(cli client.Client, ns string, listOption client.MatchingLabels) ([]corev1.Pod, error) {
+	podList := &corev1.PodList{}
+	err := cli.List(context.Background(), podList, client.InNamespace(ns), listOption)
+	if err != nil {
+		return nil, err
+	}
+	return podList.Items, nil
+}
+
+func ExecInPod(cli *kubernetes.Clientset, pod *corev1.Pod, cmd string) (string, error) {
+	name := pod.GetName()
+	namespace := pod.GetNamespace()
+	// only get the first container, no harm for now
+	containerName := pod.Spec.Containers[0].Name
+
+	req := cli.CoreV1().RESTClient().Post().
+		Resource("pods").
+		Name(name).
+		Namespace(namespace).
+		SubResource("exec")
+
+	req.VersionedParams(&corev1.PodExecOptions{
+		Container: containerName,
+		Command:   []string{"/bin/sh", "-c", cmd},
+		Stdin:     false,
+		Stdout:    true,
+		Stderr:    true,
+		TTY:       false,
+	}, kubectlScheme.ParameterCodec)
+
+	var stdout, stderr bytes.Buffer
+	exec, err := remotecommand.NewSPDYExecutor(config.GetConfigOrDie(), "POST", req.URL())
+	if err != nil {
+		return "", errors.Wrapf(err, "error in creating NewSPDYExecutor for pod %s in ns: %s", name, namespace)
+	}
+	err = exec.Stream(remotecommand.StreamOptions{
+		Stdin:  nil,
+		Stdout: &stdout,
+		Stderr: &stderr,
+	})
+	if stderr.String() != "" {
+		stderror := errors.New(stderr.String())
+		return "", errors.Wrapf(stderror, "pod: %s\ncommand: %s", name, cmd)
+	}
+	if err != nil {
+		return "", errors.Wrapf(err, "error in streaming remote command: pod: %s in ns: %s\n command: %s", name, namespace, cmd)
+	}
+	return stdout.String(), nil
+}
+
+// Log print log of pod
+func Log(pod *corev1.Pod, c *kubernetes.Clientset, sinceTime time.Time) (string, error) {
+	podLogOpts := corev1.PodLogOptions{}
+	if !sinceTime.IsZero() {
+		podLogOpts.SinceTime = &metav1.Time{Time: sinceTime}
+	}
+
+	req := c.CoreV1().Pods(pod.Namespace).GetLogs(pod.Name, &podLogOpts)
+	podLogs, err := req.Stream()
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to open log stream for pod %s/%s", pod.GetNamespace(), pod.GetName())
+	}
+	defer podLogs.Close()
+
+	buf := new(bytes.Buffer)
+	_, err = io.Copy(buf, podLogs)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to copy information from podLogs to buf")
+	}
+	return buf.String(), nil
+}
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/utils/setup_chaos_utils.sh
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/utils/setup_chaos_utils.sh
@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -ex
+
+start_minikube() {
+    # pin the version until chaos mesh solves https://github.com/chaos-mesh/chaos-mesh/issues/2172
+    curl -LO "https://storage.googleapis.com/kubernetes-release/release/v1.21.4/bin/linux/amd64/kubectl"
+    chmod +x ./kubectl
+    sudo mv ./kubectl /usr/local/bin/kubectl
+
+    curl -LO https://storage.googleapis.com/minikube/releases/latest/minikube_latest_amd64.deb
+    sudo dpkg -i --force-architecture minikube_latest_amd64.deb
+    minikube start --kubernetes-version "v1.21.4"
+}
+
+modify_config() {
+    DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}')
+    echo "dns_resolver:
+  - ${DNS_IP}
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - \"http://etcd.default.svc.cluster.local:2379\"
+plugin_attr:
+  prometheus:
+    enable_export_server: false
+  " > ./conf/config.yaml
+}
+
+port_forward() {
+    apisix_pod_name=$(kubectl get pod -l app=apisix-gw -o 'jsonpath={.items[0].metadata.name}')
+    nohup kubectl port-forward svc/apisix-gw-lb 9080:9080 >/dev/null 2>&1 &
+    nohup kubectl port-forward svc/apisix-gw-lb 9180:9180 >/dev/null 2>&1 &
+    nohup kubectl port-forward $apisix_pod_name 9091:9091 >/dev/null 2>&1 &
+    ps aux | grep '[p]ort-forward'
+}
+
+"$@"
--- a/CloudronPackages/APISIX/apisix-source/t/chaos/utils/utils.go
+++ b/CloudronPackages/APISIX/apisix-source/t/chaos/utils/utils.go
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/gavv/httpexpect"
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+)
+
+var (
+	token = "edd1c9f034335f136f87ad84b625c8f1"
+	// TODO: refactor the code. We should move the endpoint from the expect to the http call.
+	// So we don't need to remember to pass the correct expect.
+	Host           = "http://127.0.0.1:9180"
+	DataPanelHost  = "http://127.0.0.1:9080"
+	PrometheusHost = "http://127.0.0.1:9080"
+	setRouteBody   = `{
+		"uri": "/get",
+		"plugins": {
+			"prometheus": {}
+		},
+		"upstream": {
+			"nodes": {
+				"httpbin.default.svc.cluster.local:8000": 1
+			},
+			"type": "roundrobin"
+		}
+	}`
+	ignoreErrorFuncMap = map[string]func(e *httpexpect.Expect) *httpexpect.Response{
+		http.MethodGet: GetRouteIgnoreError,
+		http.MethodPut: SetRouteIgnoreError,
+	}
+)
+
+type httpTestCase struct {
+	E                 *httpexpect.Expect
+	Method            string
+	Path              string
+	Body              string
+	Headers           map[string]string
+	IgnoreError       bool
+	ExpectStatus      int
+	ExpectBody        string
+	ExpectStatusRange httpexpect.StatusRange
+}
+
+func caseCheck(tc httpTestCase) *httpexpect.Response {
+	e := tc.E
+	var req *httpexpect.Request
+	switch tc.Method {
+	case http.MethodGet:
+		req = e.GET(tc.Path)
+	case http.MethodPut:
+		req = e.PUT(tc.Path)
+	case http.MethodDelete:
+		req = e.DELETE(tc.Path)
+	default:
+		panic("invalid HTTP method")
+	}
+
+	if req == nil {
+		panic("fail to init request")
+	}
+	for key, val := range tc.Headers {
+		req.WithHeader(key, val)
+	}
+	if tc.Body != "" {
+		req.WithText(tc.Body)
+	}
+
+	resp := req.Expect()
+	if tc.IgnoreError {
+		return resp
+	}
+
+	if tc.ExpectStatus != 0 {
+		resp.Status(tc.ExpectStatus)
+	}
+
+	if tc.ExpectStatusRange != 0 {
+		resp.StatusRange(tc.ExpectStatusRange)
+	}
+
+	if tc.ExpectBody != "" {
+		resp.Body().Contains(tc.ExpectBody)
+	}
+
+	return resp
+}
+
+func SetRoute(e *httpexpect.Expect, expectStatusRange httpexpect.StatusRange) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:                 e,
+		Method:            http.MethodPut,
+		Path:              "/apisix/admin/routes/1",
+		Headers:           map[string]string{"X-API-KEY": token},
+		Body:              setRouteBody,
+		ExpectStatusRange: expectStatusRange,
+	})
+}
+
+func SetRouteIgnoreError(e *httpexpect.Expect) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:           e,
+		Method:      http.MethodPut,
+		Path:        "/apisix/admin/routes/1",
+		Headers:     map[string]string{"X-API-KEY": token},
+		Body:        setRouteBody,
+		IgnoreError: true,
+	})
+}
+
+func GetRoute(e *httpexpect.Expect, expectStatus int) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:            e,
+		Method:       http.MethodGet,
+		Path:         "/get",
+		ExpectStatus: expectStatus,
+	})
+}
+
+func GetRouteIgnoreError(e *httpexpect.Expect) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:           e,
+		Method:      http.MethodGet,
+		Path:        "/get",
+		IgnoreError: true,
+	})
+}
+
+func GetRouteList(e *httpexpect.Expect, expectStatus int) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:            e,
+		Method:       http.MethodGet,
+		Path:         "/apisix/admin/routes",
+		Headers:      map[string]string{"X-API-KEY": token},
+		ExpectStatus: expectStatus,
+		ExpectBody:   "httpbin.default.svc.cluster.local",
+	})
+}
+
+func DeleteRoute(e *httpexpect.Expect) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:       e,
+		Method:  http.MethodDelete,
+		Path:    "/apisix/admin/routes/1",
+		Headers: map[string]string{"X-API-KEY": token},
+	})
+}
+
+func SetPrometheusMetricsPublicAPI(e *httpexpect.Expect) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:       e,
+		Method:  http.MethodPut,
+		Path:    "/apisix/admin/routes/metrics",
+		Headers: map[string]string{"X-API-KEY": token},
+		Body: `{
+			"uri": "/apisix/prometheus/metrics",
+			"plugins": {
+				"public-api": {}
+			},
+			"upstream": {
+				"nodes": {
+					"httpbin.default.svc.cluster.local:8000": 1
+				},
+				"type": "roundrobin"
+			}
+		}`,
+	})
+}
+
+func TestPrometheusEtcdMetric(e *httpexpect.Expect, expectEtcd int) *httpexpect.Response {
+	return caseCheck(httpTestCase{
+		E:          e,
+		Method:     http.MethodGet,
+		Path:       "/apisix/prometheus/metrics",
+		ExpectBody: fmt.Sprintf("apisix_etcd_reachable %d", expectEtcd),
+	})
+}
+
+// get the first line which contains the key
+func getPrometheusMetric(e *httpexpect.Expect, key string) string {
+	resp := caseCheck(httpTestCase{
+		E:      e,
+		Method: http.MethodGet,
+		Path:   "/apisix/prometheus/metrics",
+	})
+	resps := strings.Split(resp.Body().Raw(), "\n")
+	var targetLine string
+	for _, line := range resps {
+		if strings.Contains(line, key) {
+			targetLine = line
+			break
+		}
+	}
+	targetSlice := strings.Fields(targetLine)
+	gomega.Ω(len(targetSlice)).Should(gomega.BeNumerically("==", 2))
+	return targetSlice[1]
+}
+
+func GetEgressBandwidthPerSecond(e *httpexpect.Expect) (float64, float64) {
+	key := "apisix_bandwidth{type=\"egress\","
+	bandWidthString := getPrometheusMetric(e, key)
+	bandWidthStart, err := strconv.ParseFloat(bandWidthString, 64)
+	gomega.Expect(err).To(gomega.BeNil())
+	// after etcd got killed, it would take longer time to get the metrics
+	// so need to calculate the duration
+	timeStart := time.Now()
+
+	time.Sleep(10 * time.Second)
+	bandWidthString = getPrometheusMetric(e, key)
+	bandWidthEnd, err := strconv.ParseFloat(bandWidthString, 64)
+	gomega.Expect(err).To(gomega.BeNil())
+	duration := time.Since(timeStart)
+
+	return bandWidthEnd - bandWidthStart, duration.Seconds()
+}
+
+func GetSilentHttpexpectClient() *httpexpect.Expect {
+	return httpexpect.WithConfig(httpexpect.Config{
+		BaseURL:  Host,
+		Reporter: httpexpect.NewAssertReporter(ginkgo.GinkgoT()),
+		Printers: []httpexpect.Printer{
+			newSilentPrinter(ginkgo.GinkgoT()),
+		},
+	})
+}
+
+func WaitUntilMethodSucceed(e *httpexpect.Expect, method string, interval int) {
+	f, ok := ignoreErrorFuncMap[method]
+	gomega.Expect(ok).To(gomega.BeTrue())
+	resp := f(e)
+	if resp.Raw().StatusCode != http.StatusOK {
+		for i := range [60]int{} {
+			timeWait := fmt.Sprintf("wait for %ds\n", i*interval)
+			fmt.Fprint(ginkgo.GinkgoWriter, timeWait)
+			resp = f(e)
+			if resp.Raw().StatusCode != http.StatusOK {
+				time.Sleep(5 * time.Second)
+			} else {
+				break
+			}
+		}
+	}
+	gomega.Ω(resp.Raw().StatusCode).Should(gomega.BeNumerically("==", http.StatusOK))
+}
+
+func RoughCompare(a float64, b float64) bool {
+	ratio := a / b
+	if ratio < 1.3 && ratio > 0.7 {
+		return true
+	}
+	return false
+}
+
+type silentPrinter struct {
+	logger httpexpect.Logger
+}
+
+func newSilentPrinter(logger httpexpect.Logger) silentPrinter {
+	return silentPrinter{logger}
+}
+
+// Request implements Printer.Request.
+func (p silentPrinter) Request(req *http.Request) {
+}
+
+// Response implements Printer.Response.
+func (silentPrinter) Response(*http.Response, time.Duration) {
+}