[ Agent ] [ CMII ] - 新增版本更新的模式；新增部署模板；新增消息推送模块

2024-01-18 14:22:11 +08:00
parent 6df4587106
commit ed2769062e
20 changed files with 1050 additions and 34 deletions
--- a/agent-go/go.mod
+++ b/agent-go/go.mod
@@ -3,6 +3,7 @@ module agent-go
 go 1.18

 require (
+	github.com/go-openapi/errors v0.20.3
 	github.com/magiconair/properties v1.8.7
 	github.com/mittwald/goharbor-client/v5 v5.4.2
 	github.com/panjf2000/ants/v2 v2.7.2
@@ -25,7 +26,6 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
-	github.com/go-openapi/errors v0.20.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/loads v0.21.2 // indirect
--- a/agent-go/go.sum
+++ b/agent-go/go.sum
@@ -115,8 +115,6 @@ github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
 github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU=
 github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg=
-github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
-github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
--- a/agent-go/k8s_exec/CmiiAppDeploy.go
+++ b/agent-go/k8s_exec/CmiiAppDeploy.go
@@ -0,0 +1,45 @@
+package k8s_exec
+
+import (
+	"agent-go/utils"
+	"bytes"
+	v1 "k8s.io/api/apps/v1"
+	appsv1 "k8s.io/client-go/applyconfigurations/apps/v1"
+	"sigs.k8s.io/yaml"
+	"text/template"
+)
+
+type CmiiBackendDeploymentConfig struct {
+	Namespace  string
+	AppName    string
+	ImageTag   string
+	TagVersion string
+	Replicas   string
+}
+
+func (backend CmiiBackendDeploymentConfig) ParseToApplyConf() *appsv1.DeploymentApplyConfiguration {
+
+	// 解析模板
+	tmpl, err := template.New("cmiiBackendDeploymentTemplate").Parse(cmiiBackendDeploymentTemplate)
+	if err != nil {
+		panic(err)
+	}
+
+	// 应用数据并打印结果
+	var result bytes.Buffer
+	err = tmpl.Execute(&result, backend)
+	if err != nil {
+		panic(err)
+	}
+
+	// 创建Deployment对象
+	deployment := v1.Deployment{}
+	err = yaml.Unmarshal(result.Bytes(), &deployment)
+	if err != nil {
+		panic(err)
+	}
+
+	utils.BeautifulPrint(&deployment)
+
+	return nil
+}
--- a/agent-go/k8s_exec/CmiiAppDeployTemplate.go
+++ b/agent-go/k8s_exec/CmiiAppDeployTemplate.go
@@ -0,0 +1,120 @@
+package k8s_exec
+
+const cmiiBackendDeploymentTemplate = `
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .AppName }}
+  namespace: {{ .Namespace }}
+  labels:
+    cmii.type: backend
+    cmii.app: {{ .AppName }}
+    octopus/control: backend-app-1.0.0
+    app.kubernetes.io/managed-by: octopus/control
+    app.kubernetes.io/app-version: {{ .TagVersion }}
+spec:
+  replicas: {{ .Replicas }}
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      cmii.type: backend
+      cmii.app: {{ .AppName }}
+  template:
+    metadata:
+      labels:
+        cmii.type: backend
+        cmii.app: {{ .AppName }}
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: uavcloud.env
+                    operator: In
+                    values:
+                      - demo
+      imagePullSecrets:
+        - name: harborsecret
+      containers:
+        - name: {{ .AppName }}
+          image: "harbor.cdcyy.com.cn/cmii/{{ .AppName }}:{{ .ImageTag }}"
+          imagePullPolicy: Always
+          env:
+            - name: K8S_NAMESPACE
+              value: "{{ .Namespace }}"
+            - name: APPLICATION_NAME
+              value: "{{ .AppName }}"
+            - name: CUST_JAVA_OPTS
+              value: "-Xms500m -Xmx1500m -Dlog4j2.formatMsgNoLookups=true"
+            - name: NACOS_REGISTRY
+              value: "helm-nacos:8848"
+            - name: NACOS_DISCOVERY_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NACOS_DISCOVERY_PORT
+              value: "8080"
+            - name: BIZ_CONFIG_GROUP
+              value: {{ .TagVersion }}
+            - name: SYS_CONFIG_GROUP
+              value: {{ .TagVersion }}
+            - name: IMAGE_VERSION
+              value: {{ .TagVersion }}
+            - name: NACOS_USERNAME
+              value: "developer"
+            - name: NACOS_PASSWORD
+              value: "Deve@9128201"
+          ports:
+            - name: pod-port
+              containerPort: 8080
+              protocol: TCP
+          resources:
+             limits:
+               memory: 2Gi
+               cpu: 2
+             requests:
+               memory: 1Gi
+               cpu: 200m
+          livenessProbe:
+            httpGet:
+              path: /cmii/ping
+              port: pod-port
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            periodSeconds: 20
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /cmii/ping
+              port: pod-port
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            periodSeconds: 20
+            successThreshold: 1
+            failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /cmii/ping
+              port: pod-port
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 3
+            periodSeconds: 20
+            successThreshold: 1
+            failureThreshold: 5
+          volumeMounts:
+            - name: glusterfs-backend-log-volume
+              mountPath: /cmii/logs
+              readOnly: false
+              subPath: {{ .Namespace }}/{{ .AppName }}
+      volumes:
+        - name: glusterfs-backend-log-volume
+          persistentVolumeClaim:
+            claimName: glusterfs-backend-log-pvc
+    `
--- a/agent-go/k8s_exec/CmiiAppDeploy_test.go
+++ b/agent-go/k8s_exec/CmiiAppDeploy_test.go
@@ -0,0 +1,17 @@
+package k8s_exec
+
+import "testing"
+
+func TestCmiiBackendDeploymentConfig_ParseToApplyConf(t *testing.T) {
+
+	deploymentConfig := CmiiBackendDeploymentConfig{
+		Namespace:  "uavcloud-dev",
+		AppName:    "cmii-uav-gateway",
+		ImageTag:   "5.2.0-123",
+		TagVersion: "5.2.0",
+		Replicas:   "2",
+	}
+
+	deploymentConfig.ParseToApplyConf()
+
+}
--- a/agent-go/k8s_exec/CmiiK8sConfig.go
+++ b/agent-go/k8s_exec/CmiiK8sConfig.go
@@ -65,7 +65,7 @@ var CmiiBackendAppMap = map[string]string{
 }

 var CmiiFrontendAppMap = map[string]string{
-	"cmii-suav-platform-supervision":     "5 .2.0",
+	"cmii-suav-platform-supervision":     "5.2.0",
 	"cmii-suav-platform-supervisionh5":   "5.2.0",
 	"cmii-uav-platform":                  "5.2.0-011004",
 	"cmii-uav-platform-ai-brain":         "5.2.0",
@@ -93,6 +93,14 @@ var CmiiFrontendAppMap = map[string]string{
 	"cmii-uav-platform-visualization":    "5.2.0",
 }

+var CmiiMiddlewareNameMap = map[string]string{
+	"helm-nacos":    "single",
+	"helm-emqxs":    "single",
+	"helm-mysql":    "single",
+	"helm-redis":    "replication",
+	"helm-rabbitmq": "single",
+}
+
 var CmiiBackendAppName = []string{
 	"cmii-uav-gateway",
 	"cmii-uav-oauth",
--- a/agent-go/k8s_exec/CmiiK8sOperator.go
+++ b/agent-go/k8s_exec/CmiiK8sOperator.go
@@ -139,6 +139,21 @@ func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
 	return podList
 }

+func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
+
+	cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
+
+	for _, podInterface := range cmiiPodInterfaces {
+		for key, _ := range CmiiMiddlewareNameMap {
+			if strings.Contains(podInterface.Name, key) {
+				podList = append(podList, podInterface)
+			}
+		}
+	}
+
+	return podList
+}
+
 func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {

 	result := true
@@ -207,7 +222,13 @@ func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
 	// check if need to update
 	if cmiiDeploymentInterface.ImageTag == newTag {
 		log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
-		return true
+
+		// restart
+		if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
+			return true
+		} else {
+			return false
+		}
 	}

 	content := executor.BasicWordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 45)
@@ -240,6 +261,22 @@ func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
 	return true
 }

+func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
+
+	result = make(map[string]string, len(nameTagMap))
+	for appName, newTag := range nameTagMap {
+		if AppNameBelongsToCmiiImage(appName) {
+			if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
+				result[appName] = newTag
+			} else {
+				result[appName] = "false"
+			}
+		}
+	}
+
+	return result
+}
+
 func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {

 	if !executor.BasicFindContentInFile(updateLog, updateLogPath) {
@@ -449,3 +486,37 @@ func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface

 	return result
 }
+
+func FilterAllCmiiNodeSoft(nodeList []CmiiNodeInterface) (result []CmiiNodeInterface) {
+
+	for _, nodeInterface := range nodeList {
+
+		if strings.HasPrefix(nodeInterface.Name, "ai") {
+			continue
+		}
+
+		if strings.HasPrefix(nodeInterface.Name, "35") {
+			continue
+		}
+
+		result = append(result, nodeInterface)
+	}
+
+	return result
+}
+
+func AppNameBelongsToCmiiImage(appName string) bool {
+	_, ok := CmiiBackendAppMap[appName]
+	if !ok {
+		_, ok = CmiiFrontendAppMap[appName]
+		if !ok {
+			log.WarnF("[AppNameBelongsToCmiiImage] - [%s] not cmii app ！", appName)
+			return false
+		} else {
+			return true
+		}
+	} else {
+		return true
+	}
+
+}
--- a/agent-go/k8s_exec/CmiiK8sOperator_test.go
+++ b/agent-go/k8s_exec/CmiiK8sOperator_test.go
@@ -8,6 +8,12 @@ import (
 	"time"
 )

+var CmiiDevNamespaceList = []string{
+	"uavcloud-dev",
+	"uavcloud-devflight",
+	"uavcloud-devoperation",
+}
+
 func TestFindAppNotHealthyOrRestartCountGreaterThanN(t *testing.T) {

 	deploymentRestartCountGreaterThanN := FindAppNotHealthyOrRestartCountGreaterThanN("devflight", 10)
@@ -33,9 +39,19 @@ func TestFindDeploymentReplicasSmallerThanN(t *testing.T) {

 }

-func TestGetCmiiAllDeploymentFromEnv(t *testing.T) {
+func TestFindCmiiMiddlewarePodInterface(t *testing.T) {
+	middlewarePodInterface := FindCmiiMiddlewarePodInterface(devFlight)

-	BackupAllDeploymentFromEnv("uavms")
+	for _, middlePod := range middlewarePodInterface {
+		println()
+		utils.BeautifulPrint(middlePod)
+		println()
+	}
+}
+
+func TestBackupAllDeploymentFromEnv(t *testing.T) {
+
+	BackupAllDeploymentFromEnv("demo")

 }

@@ -47,20 +63,6 @@ func TestBackupAllCmiiDeploymentToMap(t *testing.T) {

 }

-func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
-
-	cmiiEnv := "demo"
-	appName := "cmii-uav-platform"
-	newTag := "5.2.0-011201"
-
-	tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
-	assert.Equal(t, tag, true, "update image tag failed !")
-
-	check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
-	assert.Equal(t, check, true, "deployment run failed!")
-
-}
-
 func TestRollBackCmiiDeploymentFromUpdateLog(t *testing.T) {
 	updateLog := RollBackCmiiDeploymentFromUpdateLog("2024-01-10-14-37-07 uavcloud-devflight       cmii-uav-depotautoreturn                12345678            123sdsa45678")

@@ -69,19 +71,21 @@ func TestRollBackCmiiDeploymentFromUpdateLog(t *testing.T) {

 func TestRestartCmiiBackendDeployment(t *testing.T) {

-	RestartCmiiBackendDeployment("test")
+	RestartCmiiBackendDeployment("dev")
 }

 func TestRestartCmiiFrontendDeployment(t *testing.T) {
-	RestartCmiiFrontendDeployment("devflight")
+	RestartCmiiFrontendDeployment("dev")
 }

 func TestFindDeploymentNotHealthy(t *testing.T) {
-	notHealthy := FindDeploymentNotHealthy("devflight")

-	notHealthy = FilterAllCmiiAppSoft(notHealthy)
-	for _, deploymentInterface := range notHealthy {
-		utils.BeautifulPrint(deploymentInterface)
+	for _, devNamespace := range CmiiDevNamespaceList {
+		notHealthy := FindDeploymentNotHealthy(devNamespace)
+		notHealthy = FilterAllCmiiAppSoft(notHealthy)
+		for _, deploymentInterface := range notHealthy {
+			utils.BeautifulPrint(deploymentInterface)
+		}
 	}
 }

@@ -91,6 +95,10 @@ func TestFindAllNodeNotHealthy(t *testing.T) {
 	elapsed := time.Since(start).Milliseconds()
 	fmt.Printf("执行耗时： %d ms\n", elapsed)

+	allNodeNotHealthy = FilterAllCmiiNodeSoft(allNodeNotHealthy)
+
+	assert.Equal(t, len(allNodeNotHealthy), 0, "have unhealthy pod !")
+
 	for _, nodeInterface := range allNodeNotHealthy {
 		println()
 		utils.BeautifulPrint(nodeInterface)
@@ -100,7 +108,8 @@ func TestFindAllNodeNotHealthy(t *testing.T) {
 }

 func TestFindPodNotHealthy(t *testing.T) {
-	podNotHealthy := FindPodNotHealthy("valida")
+
+	podNotHealthy := FindPodNotHealthy("devfl")
 	podNotHealthy = FilterAllCmiiPodSoft(podNotHealthy)

 	for _, podInterface := range podNotHealthy {
@@ -109,7 +118,7 @@ func TestFindPodNotHealthy(t *testing.T) {
 }

 func TestFindPodNotHealthy_And_Delete(t *testing.T) {
-	podNotHealthy := FindPodNotHealthy("uavms")
+	podNotHealthy := FindPodNotHealthy("devf")
 	podNotHealthy = FilterAllCmiiPodSoft(podNotHealthy)

 	for _, podInterface := range podNotHealthy {
@@ -133,3 +142,89 @@ func TestRestartDeploymentFromList(t *testing.T) {
 	RestartDeploymentFromList(allInterface)

 }
+
+func TestUpdateCmiiImageTagFromNameTagMap(t *testing.T) {
+
+	cmii530BackendMap := map[string]string{
+		"cmii-admin-data":               "5.3.0",
+		"cmii-admin-gateway":            "5.3.0",
+		"cmii-admin-user":               "5.3.0",
+		"cmii-open-gateway":             "5.3.0",
+		"cmii-suav-supervision":         "5.3.0",
+		"cmii-uav-airspace":             "5.3.0",
+		"cmii-uav-alarm":                "5.3.0",
+		"cmii-uav-brain":                "5.3.0",
+		"cmii-uav-cloud-live":           "5.3.0",
+		"cmii-uav-cms":                  "5.3.0",
+		"cmii-uav-data-post-process":    "5.3.0",
+		"cmii-uav-developer":            "5.3.0",
+		"cmii-uav-device":               "5.3.0",
+		"cmii-uav-emergency":            "5.3.0",
+		"cmii-uav-gateway":              "5.3.0",
+		"cmii-uav-gis-server":           "5.3.0",
+		"cmii-uav-industrial-portfolio": "5.3.0",
+		"cmii-uav-integration":          "5.3.0",
+		"cmii-uav-logger":               "5.3.0",
+		"cmii-uav-material-warehouse":   "5.3.0",
+		"cmii-uav-mission":              "5.3.0",
+		"cmii-uav-mqtthandler":          "5.3.0",
+		"cmii-uav-notice":               "5.3.0",
+		"cmii-uav-oauth":                "5.3.0",
+		"cmii-uav-process":              "5.3.0",
+		"cmii-uav-surveillance":         "5.3.0",
+		"cmii-uav-threedsimulation":     "5.3.0",
+		"cmii-uav-tower":                "5.3.0",
+		"cmii-uav-user":                 "5.3.0",
+		"cmii-uav-waypoint":             "5.3.0",
+		//"cmii-uav-grid-datasource":      "5.2.0-24810",
+		//"cmii-uav-grid-engine":          "5.1.0",
+		//"cmii-uav-grid-manage":          "5.1.0",
+	}
+
+	cmii530FrontendMap := map[string]string{
+		"cmii-suav-platform-supervision":     "5.3.0",
+		"cmii-suav-platform-supervisionh5":   "5.3.0",
+		"cmii-uav-platform":                  "5.3.0",
+		"cmii-uav-platform-ai-brain":         "5.3.0",
+		"cmii-uav-platform-armypeople":       "5.3.0",
+		"cmii-uav-platform-base":             "5.3.0",
+		"cmii-uav-platform-cms-portal":       "5.3.0",
+		"cmii-uav-platform-detection":        "5.3.0",
+		"cmii-uav-platform-emergency-rescue": "5.3.0",
+		"cmii-uav-platform-logistics":        "5.3.0",
+		"cmii-uav-platform-media":            "5.3.0",
+		"cmii-uav-platform-multiterminal":    "5.3.0",
+		"cmii-uav-platform-mws":              "5.3.0",
+		"cmii-uav-platform-oms":              "5.3.0",
+		"cmii-uav-platform-open":             "5.3.0",
+		"cmii-uav-platform-securityh5":       "5.3.0",
+		"cmii-uav-platform-seniclive":        "5.3.0",
+		"cmii-uav-platform-share":            "5.3.0",
+		"cmii-uav-platform-splice":           "5.3.0",
+		"cmii-uav-platform-threedsimulation": "5.3.0",
+		"cmii-uav-platform-visualization":    "5.3.0",
+		//"cmii-uav-platform-security":         "4.1.6",
+	}
+
+	result := UpdateCmiiImageTagFromNameTagMap("demo", cmii530BackendMap)
+	utils.BeautifulPrint(result)
+
+	result = UpdateCmiiImageTagFromNameTagMap("demo", cmii530FrontendMap)
+	utils.BeautifulPrint(result)
+
+}
+
+func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
+
+	cmiiEnv := "test"
+	appName := "cmii-suav-supervision"
+	newTag := "5.2.0-0117"
+
+	tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
+	assert.Equal(t, tag, true, "update image tag failed !")
+	utils.SplitLinePrint()
+
+	check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
+	assert.Equal(t, check, true, "deployment run failed!")
+
+}
--- a/agent-go/k8s_exec/K8sOperator.go
+++ b/agent-go/k8s_exec/K8sOperator.go
@@ -5,7 +5,7 @@ import (
 	"agent-go/logger"
 	"agent-go/utils"
 	"context"
-	"k8s.io/api/apps/v1"
+	v1 "k8s.io/api/apps/v1"
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -82,9 +82,9 @@ func (op *CmiiK8sOperator) changeOperatorEnv(cmiiEnv string) {
 	}

 	if strings.Contains(cmiiEnv, "dev") {
-		if strings.Contains(cmiiEnv, "f") {
+		if strings.Contains(cmiiEnv, "devf") {
 			op.CurrentNamespace = devFlight
-		} else if strings.Contains(cmiiEnv, "o") {
+		} else if strings.Contains(cmiiEnv, "devo") {
 			op.CurrentNamespace = devOperation
 		} else {
 			op.CurrentNamespace = dev
@@ -419,6 +419,15 @@ func (op *CmiiK8sOperator) DeploymentRestartByKill(cmiiEnv, appName string) bool
 	return true
 }

+func (op *CmiiK8sOperator) DeploymentNew(cmiiEnv, appName string, waitTimeOut int) bool {
+
+	op.changeOperatorEnv(cmiiEnv)
+	//op.CurrentClient.AppsV1().Deployments(op.CurrentNamespace).Apply()
+
+	return true
+
+}
+
 func (op *CmiiK8sOperator) DeploymentStatusCheck(cmiiEnv, appName string, waitTimeOut int) bool {

 	op.changeOperatorEnv(cmiiEnv)
--- a/agent-go/k8s_exec/K8sOperator_test.go
+++ b/agent-go/k8s_exec/K8sOperator_test.go
@@ -93,12 +93,17 @@ func TestCmiiK8sOperator_DeploymentRestart(t *testing.T) {
 }

 func TestCmiiK8sOperator_DeploymentRestartByKill(t *testing.T) {
-	cmiiEnv := "int"
-	appName := "cmii-suav-supervision"
+	cmiiEnv := "demo"
+	appName := "cmii-uav-platform"

 	kill := CmiiOperator.DeploymentRestartByKill(cmiiEnv, appName)
 	assert.Equal(t, kill, true, "deployment restart by kill failed !")

+	utils.SplitLinePrint()
+
+	check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
+	assert.Equal(t, check, true, "deployment run failed!")
+
 }

 func TestCmiiK8sOperator_DeploymentOneInterface(t *testing.T) {
--- a/agent-go/k8s_exec/log/cmii-update-log.txt
+++ b/agent-go/k8s_exec/log/cmii-update-log.txt
@@ -16,3 +16,10 @@
 2024-01-12-12-09-59 uavcloud-uavms           uavms-lowaltitude-platform              5.2.0-011201        5.2.0-011202
 2024-01-12-17-13-32 uavcloud-test            cmii-suav-supervision                   5.2.0-011001        5.2.0-011201
 2024-01-12-17-22-47 uavcloud-demo            cmii-uav-platform                       5.2.0-011102        5.2.0-011201
+2024-01-15-11-56-33 uavcloud-test            cmii-suav-supervision                   5.2.0-011201        5.2.0-011501
+2024-01-16-10-22-02 uavcloud-test            cmii-suav-supervision                   5.2.0-011501        5.2.0-011601
+2024-01-16-11-40-31 uavcloud-uavms           uavms-lowaltitude-platform              5.2.0-011202        5.2.0-snapshot
+2024-01-16-11-58-30 uavcloud-test            cmii-suav-supervision                   5.2.0-011601        5.2.0-011602
+2024-01-16-13-55-32 uavcloud-test            cmii-suav-supervision                   5.2.0-011602        5.2.0-011603
+2024-01-16-14-51-05 uavcloud-test            cmii-suav-supervision                   5.2.0-011603        5.2.0-011604
+2024-01-17-16-13-39 uavcloud-test            cmii-suav-supervision                   5.2.0-011604        5.2.0-0117
--- a/agent-go/message_pusher/client.go
+++ b/agent-go/message_pusher/client.go
@@ -0,0 +1,165 @@
+package message_pusher
+
+import (
+	"agent-go/logger"
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"regexp"
+	"strings"
+)
+
+var (
+	topicRegex = regexp.MustCompile(`^[-_A-Za-z0-9]{1,64}$`) // Same as in server/server.go
+	log        = logger.Log
+)
+
+const (
+	maxResponseBytes = 4096
+)
+
+type Client struct {
+	config *Config
+}
+
+// Message is a struct that represents a ntfy message
+type Message struct { // TODO combine with server.message
+	ID         string
+	Event      string
+	Time       int64
+	Topic      string
+	Message    string
+	Title      string
+	Priority   int
+	Tags       []string
+	Click      string
+	Icon       string
+	Attachment *Attachment
+
+	// Additional fields
+	TopicURL       string
+	SubscriptionID string
+	Raw            string
+}
+
+// Attachment represents a message attachment
+type Attachment struct {
+	Name    string `json:"name"`
+	Type    string `json:"type,omitempty"`
+	Size    int64  `json:"size,omitempty"`
+	Expires int64  `json:"expires,omitempty"`
+	URL     string `json:"url"`
+	Owner   string `json:"-"` // IP address of uploader, used for rate limiting
+}
+
+// New creates a new Client using a given Config
+func New(config *Config) *Client {
+	return &Client{
+		config: config,
+	}
+}
+
+func NewDefaultClient() *Client {
+	defaultConfig := NewDefaultConfig()
+	return New(defaultConfig)
+}
+
+func (c *Client) PublishDefault(message bytes.Buffer, options []PublishOption) (*Message, error) {
+	if c.config.DefaultTopic == "" {
+		return nil, errors.New("[PublishDefault] - topic empty")
+	}
+	// parse default
+	options = c.parseConfigToOption(options)
+
+	return c.PublishReader(c.config.DefaultTopic, bytes.NewReader(message.Bytes()), options)
+}
+
+// Publish sends a message to a specific topic, optionally using options.
+// See PublishReader for details.
+func (c *Client) Publish(topic, message string, options []PublishOption) (*Message, error) {
+	return c.PublishReader(topic, strings.NewReader(message), options)
+}
+
+// PublishReader sends a message to a specific topic, optionally using options.
+//
+// A topic can be either a full URL (e.g. https://myhost.lan/mytopic), a short URL which is then prepended https://
+// (e.g. myhost.lan -> https://myhost.lan), or a short name which is expanded using the default host in the
+// config (e.g. mytopic -> https://ntfy.sh/mytopic).
+//
+// To pass title, priority and tags, check out WithTitle, WithPriority, WithTagsList, WithDelay, WithNoCache,
+// WithNoFirebase, and the generic WithHeader.
+func (c *Client) PublishReader(topic string, body io.Reader, options []PublishOption) (*Message, error) {
+	topicURL, err := c.expandTopicURL(topic)
+	if err != nil {
+		return nil, err
+	}
+	req, err := http.NewRequest("POST", topicURL, body)
+	if err != nil {
+		return nil, err
+	}
+	for _, option := range options {
+		if err := option(req); err != nil {
+			return nil, err
+		}
+	}
+	log.DebugF("%s Publishing message with headers %s", topicURL, req.Header)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes))
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.New(strings.TrimSpace(string(b)))
+	}
+	m, err := toMessage(string(b), topicURL, "")
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *Client) expandTopicURL(topic string) (string, error) {
+	if strings.HasPrefix(topic, "http://") || strings.HasPrefix(topic, "https://") {
+		return topic, nil
+	} else if strings.Contains(topic, "/") {
+		return fmt.Sprintf("https://%s", topic), nil
+	}
+	if !topicRegex.MatchString(topic) {
+		return "", fmt.Errorf("invalid topic name: %s", topic)
+	}
+	return fmt.Sprintf("%s/%s", c.config.DefaultHost, topic), nil
+}
+
+func (c *Client) parseConfigToOption(options []PublishOption) []PublishOption {
+	config := c.config
+
+	if config.DefaultToken != "" {
+		options = append(options, WithBearerAuth(config.DefaultToken))
+	} else if config.DefaultUser != "" {
+		if *config.DefaultPassword != "" {
+			options = append(options, WithBasicAuth(config.DefaultUser, *config.DefaultPassword))
+		} else {
+			log.ErrorF("[parseConfigToOption] - default password is empty!")
+		}
+	}
+
+	return options
+}
+
+func toMessage(s, topicURL, subscriptionID string) (*Message, error) {
+	var m *Message
+	if err := json.NewDecoder(strings.NewReader(s)).Decode(&m); err != nil {
+		return nil, err
+	}
+	m.TopicURL = topicURL
+	m.SubscriptionID = subscriptionID
+	m.Raw = s
+	return m, nil
+}
--- a/agent-go/message_pusher/client_test.go
+++ b/agent-go/message_pusher/client_test.go
@@ -0,0 +1,34 @@
+package message_pusher
+
+import (
+	"agent-go/utils"
+	"testing"
+)
+
+func TestClient_Publish(t *testing.T) {
+
+	client := NewDefaultClient()
+
+	optionList := []PublishOption{
+		WithTitle("测试内容"),
+		WithPriority("5"),
+		WithMarkdown(),
+	}
+
+	deployPush := DeployPush{
+		Namespace:    "uavcloud-dev",
+		AppName:      "cmii-uav-platform",
+		Replicas:     "1",
+		DeployStatus: false,
+	}
+
+	deployPush.ParseCmiiDeployTemplate()
+
+	message, err := client.PublishDefault(deployPush.ParseCmiiDeployTemplate(), optionList)
+	if err != nil {
+		return
+	}
+
+	utils.BeautifulPrint(message)
+
+}
--- a/agent-go/message_pusher/config.go
+++ b/agent-go/message_pusher/config.go
@@ -0,0 +1,62 @@
+package message_pusher
+
+import (
+	"gopkg.in/yaml.v2"
+	"os"
+)
+
+const (
+	// DefaultBaseURL is the base URL used to expand short topic names
+	DefaultBaseURL = "https://push.107421.xyz"
+
+	DefaultBaseToken = "tk_zvdb67fwj1hrjivkq3ga9z7u63av5"
+
+	DefaultTopic = "cmii"
+)
+
+// Config is the config struct for a Client
+type Config struct {
+	DefaultHost     string      `yaml:"default-host"`
+	DefaultUser     string      `yaml:"default-user"`
+	DefaultPassword *string     `yaml:"default-password"`
+	DefaultToken    string      `yaml:"default-token"`
+	DefaultCommand  string      `yaml:"default-command"`
+	DefaultTopic    string      `yaml:"default-topic"`
+	Subscribe       []Subscribe `yaml:"subscribe"`
+}
+
+// Subscribe is the struct for a Subscription within Config
+type Subscribe struct {
+	Topic    string            `yaml:"topic"`
+	User     *string           `yaml:"user"`
+	Password *string           `yaml:"password"`
+	Token    *string           `yaml:"token"`
+	Command  string            `yaml:"command"`
+	If       map[string]string `yaml:"if"`
+}
+
+// NewDefaultConfig creates a new Config struct for a Client
+func NewDefaultConfig() *Config {
+	return &Config{
+		DefaultHost:     DefaultBaseURL,
+		DefaultUser:     "",
+		DefaultPassword: nil,
+		DefaultToken:    DefaultBaseToken,
+		DefaultTopic:    DefaultTopic,
+		DefaultCommand:  "",
+		Subscribe:       nil,
+	}
+}
+
+// LoadConfig loads the Client config from a yaml file
+func LoadConfig(filename string) (*Config, error) {
+	b, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	c := NewDefaultConfig()
+	if err := yaml.Unmarshal(b, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
--- a/agent-go/message_pusher/options.go
+++ b/agent-go/message_pusher/options.go
@@ -0,0 +1,204 @@
+package message_pusher
+
+import (
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// RequestOption is a generic request option that can be added to Client calls
+type RequestOption = func(r *http.Request) error
+
+// PublishOption is an option that can be passed to the Client.Publish call
+type PublishOption = RequestOption
+
+// SubscribeOption is an option that can be passed to a Client.Subscribe or Client.Poll call
+type SubscribeOption = RequestOption
+
+// WithMessage sets the notification message. This is an alternative way to passing the message body.
+func WithMessage(message string) PublishOption {
+	return WithHeader("X-Message", message)
+}
+
+// WithTitle adds a title to a message
+func WithTitle(title string) PublishOption {
+	return WithHeader("X-Title", title)
+}
+
+// WithPriority adds a priority to a message. The priority can be either a number (1=min, 5=max),
+// or the corresponding names (see util.ParsePriority).
+func WithPriority(priority string) PublishOption {
+	return WithHeader("X-Priority", priority)
+}
+
+// WithTagsList adds a list of tags to a message. The tags parameter must be a comma-separated list
+// of tags. To use a slice, use WithTags instead
+func WithTagsList(tags string) PublishOption {
+	return WithHeader("X-Tags", tags)
+}
+
+// WithTags adds a list of a tags to a message
+func WithTags(tags []string) PublishOption {
+	return WithTagsList(strings.Join(tags, ","))
+}
+
+// WithDelay instructs the server to send the message at a later date. The delay parameter can be a
+// Unix timestamp, a duration string or a natural langage string. See https://ntfy.sh/docs/publish/#scheduled-delivery
+// for details.
+func WithDelay(delay string) PublishOption {
+	return WithHeader("X-Delay", delay)
+}
+
+// WithClick makes the notification action open the given URL as opposed to entering the detail view
+func WithClick(url string) PublishOption {
+	return WithHeader("X-Click", url)
+}
+
+// WithIcon makes the notification use the given URL as its icon
+func WithIcon(icon string) PublishOption {
+	return WithHeader("X-Icon", icon)
+}
+
+// WithActions adds custom user actions to the notification. The value can be either a JSON array or the
+// simple format definition. See https://ntfy.sh/docs/publish/#action-buttons for details.
+func WithActions(value string) PublishOption {
+	return WithHeader("X-Actions", value)
+}
+
+// WithAttach sets a URL that will be used by the client to download an attachment
+func WithAttach(attach string) PublishOption {
+	return WithHeader("X-Attach", attach)
+}
+
+// WithMarkdown instructs the server to interpret the message body as Markdown
+func WithMarkdown() PublishOption {
+	return WithHeader("X-Markdown", "yes")
+}
+
+// WithFilename sets a filename for the attachment, and/or forces the HTTP body to interpreted as an attachment
+func WithFilename(filename string) PublishOption {
+	return WithHeader("X-Filename", filename)
+}
+
+// WithEmail instructs the server to also send the message to the given e-mail address
+func WithEmail(email string) PublishOption {
+	return WithHeader("X-Email", email)
+}
+
+// WithBasicAuth adds the Authorization header for basic auth to the request
+func WithBasicAuth(user, pass string) PublishOption {
+	return WithHeader("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", user, pass)))))
+}
+
+// WithBearerAuth adds the Authorization header for Bearer auth to the request
+func WithBearerAuth(token string) PublishOption {
+	return WithHeader("Authorization", fmt.Sprintf("Bearer %s", token))
+}
+
+// WithEmptyAuth clears the Authorization header
+func WithEmptyAuth() PublishOption {
+	return RemoveHeader("Authorization")
+}
+
+// WithNoCache instructs the server not to cache the message server-side
+func WithNoCache() PublishOption {
+	return WithHeader("X-Cache", "no")
+}
+
+// WithNoFirebase instructs the server not to forward the message to Firebase
+func WithNoFirebase() PublishOption {
+	return WithHeader("X-Firebase", "no")
+}
+
+// WithSince limits the number of messages returned from the server. The parameter since can be a Unix
+// timestamp (see WithSinceUnixTime), a duration (WithSinceDuration) the word "all" (see WithSinceAll).
+func WithSince(since string) SubscribeOption {
+	return WithQueryParam("since", since)
+}
+
+// WithSinceAll instructs the server to return all messages for the given topic from the server
+func WithSinceAll() SubscribeOption {
+	return WithSince("all")
+}
+
+// WithSinceDuration instructs the server to return all messages since the given duration ago
+func WithSinceDuration(since time.Duration) SubscribeOption {
+	return WithSinceUnixTime(time.Now().Add(-1 * since).Unix())
+}
+
+// WithSinceUnixTime instructs the server to return only messages newer or equal to the given timestamp
+func WithSinceUnixTime(since int64) SubscribeOption {
+	return WithSince(fmt.Sprintf("%d", since))
+}
+
+// WithPoll instructs the server to close the connection after messages have been returned. Don't use this option
+// directly. Use Client.Poll instead.
+func WithPoll() SubscribeOption {
+	return WithQueryParam("poll", "1")
+}
+
+// WithScheduled instructs the server to also return messages that have not been sent yet, i.e. delayed/scheduled
+// messages (see WithDelay). The messages will have a future date.
+func WithScheduled() SubscribeOption {
+	return WithQueryParam("scheduled", "1")
+}
+
+// WithFilter is a generic subscribe option meant to be used to filter for certain messages only
+func WithFilter(param, value string) SubscribeOption {
+	return WithQueryParam(param, value)
+}
+
+// WithMessageFilter instructs the server to only return messages that match the exact message
+func WithMessageFilter(message string) SubscribeOption {
+	return WithQueryParam("message", message)
+}
+
+// WithTitleFilter instructs the server to only return messages with a title that match the exact string
+func WithTitleFilter(title string) SubscribeOption {
+	return WithQueryParam("title", title)
+}
+
+// WithPriorityFilter instructs the server to only return messages with the matching priority. Not that messages
+// without priority also implicitly match priority 3.
+func WithPriorityFilter(priority int) SubscribeOption {
+	return WithQueryParam("priority", fmt.Sprintf("%d", priority))
+}
+
+// WithTagsFilter instructs the server to only return messages that contain all of the given tags
+func WithTagsFilter(tags []string) SubscribeOption {
+	return WithQueryParam("tags", strings.Join(tags, ","))
+}
+
+// WithHeader is a generic option to add headers to a request
+func WithHeader(header, value string) RequestOption {
+	return func(r *http.Request) error {
+		if value != "" {
+			r.Header.Set(header, value)
+		}
+		return nil
+	}
+}
+
+// WithQueryParam is a generic option to add query parameters to a request
+func WithQueryParam(param, value string) RequestOption {
+	return func(r *http.Request) error {
+		if value != "" {
+			q := r.URL.Query()
+			q.Add(param, value)
+			r.URL.RawQuery = q.Encode()
+		}
+		return nil
+	}
+}
+
+// RemoveHeader is a generic option to remove a header from a request
+func RemoveHeader(header string) RequestOption {
+	return func(r *http.Request) error {
+		if header != "" {
+			delete(r.Header, header)
+		}
+		return nil
+	}
+}
--- a/agent-go/message_pusher/publish.go
+++ b/agent-go/message_pusher/publish.go
@@ -0,0 +1 @@
+package message_pusher
--- a/agent-go/message_pusher/push_template.go
+++ b/agent-go/message_pusher/push_template.go
@@ -0,0 +1,41 @@
+package message_pusher
+
+import (
+	"bytes"
+	"text/template"
+)
+
+const cmiiDeployTemplate = `
+	{{if .DeployStatus}}
+	部署状态: 成功😍
+	{{- else }}
+	部署状态: 失败👿👿👿
+	{{- end}}
+命名空间: {{.Namespace}} 
+应用名称: {{.AppName}} 
+副本数量: {{.Replicas}}
+	`
+
+type DeployPush struct {
+	Namespace    string
+	AppName      string
+	Replicas     string
+	DeployStatus bool
+}
+
+func (d DeployPush) ParseCmiiDeployTemplate() bytes.Buffer {
+	// 解析模板
+	tmpl, err := template.New("cmiiDeployTemplate").Parse(cmiiDeployTemplate)
+	if err != nil {
+		panic(err)
+	}
+
+	// 应用数据并打印结果
+	var result bytes.Buffer
+	err = tmpl.Execute(&result, d)
+	if err != nil {
+		panic(err)
+	}
+
+	return result
+}
--- a/agent-go/message_pusher/push_template_test.go
+++ b/agent-go/message_pusher/push_template_test.go
@@ -0,0 +1,14 @@
+package message_pusher
+
+import "testing"
+
+func TestDeployPush_ParseCmiiDeployTemplate(t *testing.T) {
+	deployPush := DeployPush{
+		Namespace:    "casc",
+		AppName:      "sdasdas",
+		Replicas:     "dasdasd",
+		DeployStatus: false,
+	}
+
+	deployPush.ParseCmiiDeployTemplate()
+}
--- a/agent-go/tmp/test.yaml
+++ b/agent-go/tmp/test.yaml
@@ -0,0 +1,114 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cmii-uav-gateway
+  namespace: uavcloud-dev
+  labels:
+    cmii.type: backend
+    cmii.app: cmii-uav-gateway
+    octopus/control: backend-app-1.0.0
+    app.kubernetes.io/managed-by: octopus/control
+    app.kubernetes.io/app-version: 5.2.0
+spec:
+  replicas: 2
+  strategy:
+    rollingUpdate:
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      cmii.type: backend
+      cmii.app: cmii-uav-gateway
+  template:
+    metadata:
+      labels:
+        cmii.type: backend
+        cmii.app: cmii-uav-gateway
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: uavcloud.env
+                    operator: In
+                    values:
+                      - demo
+      imagePullSecrets:
+        - name: harborsecret
+      containers:
+        - name: cmii-uav-gateway
+          image: \"harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:5.2.0-123\"
+          imagePullPolicy: Always
+          env:
+            - name: K8S_NAMESPACE
+              value: \"uavcloud-dev\"
+            - name: APPLICATION_NAME
+              value: \"cmii-uav-gateway\"
+            - name: CUST_JAVA_OPTS
+              value: \"-Xms500m -Xmx1500m -Dlog4j2.formatMsgNoLookups=true\"
+            - name: NACOS_REGISTRY
+              value: \"helm-nacos:8848\"
+            - name: NACOS_DISCOVERY_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: NACOS_DISCOVERY_PORT
+              value: \"8080\"
+            - name: BIZ_CONFIG_GROUP
+            - name: SYS_CONFIG_GROUP
+            - name: IMAGE_VERSION
+              value: 5.2.0
+            - name: NACOS_USERNAME
+              value: \"developer\"
+            - name: NACOS_PASSWORD
+              value: \"Deve@9128201\"
+          ports:
+            - name: pod-port
+              containerPort: 8080
+              protocol: TCP
+          resources:
+            limits:
+              memory: 2Gi
+              cpu: 2c
+            requests:
+              memory: 1Gi
+              cpu: 200m
+          livenessProbe:
+            httpGet:
+              path: /cmii/ping
+              port: pod-port
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            periodSeconds: 20
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /cmii/ping
+              port: pod-port
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 5
+            periodSeconds: 20
+            successThreshold: 1
+            failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /cmii/ping
+              port: pod-port
+              scheme: HTTP
+            initialDelaySeconds: 60
+            timeoutSeconds: 3
+            periodSeconds: 20
+            successThreshold: 1
+            failureThreshold: 5
+          volumeMounts:
+            - name: glusterfs-backend-log-volume
+              mountPath: /cmii/logs
+              readOnly: false
+              subPath: uavcloud-dev/cmii-uav-gateway
+      volumes:
+        - name: glusterfs-backend-log-volume
+          persistentVolumeClaim:
+            claimName: glusterfs-backend-log-pvc
--- a/agent-go/utils/PrintUtils.go
+++ b/agent-go/utils/PrintUtils.go
@@ -20,3 +20,9 @@ func BeautifulPrint(object interface{}) {
 	fmt.Println()

 }
+
+func SplitLinePrint() {
+	fmt.Println()
+	fmt.Println()
+	fmt.Println()
+}