diff --git a/agent-go/k8s_exec/CmiiK8sOperator.go b/agent-go/k8s_exec/CmiiK8sOperator.go index 8ddfd54..ac3a2de 100644 --- a/agent-go/k8s_exec/CmiiK8sOperator.go +++ b/agent-go/k8s_exec/CmiiK8sOperator.go @@ -10,45 +10,46 @@ import ( var CmiiOperator = CmiiK8sOperator{} var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\agent-go\\k8s_exec\\log\\cmii-update-log.txt" -// FindDeploymentRestartCountGreaterThanN 重启次数大于N的所有Deployment -func FindDeploymentRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface { +// FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment +func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface { //podInterface := CmiiPodInterface{} // get all pods - podAll := CmiiOperator.PodAll(cmiiEnv) + podAll := CmiiOperator.PodAllInterface(cmiiEnv) // restart map restartMap := make(map[string]int32, len(podAll)) // restart count for _, pod := range podAll { - for _, containerStatus := range pod.Status.ContainerStatuses { - if containerStatus.RestartCount > restartCount { - restart, exists := restartMap[containerStatus.Name] - if exists { - restartMap[containerStatus.Name] = utils.MaxInt32(containerStatus.RestartCount, restart) - } else { - restartMap[containerStatus.Name] = containerStatus.RestartCount - } + if pod.RestartCount > restartCount { + restart, exists := restartMap[pod.ContainerName] + if exists { + restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart) + } else { + restartMap[pod.ContainerName] = pod.RestartCount } } + + // unhealthy + if !pod.PodStatus { + restartMap[pod.ContainerName] = pod.RestartCount + } } result := make([]CmiiDeploymentInterface, len(restartMap)) - cmiiDeploymentInterface := CmiiDeploymentInterface{} index := 0 - - log.DebugF("[FindDeploymentRestartCountGreaterThanN] - restart map is => %v", restartMap) + log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap) // find deployment convert to interface for key, value := range restartMap { - deployment := CmiiOperator.DeploymentExist(cmiiEnv, key) + // container Name must equals deployment name + deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deployment != nil { // deployment exists - log.DebugF("[FindDeploymentRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value) + log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value) - convert := cmiiDeploymentInterface.Convert(*deployment) - result[index] = convert + result[index] = *deployment index++ } } @@ -60,6 +61,7 @@ func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (depl // get all deployments cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) + cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces) // filter for _, deploymentInterface := range cmiiDeploymentInterfaces { @@ -72,6 +74,25 @@ func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (depl return deploymentList } +func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) { + + // all unhealthy pods + allInterface := CmiiOperator.PodAllInterface(cmiiEnv) + + // find the deployments + for _, podInterface := range allInterface { + if !podInterface.PodStatus { + // unhealthy pod + deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName) + if deploymentInterface != nil { + deploymentList = append(deploymentList, *deploymentInterface) + } + } + } + + return deploymentList +} + func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool { result := true @@ -96,11 +117,35 @@ func RestartCmiiBackendDeployment(cmiiEnv string) { cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { - if strings.Contains(deploymentInterface.Name, "platform") { - + _, ok := CmiiBackendAppMap[deploymentInterface.Name] + if ok { + if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { + log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) + } else { + log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) + } } } + log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) +} + +func RestartCmiiFrontendDeployment(cmiiEnv string) { + + cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) + + for _, deploymentInterface := range cmiiDeploymentInterfaces { + _, ok := CmiiFrontendAppMap[deploymentInterface.Name] + if ok { + if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { + log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) + } else { + log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) + } + } + } + + log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) } func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool { @@ -233,6 +278,7 @@ func BackupAllDeploymentFromEnv(cmiiEnv string) bool { func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[string]int32) { allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) + allInterface = FilterAllCmiiAppSoft(allInterface) backendMap = make(map[string]int32, len(allInterface)) frontendMap = make(map[string]int32, len(allInterface)) @@ -247,3 +293,58 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[s return backendMap, frontendMap } + +func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) { + + for _, c := range source { + _, ok := CmiiBackendAppMap[c.ContainerName] + if !ok { + _, ok = CmiiFrontendAppMap[c.ContainerName] + if !ok { + log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod ！", c.ContainerName) + continue + } + } + + result = append(result, c) + } + + return result +} + +func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) { + + for _, c := range source { + if strings.Contains(c.ContainerName, "redis") { + continue + } + if strings.Contains(c.ContainerName, "emqxs") { + continue + } + if strings.Contains(c.ContainerName, "rabbitmq") { + continue + } + if strings.Contains(c.ContainerName, "nacos") { + continue + } + if strings.Contains(c.ContainerName, "oss") { + continue + } + if strings.Contains(c.ContainerName, "minio") { + continue + } + if strings.HasPrefix(c.ContainerName, "nfs") { + continue + } + if strings.HasPrefix(c.ContainerName, "operator") { + continue + } + if strings.HasPrefix(c.ContainerName, "proxy") { + continue + } + + result = append(result, c) + } + + return result +} diff --git a/agent-go/k8s_exec/CmiiK8sOperator_test.go b/agent-go/k8s_exec/CmiiK8sOperator_test.go index 78141b9..06d5614 100644 --- a/agent-go/k8s_exec/CmiiK8sOperator_test.go +++ b/agent-go/k8s_exec/CmiiK8sOperator_test.go @@ -6,9 +6,10 @@ import ( "testing" ) -func TestFindDeploymentRestartCountGreaterThanN(t *testing.T) { +func TestFindAppNotHealthyOrRestartCountGreaterThanN(t *testing.T) { - deploymentRestartCountGreaterThanN := FindDeploymentRestartCountGreaterThanN("devflight", 10) + deploymentRestartCountGreaterThanN := FindAppNotHealthyOrRestartCountGreaterThanN("devflight", 10) + deploymentRestartCountGreaterThanN = FilterAllCmiiAppSoft(deploymentRestartCountGreaterThanN) for _, deploymentInterface := range deploymentRestartCountGreaterThanN { println() @@ -57,3 +58,17 @@ func TestRollBackCmiiDeploymentFromUpdateLog(t *testing.T) { assert.Equal(t, updateLog, true, "roll back from update log failed !") } + +func TestRestartCmiiBackendDeployment(t *testing.T) { + + RestartCmiiBackendDeployment("devflight") +} + +func TestFindDeploymentNotHealthy(t *testing.T) { + notHealthy := FindDeploymentNotHealthy("devflight") + + notHealthy = FilterAllCmiiAppSoft(notHealthy) + for _, deploymentInterface := range notHealthy { + utils.BeautifulPrint(deploymentInterface) + } +} diff --git a/agent-go/k8s_exec/CmiiStatus.go b/agent-go/k8s_exec/CmiiStatus.go index 6046d2d..c5b0f81 100644 --- a/agent-go/k8s_exec/CmiiStatus.go +++ b/agent-go/k8s_exec/CmiiStatus.go @@ -1,6 +1,7 @@ package k8s_exec import ( + "agent-go/utils" v1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "strings" @@ -18,11 +19,14 @@ type CmiiPodInterface struct { HostIP string NodeName string RestartCountMap map[string]int32 + RestartCount int32 ContainerName string Image string ImageTag string GitBranch string GitCommit string + PodStatus bool + PodPhase corev1.PodPhase } type CmiiDeploymentInterface struct { @@ -104,6 +108,7 @@ func (pod CmiiPodInterface) Convert(podDetail corev1.Pod) CmiiPodInterface { for _, containerStatus := range containerStatuses { containerStatusMap[containerStatus.Name] = containerStatus.RestartCount + pod.RestartCount = utils.MaxInt32(pod.RestartCount, containerStatus.RestartCount) } pod.Name = podDetail.Name @@ -113,9 +118,22 @@ func (pod CmiiPodInterface) Convert(podDetail corev1.Pod) CmiiPodInterface { pod.PodIP = podDetail.Status.PodIP pod.HostIP = podDetail.Status.HostIP pod.NodeName = podDetail.Spec.NodeName - //if len(podDetail.ObjectMeta.OwnerReferences) > 0 { - // pod.ReplicaSetName = podDetail.ObjectMeta.OwnerReferences[0].Name - //} + pod.PodPhase = podDetail.Status.Phase + + switch podDetail.Status.Phase { + case corev1.PodFailed: + pod.PodStatus = false + break + case corev1.PodPending: + pod.PodStatus = false + break + case corev1.PodReasonUnschedulable: + pod.PodStatus = false + break + default: + pod.PodStatus = true + break + } return pod } diff --git a/agent-go/k8s_exec/K8sOperator.go b/agent-go/k8s_exec/K8sOperator.go index 21854e3..bdc4c4d 100644 --- a/agent-go/k8s_exec/K8sOperator.go +++ b/agent-go/k8s_exec/K8sOperator.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "strings" "sync" + "time" ) var log = logger.Log @@ -241,19 +242,6 @@ func (op *CmiiK8sOperator) DeploymentAllInterface(cmiiEnv string) []CmiiDeployme index := 0 for deployment := range ccc { - // 过滤所有非CMII的内容 - if strings.Contains(deployment.Name, "proxy") { - continue - } - if strings.Contains(deployment.Name, "minio") { - continue - } - if strings.HasPrefix(deployment.Name, "helm-live-rtsp") { - continue - } - if strings.Contains(deployment.Name, "nfs") { - continue - } results[index] = deployment index++ } @@ -323,7 +311,7 @@ func (op *CmiiK8sOperator) DeploymentScale(cmiiEnv, appFizz string, scaleCount i Namespace: deployment.Namespace, }, Spec: autoscalingv1.ScaleSpec{ - Replicas: int32(scaleCount), + Replicas: scaleCount, }, } @@ -379,6 +367,30 @@ func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string) return true } +func (op *CmiiK8sOperator) DeploymentRestart(cmiiEnv, appName string) bool { + op.changeOperatorEnv(cmiiEnv) + + result := true + + deployment := op.DeploymentOneInterface(cmiiEnv, appName) + if deployment == nil { + log.ErrorF("[DeploymentRestart] - [%s] [%s] not exists !", cmiiEnv, appName) + return false + } + + result = op.DeploymentScale(deployment.Namespace, deployment.Name, 0) + if !result { + return result + } + time.Sleep(time.Millisecond * 200) + result = op.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas) + if !result { + return result + } + + return result +} + func (op *CmiiK8sOperator) ReplicaSet(cmiiEnv, replicaSetName string) *v1.ReplicaSet { op.changeOperatorEnv(cmiiEnv) @@ -451,7 +463,69 @@ func (op *CmiiK8sOperator) PodAll(cmiiEnv string) []corev1.Pod { index++ } - return results + return results[:index] +} + +func (op *CmiiK8sOperator) PodAllInterface(cmiiEnv string) []CmiiPodInterface { + op.changeOperatorEnv(cmiiEnv) + client := op.CurrentClient + + podList, err := client.CoreV1().Pods(op.CurrentNamespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + log.ErrorF("[PodAll] - list c in [%s] [%s] error => %s", cmiiEnv, op.CurrentNamespace, err.Error()) + return nil + } + pods := podList.Items + + length := len(pods) + results := make([]CmiiPodInterface, length) + ccc := make(chan CmiiPodInterface, length) + var wg sync.WaitGroup + podInterface := CmiiPodInterface{} + + worker := workerThread + if length <= worker { + for i, pod := range pods { + objectMeta := pod.ObjectMeta + objectMeta.SetAnnotations(nil) + objectMeta.SetManagedFields(nil) + pod.ObjectMeta = objectMeta + + results[i] = podInterface.Convert(pod) + } + return results + } + + pinch := length / worker + wg.Add(worker) + + for splice := 0; splice < worker; splice++ { + go func(podList []corev1.Pod) { + defer wg.Done() + for _, pod := range podList { + objectMeta := pod.ObjectMeta + objectMeta.SetAnnotations(nil) + objectMeta.SetManagedFields(nil) + pod.ObjectMeta = objectMeta + + ccc <- podInterface.Convert(pod) + } + + }(pods[splice*pinch : utils.MinInt((splice+1)*pinch, length)]) + } + + go func() { + wg.Wait() + close(ccc) + }() + + index := 0 + for c := range ccc { + results[index] = c + index++ + } + + return results[:index] } func (op *CmiiK8sOperator) PodFizz(cmiiEnv, appFizz string) (fizzPod []corev1.Pod) { diff --git a/agent-go/k8s_exec/K8sOperator_test.go b/agent-go/k8s_exec/K8sOperator_test.go index e752cb6..d4873ef 100644 --- a/agent-go/k8s_exec/K8sOperator_test.go +++ b/agent-go/k8s_exec/K8sOperator_test.go @@ -52,7 +52,7 @@ func TestCmiiK8sOperator_DeploymentAllInterface(t *testing.T) { func TestCmiiK8sOperator_DeploymentFizz(t *testing.T) { start := time.Now() - deploymentFizz := CmiiOperator.DeploymentFizz("demo", "cmii-uav-platform") + deploymentFizz := CmiiOperator.DeploymentFizz("demo", "helm-proxysql") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时： %d ms\n", elapsed) @@ -73,7 +73,7 @@ func TestCmiiK8sOperator_DeploymentFizz(t *testing.T) { func TestCmiiK8sOperator_DeploymentScale(t *testing.T) { start := time.Now() - CmiiOperator.DeploymentScale("devflight", "cmii-uav-gis-server", 1) + CmiiOperator.DeploymentScale("devflight", "cmii-uav-depotautoreturn", 0) elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时： %d ms\n", elapsed) @@ -112,7 +112,7 @@ func TestCmiiK8sOperator_PodAll(t *testing.T) { func TestCmiiK8sOperator_PodFizz(t *testing.T) { start := time.Now() - podList := CmiiOperator.PodFizz("devflight", "cmii-uav-depotautoreturn") + podList := CmiiOperator.PodFizz("devflight", "cmii-uav-data-post-process") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时： %d ms\n", elapsed) t.Logf("pod list lenght is => %d", len(podList)) diff --git a/agent-go/tmp/test.json b/agent-go/tmp/test.json index 4a74790..e69de29 100644 --- a/agent-go/tmp/test.json +++ b/agent-go/tmp/test.json @@ -1,302 +0,0 @@ -{ - "metadata": { - "name": "cmii-uav-gateway-64ddc6c4c7-77k2w", - "generateName": "cmii-uav-gateway-64ddc6c4c7-", - "namespace": "uavcloud-devflight", - "selfLink": "/api/v1/namespaces/uavcloud-devflight/pods/cmii-uav-gateway-64ddc6c4c7-77k2w", - "uid": "02e1277f-b632-4e7a-b204-dad30885de48", - "resourceVersion": "468813592", - "creationTimestamp": "2023-12-25T09:04:01Z", - "labels": { - "app.kubernetes.io/instance": "cmii-uav-gateway", - "app.kubernetes.io/managed-by": "Helm", - "cmii.app": "cmii-uav-gateway", - "cmii.modules": "cmlc", - "cmii.type": "backend", - "cmii/managed-by": "jenkins-pipeline-standalone-job", - "pod-template-hash": "64ddc6c4c7" - }, - "ownerReferences": [ - { - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "cmii-uav-gateway-64ddc6c4c7", - "uid": "1d76ec6d-4c9c-435b-8aca-5bb0755f9ab1", - "controller": true, - "blockOwnerDeletion": true - } - ] - }, - "spec": { - "volumes": [ - { - "name": "glusterfs-backend-log-volume", - "persistentVolumeClaim": { - "claimName": "glusterfs-backend-log-pvc" - } - }, - { - "name": "elk-json-log", - "hostPath": { - "path": "/var/log/", - "type": "" - } - }, - { - "name": "default-token-hzddz", - "secret": { - "secretName": "default-token-hzddz", - "defaultMode": 420 - } - } - ], - "containers": [ - { - "name": "cmii-uav-gateway", - "image": "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta", - "ports": [ - { - "name": "pod-port", - "containerPort": 8080, - "protocol": "TCP" - } - ], - "env": [ - { - "name": "GIT_BRANCH", - "value": "staging" - }, - { - "name": "GIT_COMMIT", - "value": "bdcdf85720bd1da671bdb55e80f2192968d98633" - }, - { - "name": "K8S_NAMESPACE", - "value": "uavcloud-devflight" - }, - { - "name": "APPLICATION_NAME", - "value": "cmii-uav-gateway" - }, - { - "name": "CUST_JAVA_OPTS", - "value": "-Xms1000m -Xmx2048m -Dreactor.netty.ioWorkerCount=256 -Dreactor.netty.ioSelectCount=1" - }, - { - "name": "NACOS_REGISTRY", - "value": "helm-nacos:8848" - }, - { - "name": "NACOS_USERNAME", - "value": "developer" - }, - { - "name": "NACOS_PASSWORD", - "value": "Deve@9128" - }, - { - "name": "NACOS_DISCOVERY_IP", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "status.podIP" - } - } - }, - { - "name": "NACOS_DISCOVERY_PORT", - "value": "8080" - }, - { - "name": "IMAGE_VERSION", - "value": "4.1.6-beta" - }, - { - "name": "BIZ_CONFIG_GROUP" - }, - { - "name": "SYS_CONFIG_GROUP" - } - ], - "resources": { - "limits": { - "cpu": "2", - "memory": "2Gi" - }, - "requests": { - "cpu": "100m", - "memory": "200Mi" - } - }, - "volumeMounts": [ - { - "name": "glusterfs-backend-log-volume", - "mountPath": "/cmii/logs", - "subPath": "uavcloud-devflight/cmii-uav-gateway" - }, - { - "name": "elk-json-log", - "mountPath": "/cmii/jsonlogs", - "subPath": "json_pv_logfile" - }, - { - "name": "default-token-hzddz", - "readOnly": true, - "mountPath": "/var/run/secrets/kubernetes.io/serviceaccount" - } - ], - "livenessProbe": { - "httpGet": { - "path": "/cmii/ping", - "port": "pod-port", - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "timeoutSeconds": 2, - "periodSeconds": 10, - "successThreshold": 1, - "failureThreshold": 3 - }, - "readinessProbe": { - "httpGet": { - "path": "/cmii/ping", - "port": "pod-port", - "scheme": "HTTP" - }, - "initialDelaySeconds": 5, - "timeoutSeconds": 2, - "periodSeconds": 10, - "successThreshold": 1, - "failureThreshold": 3 - }, - "startupProbe": { - "httpGet": { - "path": "/cmii/ping", - "port": "pod-port", - "scheme": "HTTP" - }, - "initialDelaySeconds": 120, - "timeoutSeconds": 2, - "periodSeconds": 10, - "successThreshold": 1, - "failureThreshold": 10 - }, - "terminationMessagePath": "/dev/termination-log", - "terminationMessagePolicy": "File", - "imagePullPolicy": "Always" - } - ], - "restartPolicy": "Always", - "terminationGracePeriodSeconds": 30, - "dnsPolicy": "ClusterFirst", - "serviceAccountName": "default", - "serviceAccount": "default", - "nodeName": "dev-07.ecs.io", - "securityContext": {}, - "imagePullSecrets": [ - { - "name": "harborsecret" - } - ], - "affinity": { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - { - "key": "uavcloud.affinity", - "operator": "In", - "values": [ - "common" - ] - } - ] - } - ] - } - } - }, - "schedulerName": "default-scheduler", - "tolerations": [ - { - "key": "node.kubernetes.io/not-ready", - "operator": "Exists", - "effect": "NoExecute", - "tolerationSeconds": 300 - }, - { - "key": "node.kubernetes.io/unreachable", - "operator": "Exists", - "effect": "NoExecute", - "tolerationSeconds": 300 - } - ], - "priority": 0, - "enableServiceLinks": true - }, - "status": { - "phase": "Running", - "conditions": [ - { - "type": "Initialized", - "status": "True", - "lastProbeTime": null, - "lastTransitionTime": "2023-12-25T09:04:01Z" - }, - { - "type": "Ready", - "status": "True", - "lastProbeTime": null, - "lastTransitionTime": "2023-12-26T07:22:36Z" - }, - { - "type": "ContainersReady", - "status": "True", - "lastProbeTime": null, - "lastTransitionTime": "2023-12-26T07:22:36Z" - }, - { - "type": "PodScheduled", - "status": "True", - "lastProbeTime": null, - "lastTransitionTime": "2023-12-25T09:04:01Z" - } - ], - "hostIP": "192.168.11.161", - "podIP": "10.244.165.25", - "podIPs": [ - { - "ip": "10.244.165.25" - } - ], - "startTime": "2023-12-25T09:04:01Z", - "containerStatuses": [ - { - "name": "cmii-uav-gateway", - "state": { - "running": { - "startedAt": "2023-12-26T07:20:22Z" - } - }, - "lastState": { - "terminated": { - "exitCode": 1, - "reason": "Error", - "startedAt": "2023-12-26T07:14:52Z", - "finishedAt": "2023-12-26T07:15:18Z", - "containerID": "docker://da24e4587f76139c30f4730b18210c11c07f1d4f5afda93b3d70d2eef6aeeb53" - } - }, - "ready": true, - "restartCount": 9, - "image": "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta", - "imageID": "docker-pullable://harbor.cdcyy.com.cn/cmii/cmii-uav-gateway@sha256:138ae1ed2596746b429f158795791eb677f26c2aa523784f42a13d7256652310", - "containerID": "docker://13a6bfe5218ba9b9edeaf5c1378d73ecb06fc90cb648e0c158baa9f528c1e391", - "started": true - } - ], - "qosClass": "Burstable" - } -} -