package k8s_exec import ( "agent-go/executor" "agent-go/utils" "strings" "time" ) var CmiiOperator = CmiiK8sOperator{} var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\agent-go\\k8s_exec\\log\\cmii-update-log.txt" // FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface { //podInterface := CmiiPodInterface{} // get all pods podAll := CmiiOperator.PodAllInterface(cmiiEnv) // restart map restartMap := make(map[string]int32, len(podAll)) // restart count for _, pod := range podAll { if pod.RestartCount > restartCount { restart, exists := restartMap[pod.ContainerName] if exists { restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart) } else { restartMap[pod.ContainerName] = pod.RestartCount } } // unhealthy if !pod.PodStatus { restartMap[pod.ContainerName] = pod.RestartCount } } result := make([]CmiiDeploymentInterface, len(restartMap)) index := 0 log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap) // find deployment convert to interface for key, value := range restartMap { // container Name must equals deployment name deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deployment != nil { // deployment exists log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value) result[index] = *deployment index++ } } return result[:index] } func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) { // get all deployments cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces) // filter for _, deploymentInterface := range cmiiDeploymentInterfaces { if deploymentInterface.Replicas <= replicasMin { deploymentList = append(deploymentList, deploymentInterface) } } // convert return deploymentList } func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) { // all unhealthy pods allInterface := CmiiOperator.PodAllInterface(cmiiEnv) // find the deployments for _, podInterface := range allInterface { if !podInterface.PodStatus { // unhealthy pod deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName) if deploymentInterface != nil { deploymentList = append(deploymentList, *deploymentInterface) } } } return deploymentList } func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) { // dev-cluster devNodeList := CmiiOperator.NodeAllInterface("dev") // core-cluster coreNodeList := CmiiOperator.NodeAllInterface("uat") // append coreNodeList = append(coreNodeList, devNodeList...) // filter for _, node := range coreNodeList { if node.Unschedulable { nodeList = append(nodeList, node) continue } if !node.NodeStatus { nodeList = append(nodeList, node) continue } if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure { nodeList = append(nodeList, node) continue } } return nodeList } func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) { // all unhealthy pods allInterface := CmiiOperator.PodAllInterface(cmiiEnv) // find the deployments for _, podInterface := range allInterface { if !podInterface.PodStatus { // unhealthy pod podList = append(podList, podInterface) } } return podList } func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool { result := true for _, deployment := range deploymentList { result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0) if !result { return result } time.Sleep(time.Second) result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas) if !result { return result } } return result } func RestartCmiiBackendDeployment(cmiiEnv string) { cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { _, ok := CmiiBackendAppMap[deploymentInterface.Name] if ok { if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) } else { log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) } } } log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) } func RestartCmiiFrontendDeployment(cmiiEnv string) { cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { _, ok := CmiiFrontendAppMap[deploymentInterface.Name] if ok { if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) } else { log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) } } } log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) } func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool { deployment := CmiiOperator.DeploymentExist(cmiiEnv, appName) if deployment == nil { log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] not exists !", cmiiEnv, appName) return false } deploymentInterface := CmiiDeploymentInterface{} cmiiDeploymentInterface := deploymentInterface.Convert(*deployment) // check if need to update if cmiiDeploymentInterface.ImageTag == newTag { log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName) return true } content := executor.BasicWordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 45) content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.Name, 85) content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105) content = content + newTag + "\n" log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content) // update tag := CmiiOperator.DeploymentUpdateTag(cmiiEnv, appName, newTag) if !tag { log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content) return false } // append log executor.BasicAppendContentToFile(content, updateLogPath) // re-get from env time.Sleep(time.Second) deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName) if deploy == nil { log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName) return false } // log log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] \n update tag [%s] success ! ", deploy.Image, content) return true } func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool { if !executor.BasicFindContentInFile(updateLog, updateLogPath) { log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog) return false } split := strings.Split(updateLog, " ") index := 0 cmiiEnv := "" appName := "" fromTag := "" newTag := "" for _, s := range split { if s != "" { if index == 1 { cmiiEnv = s } else if index == 2 { appName = s } else if index == 3 { fromTag = s } else if index == 4 { newTag = s } index++ } } log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag) rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag) return rollback } // BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用 func BackupAllDeploymentFromEnv(cmiiEnv string) bool { allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\agent-go\\k8s_exec\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath) firstCol := 0 secondCol := 0 thirdCol := 0 fourthCol := 0 for _, deploymentInterface := range allInterface { firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol) secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol) thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol) fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol) } firstCol += 2 secondCol += 2 secondCol += firstCol thirdCol += 2 thirdCol += secondCol fourthCol += 2 fourthCol += thirdCol for _, deploymentInterface := range allInterface { content := executor.BasicWordSpaceCompletion(deploymentInterface.Name, firstCol) content = executor.BasicWordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol) content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol) content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol) content += "\n" if !executor.BasicAppendContentToFile(content, filePath) { log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content) return false } } return true } func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[string]string) { allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) allInterface = FilterAllCmiiAppSoft(allInterface) backendMap = make(map[string]string, len(allInterface)) frontendMap = make(map[string]string, len(allInterface)) for _, deploymentInterface := range allInterface { if strings.Contains(deploymentInterface.Name, "platform") { frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag } else { backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag } } return backendMap, frontendMap } func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) { for _, c := range source { _, ok := CmiiBackendAppMap[c.ContainerName] if !ok { _, ok = CmiiFrontendAppMap[c.ContainerName] if !ok { log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod !", c.ContainerName) continue } } result = append(result, c) } return result } func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) { for _, c := range source { if strings.Contains(c.ContainerName, "redis") { continue } if strings.Contains(c.ContainerName, "emqxs") { continue } if strings.Contains(c.ContainerName, "rabbitmq") { continue } if strings.Contains(c.ContainerName, "nacos") { continue } if strings.Contains(c.ContainerName, "oss") { continue } if strings.Contains(c.ContainerName, "minio") { continue } if strings.HasPrefix(c.ContainerName, "nfs") { continue } if strings.HasPrefix(c.ContainerName, "operator") { continue } if strings.HasPrefix(c.ContainerName, "proxy") { continue } result = append(result, c) } return result } func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) { for _, c := range podList { _, ok := CmiiBackendAppMap[c.ContainerName] if !ok { _, ok = CmiiFrontendAppMap[c.ContainerName] if !ok { log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod !", c.ContainerName) continue } } result = append(result, c) } return result } func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) { for _, c := range podList { if strings.Contains(c.ContainerName, "redis") { continue } if strings.Contains(c.ContainerName, "emqxs") { continue } if strings.Contains(c.ContainerName, "rabbitmq") { continue } if strings.Contains(c.ContainerName, "nacos") { continue } if strings.Contains(c.ContainerName, "oss") { continue } if strings.Contains(c.ContainerName, "minio") { continue } if strings.HasPrefix(c.ContainerName, "nfs") { continue } if strings.HasPrefix(c.ContainerName, "operator") { continue } if strings.HasPrefix(c.ContainerName, "proxy") { continue } if strings.HasPrefix(c.ContainerName, "cleanlog") { continue } result = append(result, c) } return result }