package main import ( "bufio" "strings" "time" image2 "wdd.io/agent-common/image" "wdd.io/agent-common/utils" ) var CmiiOperator = CmiiK8sOperator{} // var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt" var updateLogPath = "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log/cmii-update-log.txt" // FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface { //podInterface := CmiiPodInterface{} // get all pods podAll := CmiiOperator.PodAllInterface(cmiiEnv) // restart map restartMap := make(map[string]int32, len(podAll)) // restart count for _, pod := range podAll { if pod.RestartCount > restartCount { restart, exists := restartMap[pod.ContainerName] if exists { restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart) } else { restartMap[pod.ContainerName] = pod.RestartCount } } // unhealthy if !pod.PodStatus { restartMap[pod.ContainerName] = pod.RestartCount } } result := make([]CmiiDeploymentInterface, len(restartMap)) index := 0 log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap) // find deployment convert to interface for key, value := range restartMap { // container Name must equals deployment name deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deployment != nil { // deployment exists log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value) result[index] = *deployment index++ } } return result[:index] } func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) { // get all deployments cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces) // filter for _, deploymentInterface := range cmiiDeploymentInterfaces { if deploymentInterface.Replicas <= replicasMin { deploymentList = append(deploymentList, deploymentInterface) } } // convert return deploymentList } func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) { // all unhealthy pods allInterface := CmiiOperator.PodAllInterface(cmiiEnv) // find the deployments for _, podInterface := range allInterface { if !podInterface.PodStatus { // unhealthy pod deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName) if deploymentInterface != nil { deploymentList = append(deploymentList, *deploymentInterface) } } } return deploymentList } func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) { // dev-cluster devNodeList := CmiiOperator.NodeAllInterface("dev") // core-cluster coreNodeList := CmiiOperator.NodeAllInterface("uat") // append coreNodeList = append(coreNodeList, devNodeList...) // filter for _, node := range coreNodeList { if node.Unschedulable { nodeList = append(nodeList, node) continue } if !node.NodeStatus { nodeList = append(nodeList, node) continue } if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure { nodeList = append(nodeList, node) continue } } return nodeList } func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) { // all unhealthy pods allInterface := CmiiOperator.PodAllInterface(cmiiEnv) // find the deployments for _, podInterface := range allInterface { if !podInterface.PodStatus { // unhealthy pod podList = append(podList, podInterface) } } return podList } func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) { // get app podList := CmiiOperator.PodByAppName(cmiiEnv, appName) // get pod if podList == nil || len(podList) == 0 { log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - get app pod error [%s] [%s]", cmiiEnv, appName) return "", "" } // exec env stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"}) errLog := stderr.String() if errLog != "" { log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - pod Exec error %s", errLog) return "", "" } scanner := bufio.NewScanner(stdout) for scanner.Scan() { // Get the current line line := scanner.Text() // Check if the current line contains the search term if strings.HasPrefix(line, "GIT_BRANCH") { gitBranch = strings.Split(line, "=")[1] } if strings.HasPrefix(line, "GIT_COMMIT") { gitCommit = strings.Split(line, "=")[1] } } // get out git info return gitBranch, gitCommit } func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) { cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv) for _, podInterface := range cmiiPodInterfaces { for key := range CmiiMiddlewareNameMap { if strings.Contains(podInterface.Name, key) { podList = append(podList, podInterface) } } } return podList } func ScaleCmiiFrontendDeploymentToDesiredReplicas(cmiiEnv string, desiredReplicas int32) (errorUpdateMap map[string]int32) { frontMap := make(map[string]int32, len(CmiiFrontendAppMap)) for aooName := range CmiiFrontendAppMap { frontMap[aooName] = desiredReplicas } return ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv, frontMap) } func ScaleCmiiBackendDeploymentToDesiredReplicas(cmiiEnv string, desiredReplicas int32) (errorUpdateMap map[string]int32) { backMap := make(map[string]int32, len(CmiiBackendAppMap)) for aooName := range CmiiBackendAppMap { backMap[aooName] = desiredReplicas } return ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv, backMap) } func ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv string, nameReplicasMap map[string]int32) map[string]int32 { errorUpdateMap := make(map[string]int32, len(nameReplicasMap)) // 遍历 for appName, replica := range nameReplicasMap { exists := CmiiOperator.DeploymentExist(cmiiEnv, appName) if exists != nil { scale := CmiiOperator.DeploymentScale(cmiiEnv, appName, replica) if !scale { errorUpdateMap[appName] = replica } } else { errorUpdateMap[appName] = replica } } return errorUpdateMap } func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool { result := true for _, deployment := range deploymentList { result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0) if !result { return result } time.Sleep(time.Second) result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas) if !result { return result } } return result } func RestartCmiiBackendDeployment(cmiiEnv string) { cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { if AppNameBelongsToCmiiImage(deploymentInterface.Name) { if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) } else { log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) } } } log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) } func RestartCmiiFrontendDeployment(cmiiEnv string) { cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { _, ok := CmiiFrontendAppMap[deploymentInterface.Name] if ok { if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) } else { log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) } } } log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) } func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool { cmiiDeploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName) if cmiiDeploymentInterface == nil { return false } // check if need to update if cmiiDeploymentInterface.ImageTag == newTag { log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName) // restart if CmiiOperator.DeploymentRestart(cmiiEnv, appName) { return true } else { return false } } content := utils.WordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 35) content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.Name, 75) content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105) content = content + newTag + "\n" log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content) // update tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag) if !tag { log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content) return false } // append log utils.AppendContentToFile(content, updateLogPath) // re-get from env time.Sleep(time.Second) deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName) if deploy == nil { log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName) return false } // log //log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] update tag [%s] success ! ", deploy.Image, content) return true } func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) { result = make(map[string]string, len(nameTagMap)) for appName, newTag := range nameTagMap { if AppNameBelongsToCmiiImage(appName) { if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) { log.InfoF("[UpdateCmiiImageTagFromNameTagMap] - %s %s to %s", cmiiEnv, appName, newTag) result[appName] = newTag } else { result[appName] = "false" } } } return result } func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool { //if !executor.BasicFindContentInFile(updateLog, updateLogPath) { // log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog) // return false //} split := strings.Split(updateLog, " ") index := 0 cmiiEnv := "" appName := "" fromTag := "" newTag := "" for _, s := range split { if s != "" { if index == 1 { cmiiEnv = s } else if index == 2 { appName = s } else if index == 3 { fromTag = s } else if index == 4 { newTag = s } index++ } } log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag) rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag) return rollback } // BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用 func BackupAllDeploymentFromEnv(cmiiEnv string) bool { allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) // must filter allInterface = FilterAllCmiiAppSoft(allInterface) //filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath) firstCol := 0 secondCol := 0 thirdCol := 0 fourthCol := 0 for _, deploymentInterface := range allInterface { firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol) secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol) thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol) fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol) } firstCol += 2 secondCol += 2 secondCol += firstCol thirdCol += 2 thirdCol += secondCol fourthCol += 2 fourthCol += thirdCol for _, deploymentInterface := range allInterface { if deploymentInterface.GitBranch == "" { branch, commit := GetDeploymentGitInfoFromInnerEnv(deploymentInterface.Namespace, deploymentInterface.Name) deploymentInterface.GitBranch = branch deploymentInterface.GitCommit = commit } content := utils.WordSpaceCompletion(deploymentInterface.Name, firstCol) content = utils.WordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol) content = utils.WordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol) content = utils.WordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol) content += "\n" if !utils.AppendContentToFile(content, filePath) { log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content) return false } } return true } func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsMap map[string]string) { allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) allInterface = FilterAllCmiiAppSoft(allInterface) backendMap = make(map[string]string, len(allInterface)) frontendMap = make(map[string]string, len(allInterface)) srsMap = make(map[string]string, len(allInterface)) for _, deploymentInterface := range allInterface { if strings.Contains(deploymentInterface.Name, "platform") { frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag } else { backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag } } // add srs part for key, value := range CmiiSrsAppMap { var app *CmiiDeploymentInterface if strings.Contains(value, "deployment") { app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if app != nil { for _, imageName := range app.ContainerImageMap { split := strings.Split(imageName, ":") if strings.Contains(split[0], image2.CmiiHarborPrefix) { split[0] = strings.Split(split[0], image2.CmiiHarborPrefix)[1] } srsMap[split[0]] = split[1] } } } else if strings.Contains(value, "state") { app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key) if app != nil { for _, imageName := range app.ContainerImageMap { split := strings.Split(imageName, ":") split[0], _ = strings.CutPrefix(split[0], image2.CmiiHarborPrefix) srsMap[split[0]] = split[1] } } } } return backendMap, frontendMap, srsMap } func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) { CmiiOperator.changeOperatorEnv(cmiiEnv) //filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" only := make(map[string]string, 150) // front utils.AppendContentToFile("---\n", filePath) for key, value := range CmiiFrontendAppMap { _, ok := only[key] if !ok { deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deploy != nil { only[key] = value utils.AppendContentToFile(deploy.Image+"\n", filePath) } } } utils.AppendContentToFile("---\n", filePath) for key, value := range CmiiBackendAppMap { _, ok := only[key] if !ok { deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deploy != nil { only[key] = value utils.AppendContentToFile(deploy.Image+"\n", filePath) } } } // backend utils.AppendContentToFile("---\n", filePath) // gis server for key, value := range CmiiGISAppMap { _, ok := only[key] if !ok { deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deploy != nil { only[key] = value utils.AppendContentToFile(deploy.Image+"\n", filePath) } } } // srs utils.AppendContentToFile("---\n", filePath) for key, value := range CmiiSrsAppMap { _, ok := only[key] if !ok { var app *CmiiDeploymentInterface if strings.Contains(value, "deployment") { app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key) if app != nil { only[key] = value utils.AppendContentToFile(app.Image+"\n", filePath) } } else if strings.Contains(value, "state") { app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key) if app != nil { only[key] = value for _, imageName := range app.ContainerImageMap { utils.AppendContentToFile(imageName+"\n", filePath) } } } } } utils.AppendContentToFile("---\n", filePath) } func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) { for _, c := range source { _, ok := CmiiBackendAppMap[c.ContainerName] if !ok { _, ok = CmiiFrontendAppMap[c.ContainerName] if !ok { log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod !", c.ContainerName) continue } } result = append(result, c) } return result } func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) { for _, c := range source { if strings.Contains(c.ContainerName, "redis") { continue } if strings.Contains(c.ContainerName, "emqxs") { continue } if strings.Contains(c.ContainerName, "rabbitmq") { continue } if strings.Contains(c.ContainerName, "nacos") { continue } if strings.Contains(c.ContainerName, "oss") { continue } if strings.Contains(c.ContainerName, "minio") { continue } if strings.HasPrefix(c.ContainerName, "nfs") { continue } if strings.HasPrefix(c.ContainerName, "operator") { continue } if strings.HasPrefix(c.ContainerName, "proxy") { continue } result = append(result, c) } return result } func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) { for _, c := range podList { _, ok := CmiiBackendAppMap[c.ContainerName] if !ok { _, ok = CmiiFrontendAppMap[c.ContainerName] if !ok { log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod !", c.ContainerName) continue } } result = append(result, c) } return result } func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) { for _, c := range podList { if strings.Contains(c.ContainerName, "redis") { continue } if strings.Contains(c.ContainerName, "emqxs") { continue } if strings.Contains(c.ContainerName, "rabbitmq") { continue } if strings.Contains(c.ContainerName, "nacos") { continue } if strings.Contains(c.ContainerName, "oss") { continue } if strings.Contains(c.ContainerName, "minio") { continue } if strings.HasPrefix(c.ContainerName, "nfs") { continue } if strings.HasPrefix(c.ContainerName, "operator") { continue } if strings.HasPrefix(c.ContainerName, "proxy") { continue } if strings.HasPrefix(c.ContainerName, "cleanlog") { continue } result = append(result, c) } return result } func FilterAllCmiiNodeSoft(nodeList []CmiiNodeInterface) (result []CmiiNodeInterface) { for _, nodeInterface := range nodeList { if strings.HasPrefix(nodeInterface.Name, "ai") { continue } if strings.HasPrefix(nodeInterface.Name, "35") { continue } result = append(result, nodeInterface) } return result } func AppNameBelongsToCmiiImage(appName string) bool { _, ok := CmiiBackendAppMap[appName] if !ok { _, ok = CmiiFrontendAppMap[appName] if !ok { log.WarnF("[AppNameBelongsToCmiiImage] - [%s] not cmii app !", appName) return false } else { return true } } else { return true } }