Files
ProjectOctopus/agent-operator/CmiiK8sOperator.go
2024-04-12 10:57:22 +08:00

729 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package main
import (
"bufio"
"strings"
"time"
image2 "wdd.io/agent-common/image"
"wdd.io/agent-common/utils"
)
var CmiiOperator = CmiiK8sOperator{}
// var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt"
var updateLogPath = "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log/cmii-update-log.txt"
// FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment
func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface {
//podInterface := CmiiPodInterface{}
// get all pods
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
// restart map
restartMap := make(map[string]int32, len(podAll))
// restart count
for _, pod := range podAll {
if pod.RestartCount > restartCount {
restart, exists := restartMap[pod.ContainerName]
if exists {
restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart)
} else {
restartMap[pod.ContainerName] = pod.RestartCount
}
}
// unhealthy
if !pod.PodStatus {
restartMap[pod.ContainerName] = pod.RestartCount
}
}
result := make([]CmiiDeploymentInterface, len(restartMap))
index := 0
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap)
// find deployment convert to interface
for key, value := range restartMap {
// container Name must equals deployment name
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deployment != nil {
// deployment exists
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
result[index] = *deployment
index++
}
}
return result[:index]
}
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
// get all deployments
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
// filter
for _, deploymentInterface := range cmiiDeploymentInterfaces {
if deploymentInterface.Replicas <= replicasMin {
deploymentList = append(deploymentList, deploymentInterface)
}
}
// convert
return deploymentList
}
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
if !podInterface.PodStatus {
// unhealthy pod
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
if deploymentInterface != nil {
deploymentList = append(deploymentList, *deploymentInterface)
}
}
}
return deploymentList
}
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
// dev-cluster
devNodeList := CmiiOperator.NodeAllInterface("dev")
// core-cluster
coreNodeList := CmiiOperator.NodeAllInterface("uat")
// append
coreNodeList = append(coreNodeList, devNodeList...)
// filter
for _, node := range coreNodeList {
if node.Unschedulable {
nodeList = append(nodeList, node)
continue
}
if !node.NodeStatus {
nodeList = append(nodeList, node)
continue
}
if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure {
nodeList = append(nodeList, node)
continue
}
}
return nodeList
}
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
if !podInterface.PodStatus {
// unhealthy pod
podList = append(podList, podInterface)
}
}
return podList
}
func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) {
// get app
podList := CmiiOperator.PodByAppName(cmiiEnv, appName)
// get pod
if podList == nil || len(podList) == 0 {
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - get app pod error [%s] [%s]", cmiiEnv, appName)
return "", ""
}
// exec env
stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
errLog := stderr.String()
if errLog != "" {
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - pod Exec error %s", errLog)
return "", ""
}
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
// Get the current line
line := scanner.Text()
// Check if the current line contains the search term
if strings.HasPrefix(line, "GIT_BRANCH") {
gitBranch = strings.Split(line, "=")[1]
}
if strings.HasPrefix(line, "GIT_COMMIT") {
gitCommit = strings.Split(line, "=")[1]
}
}
// get out git info
return gitBranch, gitCommit
}
func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
for _, podInterface := range cmiiPodInterfaces {
for key := range CmiiMiddlewareNameMap {
if strings.Contains(podInterface.Name, key) {
podList = append(podList, podInterface)
}
}
}
return podList
}
func ScaleCmiiFrontendDeploymentToDesiredReplicas(cmiiEnv string, desiredReplicas int32) (errorUpdateMap map[string]int32) {
frontMap := make(map[string]int32, len(CmiiFrontendAppMap))
for aooName := range CmiiFrontendAppMap {
frontMap[aooName] = desiredReplicas
}
return ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv, frontMap)
}
func ScaleCmiiBackendDeploymentToDesiredReplicas(cmiiEnv string, desiredReplicas int32) (errorUpdateMap map[string]int32) {
backMap := make(map[string]int32, len(CmiiBackendAppMap))
for aooName := range CmiiBackendAppMap {
backMap[aooName] = desiredReplicas
}
return ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv, backMap)
}
func ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv string, nameReplicasMap map[string]int32) map[string]int32 {
errorUpdateMap := make(map[string]int32, len(nameReplicasMap))
// 遍历
for appName, replica := range nameReplicasMap {
exists := CmiiOperator.DeploymentExist(cmiiEnv, appName)
if exists != nil {
scale := CmiiOperator.DeploymentScale(cmiiEnv, appName, replica)
if !scale {
errorUpdateMap[appName] = replica
}
} else {
errorUpdateMap[appName] = replica
}
}
return errorUpdateMap
}
func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
result := true
for _, deployment := range deploymentList {
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
if !result {
return result
}
time.Sleep(time.Second)
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
if !result {
return result
}
}
return result
}
func RestartCmiiBackendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
if AppNameBelongsToCmiiImage(deploymentInterface.Name) {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
}
}
}
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
}
func RestartCmiiFrontendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
if ok {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
}
}
}
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
}
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
cmiiDeploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if cmiiDeploymentInterface == nil {
return false
}
// check if need to update
if cmiiDeploymentInterface.ImageTag == newTag {
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
// restart
if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
return true
} else {
return false
}
}
content := utils.WordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 35)
content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.Name, 75)
content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
content = content + newTag + "\n"
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
// update
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
if !tag {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
return false
}
// append log
utils.AppendContentToFile(content, updateLogPath)
// re-get from env
time.Sleep(time.Second)
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if deploy == nil {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
return false
}
// log
//log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] update tag [%s] success ! ", deploy.Image, content)
return true
}
func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
result = make(map[string]string, len(nameTagMap))
for appName, newTag := range nameTagMap {
if AppNameBelongsToCmiiImage(appName) {
if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
log.InfoF("[UpdateCmiiImageTagFromNameTagMap] - %s %s to %s", cmiiEnv, appName, newTag)
result[appName] = newTag
} else {
result[appName] = "false"
}
}
}
return result
}
func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
//if !executor.BasicFindContentInFile(updateLog, updateLogPath) {
// log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog)
// return false
//}
split := strings.Split(updateLog, " ")
index := 0
cmiiEnv := ""
appName := ""
fromTag := ""
newTag := ""
for _, s := range split {
if s != "" {
if index == 1 {
cmiiEnv = s
} else if index == 2 {
appName = s
} else if index == 3 {
fromTag = s
} else if index == 4 {
newTag = s
}
index++
}
}
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
return rollback
}
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
// must filter
allInterface = FilterAllCmiiAppSoft(allInterface)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
firstCol := 0
secondCol := 0
thirdCol := 0
fourthCol := 0
for _, deploymentInterface := range allInterface {
firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol)
secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol)
thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol)
fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol)
}
firstCol += 2
secondCol += 2
secondCol += firstCol
thirdCol += 2
thirdCol += secondCol
fourthCol += 2
fourthCol += thirdCol
for _, deploymentInterface := range allInterface {
if deploymentInterface.GitBranch == "" {
branch, commit := GetDeploymentGitInfoFromInnerEnv(deploymentInterface.Namespace, deploymentInterface.Name)
deploymentInterface.GitBranch = branch
deploymentInterface.GitCommit = commit
}
content := utils.WordSpaceCompletion(deploymentInterface.Name, firstCol)
content = utils.WordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol)
content = utils.WordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol)
content = utils.WordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol)
content += "\n"
if !utils.AppendContentToFile(content, filePath) {
log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content)
return false
}
}
return true
}
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsMap map[string]string) {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface = FilterAllCmiiAppSoft(allInterface)
backendMap = make(map[string]string, len(allInterface))
frontendMap = make(map[string]string, len(allInterface))
srsMap = make(map[string]string, len(allInterface))
for _, deploymentInterface := range allInterface {
if strings.Contains(deploymentInterface.Name, "platform") {
frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
} else {
backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
}
}
// add srs part
for key, value := range CmiiSrsAppMap {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
split := strings.Split(imageName, ":")
if strings.Contains(split[0], image2.CmiiHarborPrefix) {
split[0] = strings.Split(split[0], image2.CmiiHarborPrefix)[1]
}
srsMap[split[0]] = split[1]
}
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
split := strings.Split(imageName, ":")
split[0], _ = strings.CutPrefix(split[0], image2.CmiiHarborPrefix)
srsMap[split[0]] = split[1]
}
}
}
}
return backendMap, frontendMap, srsMap
}
func BackupAllCmiiDeploymentToList(cmiiEnv string, completePrefix bool) (allCmiiImageList []string) {
backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(cmiiEnv)
allImageMap := make(map[string]string, len(backendMap)+len(frontendMap)+len(srsMap))
for name, tag := range backendMap {
allImageMap[name] = tag
}
for name, tag := range frontendMap {
allImageMap[name] = tag
}
for name, tag := range srsMap {
allImageMap[name] = tag
}
for name, tag := range allImageMap {
if completePrefix {
allCmiiImageList = append(allCmiiImageList, image2.CmiiHarborPrefix+name+":"+tag)
} else {
allCmiiImageList = append(allCmiiImageList, name+":"+tag)
}
}
return allCmiiImageList
}
func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
CmiiOperator.changeOperatorEnv(cmiiEnv)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
only := make(map[string]string, 150)
// front
utils.AppendContentToFile("---\n", filePath)
for key, value := range CmiiFrontendAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
}
}
}
utils.AppendContentToFile("---\n", filePath)
for key, value := range CmiiBackendAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
}
}
}
// backend
utils.AppendContentToFile("---\n", filePath)
// gis server
for key, value := range CmiiGISAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
}
}
}
// srs
utils.AppendContentToFile("---\n", filePath)
for key, value := range CmiiSrsAppMap {
_, ok := only[key]
if !ok {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if app != nil {
only[key] = value
utils.AppendContentToFile(app.Image+"\n", filePath)
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
if app != nil {
only[key] = value
for _, imageName := range app.ContainerImageMap {
utils.AppendContentToFile(imageName+"\n", filePath)
}
}
}
}
}
utils.AppendContentToFile("---\n", filePath)
}
func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
for _, c := range source {
_, ok := CmiiBackendAppMap[c.ContainerName]
if !ok {
_, ok = CmiiFrontendAppMap[c.ContainerName]
if !ok {
log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod ", c.ContainerName)
continue
}
}
result = append(result, c)
}
return result
}
func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
for _, c := range source {
if strings.Contains(c.ContainerName, "redis") {
continue
}
if strings.Contains(c.ContainerName, "emqxs") {
continue
}
if strings.Contains(c.ContainerName, "rabbitmq") {
continue
}
if strings.Contains(c.ContainerName, "nacos") {
continue
}
if strings.Contains(c.ContainerName, "oss") {
continue
}
if strings.Contains(c.ContainerName, "minio") {
continue
}
if strings.HasPrefix(c.ContainerName, "nfs") {
continue
}
if strings.HasPrefix(c.ContainerName, "operator") {
continue
}
if strings.HasPrefix(c.ContainerName, "proxy") {
continue
}
result = append(result, c)
}
return result
}
func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) {
for _, c := range podList {
_, ok := CmiiBackendAppMap[c.ContainerName]
if !ok {
_, ok = CmiiFrontendAppMap[c.ContainerName]
if !ok {
log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod ", c.ContainerName)
continue
}
}
result = append(result, c)
}
return result
}
func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) {
for _, c := range podList {
if strings.Contains(c.ContainerName, "redis") {
continue
}
if strings.Contains(c.ContainerName, "emqxs") {
continue
}
if strings.Contains(c.ContainerName, "rabbitmq") {
continue
}
if strings.Contains(c.ContainerName, "nacos") {
continue
}
if strings.Contains(c.ContainerName, "oss") {
continue
}
if strings.Contains(c.ContainerName, "minio") {
continue
}
if strings.HasPrefix(c.ContainerName, "nfs") {
continue
}
if strings.HasPrefix(c.ContainerName, "operator") {
continue
}
if strings.HasPrefix(c.ContainerName, "proxy") {
continue
}
if strings.HasPrefix(c.ContainerName, "cleanlog") {
continue
}
result = append(result, c)
}
return result
}
func FilterAllCmiiNodeSoft(nodeList []CmiiNodeInterface) (result []CmiiNodeInterface) {
for _, nodeInterface := range nodeList {
if strings.HasPrefix(nodeInterface.Name, "ai") {
continue
}
if strings.HasPrefix(nodeInterface.Name, "35") {
continue
}
result = append(result, nodeInterface)
}
return result
}
func AppNameBelongsToCmiiImage(appName string) bool {
_, ok := CmiiBackendAppMap[appName]
if !ok {
_, ok = CmiiFrontendAppMap[appName]
if !ok {
log.WarnF("[AppNameBelongsToCmiiImage] - [%s] not cmii app ", appName)
return false
} else {
return true
}
} else {
return true
}
}