572 lines
15 KiB
Go
572 lines
15 KiB
Go
package cmii_operator
|
||
|
||
import (
|
||
"bufio"
|
||
"strings"
|
||
"time"
|
||
"wdd.io/agent-go/executor"
|
||
"wdd.io/agent-go/utils"
|
||
)
|
||
|
||
var CmiiOperator = CmiiK8sOperator{}
|
||
var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt"
|
||
|
||
// FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment
|
||
func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface {
|
||
|
||
//podInterface := CmiiPodInterface{}
|
||
|
||
// get all pods
|
||
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
// restart map
|
||
restartMap := make(map[string]int32, len(podAll))
|
||
|
||
// restart count
|
||
for _, pod := range podAll {
|
||
if pod.RestartCount > restartCount {
|
||
restart, exists := restartMap[pod.ContainerName]
|
||
if exists {
|
||
restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart)
|
||
} else {
|
||
restartMap[pod.ContainerName] = pod.RestartCount
|
||
}
|
||
}
|
||
|
||
// unhealthy
|
||
if !pod.PodStatus {
|
||
restartMap[pod.ContainerName] = pod.RestartCount
|
||
}
|
||
}
|
||
|
||
result := make([]CmiiDeploymentInterface, len(restartMap))
|
||
index := 0
|
||
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap)
|
||
// find deployment convert to interface
|
||
for key, value := range restartMap {
|
||
// container Name must equals deployment name
|
||
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
|
||
if deployment != nil {
|
||
// deployment exists
|
||
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
|
||
|
||
result[index] = *deployment
|
||
index++
|
||
}
|
||
}
|
||
|
||
return result[:index]
|
||
}
|
||
|
||
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
|
||
|
||
// get all deployments
|
||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
|
||
|
||
// filter
|
||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||
if deploymentInterface.Replicas <= replicasMin {
|
||
deploymentList = append(deploymentList, deploymentInterface)
|
||
}
|
||
}
|
||
|
||
// convert
|
||
return deploymentList
|
||
}
|
||
|
||
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
|
||
|
||
// all unhealthy pods
|
||
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
// find the deployments
|
||
for _, podInterface := range allInterface {
|
||
if !podInterface.PodStatus {
|
||
// unhealthy pod
|
||
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
|
||
if deploymentInterface != nil {
|
||
deploymentList = append(deploymentList, *deploymentInterface)
|
||
}
|
||
}
|
||
}
|
||
|
||
return deploymentList
|
||
}
|
||
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
|
||
|
||
// dev-cluster
|
||
devNodeList := CmiiOperator.NodeAllInterface("dev")
|
||
|
||
// core-cluster
|
||
coreNodeList := CmiiOperator.NodeAllInterface("uat")
|
||
|
||
// append
|
||
coreNodeList = append(coreNodeList, devNodeList...)
|
||
|
||
// filter
|
||
for _, node := range coreNodeList {
|
||
if node.Unschedulable {
|
||
nodeList = append(nodeList, node)
|
||
continue
|
||
}
|
||
|
||
if !node.NodeStatus {
|
||
nodeList = append(nodeList, node)
|
||
continue
|
||
}
|
||
|
||
if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure {
|
||
nodeList = append(nodeList, node)
|
||
continue
|
||
}
|
||
}
|
||
|
||
return nodeList
|
||
}
|
||
|
||
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
|
||
// all unhealthy pods
|
||
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
// find the deployments
|
||
for _, podInterface := range allInterface {
|
||
if !podInterface.PodStatus {
|
||
// unhealthy pod
|
||
podList = append(podList, podInterface)
|
||
}
|
||
}
|
||
|
||
return podList
|
||
}
|
||
|
||
func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) {
|
||
|
||
// get app
|
||
podList := CmiiOperator.PodByAppName(cmiiEnv, appName)
|
||
|
||
// get pod
|
||
if podList == nil || len(podList) == 0 {
|
||
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - get app pod error [%s] [%s]", cmiiEnv, appName)
|
||
return "", ""
|
||
}
|
||
|
||
// exec env
|
||
stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
|
||
|
||
errLog := stderr.String()
|
||
if errLog != "" {
|
||
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - pod Exec error %s", errLog)
|
||
return "", ""
|
||
}
|
||
|
||
scanner := bufio.NewScanner(stdout)
|
||
for scanner.Scan() {
|
||
// Get the current line
|
||
line := scanner.Text()
|
||
|
||
// Check if the current line contains the search term
|
||
if strings.HasPrefix(line, "GIT_BRANCH") {
|
||
gitBranch = strings.Split(line, "=")[1]
|
||
}
|
||
if strings.HasPrefix(line, "GIT_COMMIT") {
|
||
gitCommit = strings.Split(line, "=")[1]
|
||
}
|
||
}
|
||
|
||
// get out git info
|
||
return gitBranch, gitCommit
|
||
}
|
||
|
||
func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
|
||
|
||
cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
for _, podInterface := range cmiiPodInterfaces {
|
||
for key := range CmiiMiddlewareNameMap {
|
||
if strings.Contains(podInterface.Name, key) {
|
||
podList = append(podList, podInterface)
|
||
}
|
||
}
|
||
}
|
||
|
||
return podList
|
||
}
|
||
|
||
func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
|
||
|
||
result := true
|
||
|
||
for _, deployment := range deploymentList {
|
||
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
|
||
if !result {
|
||
return result
|
||
}
|
||
time.Sleep(time.Second)
|
||
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
|
||
if !result {
|
||
return result
|
||
}
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func RestartCmiiBackendDeployment(cmiiEnv string) {
|
||
|
||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
|
||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||
if AppNameBelongsToCmiiImage(deploymentInterface.Name) {
|
||
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
|
||
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
} else {
|
||
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
}
|
||
|
||
}
|
||
}
|
||
|
||
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
|
||
}
|
||
|
||
func RestartCmiiFrontendDeployment(cmiiEnv string) {
|
||
|
||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
|
||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
|
||
if ok {
|
||
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
|
||
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
} else {
|
||
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
}
|
||
}
|
||
}
|
||
|
||
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
|
||
}
|
||
|
||
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
|
||
|
||
deployment := CmiiOperator.DeploymentExist(cmiiEnv, appName)
|
||
if deployment == nil {
|
||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] not exists !", cmiiEnv, appName)
|
||
return false
|
||
}
|
||
|
||
deploymentInterface := CmiiDeploymentInterface{}
|
||
cmiiDeploymentInterface := deploymentInterface.Convert(*deployment)
|
||
// check if need to update
|
||
if cmiiDeploymentInterface.ImageTag == newTag {
|
||
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
|
||
|
||
// restart
|
||
if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
|
||
return true
|
||
} else {
|
||
return false
|
||
}
|
||
}
|
||
|
||
content := executor.BasicWordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 45)
|
||
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.Name, 85)
|
||
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
|
||
content = content + newTag + "\n"
|
||
|
||
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
|
||
|
||
// update
|
||
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
|
||
if !tag {
|
||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
|
||
return false
|
||
}
|
||
|
||
// append log
|
||
executor.BasicAppendContentToFile(content, updateLogPath)
|
||
|
||
// re-get from env
|
||
time.Sleep(time.Second)
|
||
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
|
||
if deploy == nil {
|
||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
|
||
return false
|
||
}
|
||
|
||
// log
|
||
log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] \n update tag [%s] success ! ", deploy.Image, content)
|
||
return true
|
||
}
|
||
|
||
func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
|
||
|
||
result = make(map[string]string, len(nameTagMap))
|
||
for appName, newTag := range nameTagMap {
|
||
if AppNameBelongsToCmiiImage(appName) {
|
||
if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
|
||
result[appName] = newTag
|
||
} else {
|
||
result[appName] = "false"
|
||
}
|
||
}
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
|
||
|
||
if !executor.BasicFindContentInFile(updateLog, updateLogPath) {
|
||
log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog)
|
||
return false
|
||
}
|
||
|
||
split := strings.Split(updateLog, " ")
|
||
index := 0
|
||
cmiiEnv := ""
|
||
appName := ""
|
||
fromTag := ""
|
||
newTag := ""
|
||
for _, s := range split {
|
||
if s != "" {
|
||
if index == 1 {
|
||
cmiiEnv = s
|
||
} else if index == 2 {
|
||
appName = s
|
||
} else if index == 3 {
|
||
fromTag = s
|
||
} else if index == 4 {
|
||
newTag = s
|
||
}
|
||
index++
|
||
}
|
||
}
|
||
|
||
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
|
||
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
|
||
|
||
return rollback
|
||
}
|
||
|
||
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
|
||
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
|
||
|
||
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
|
||
// must filter
|
||
allInterface = FilterAllCmiiAppSoft(allInterface)
|
||
|
||
filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
|
||
|
||
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
|
||
|
||
firstCol := 0
|
||
secondCol := 0
|
||
thirdCol := 0
|
||
fourthCol := 0
|
||
|
||
for _, deploymentInterface := range allInterface {
|
||
|
||
firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol)
|
||
secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol)
|
||
thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol)
|
||
fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol)
|
||
}
|
||
|
||
firstCol += 2
|
||
secondCol += 2
|
||
secondCol += firstCol
|
||
thirdCol += 2
|
||
thirdCol += secondCol
|
||
fourthCol += 2
|
||
fourthCol += thirdCol
|
||
|
||
for _, deploymentInterface := range allInterface {
|
||
|
||
if deploymentInterface.GitBranch == "" {
|
||
branch, commit := GetDeploymentGitInfoFromInnerEnv(deploymentInterface.Namespace, deploymentInterface.Name)
|
||
deploymentInterface.GitBranch = branch
|
||
deploymentInterface.GitCommit = commit
|
||
}
|
||
|
||
content := executor.BasicWordSpaceCompletion(deploymentInterface.Name, firstCol)
|
||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol)
|
||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol)
|
||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol)
|
||
content += "\n"
|
||
|
||
if !executor.BasicAppendContentToFile(content, filePath) {
|
||
log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content)
|
||
return false
|
||
}
|
||
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[string]string) {
|
||
|
||
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
allInterface = FilterAllCmiiAppSoft(allInterface)
|
||
|
||
backendMap = make(map[string]string, len(allInterface))
|
||
frontendMap = make(map[string]string, len(allInterface))
|
||
|
||
for _, deploymentInterface := range allInterface {
|
||
if strings.Contains(deploymentInterface.Name, "platform") {
|
||
frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
|
||
} else {
|
||
backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
|
||
}
|
||
}
|
||
|
||
return backendMap, frontendMap
|
||
}
|
||
|
||
func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
|
||
|
||
for _, c := range source {
|
||
_, ok := CmiiBackendAppMap[c.ContainerName]
|
||
if !ok {
|
||
_, ok = CmiiFrontendAppMap[c.ContainerName]
|
||
if !ok {
|
||
log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod ！", c.ContainerName)
|
||
continue
|
||
}
|
||
}
|
||
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
|
||
|
||
for _, c := range source {
|
||
if strings.Contains(c.ContainerName, "redis") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "emqxs") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "rabbitmq") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "nacos") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "oss") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "minio") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "nfs") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "operator") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "proxy") {
|
||
continue
|
||
}
|
||
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) {
|
||
for _, c := range podList {
|
||
_, ok := CmiiBackendAppMap[c.ContainerName]
|
||
if !ok {
|
||
_, ok = CmiiFrontendAppMap[c.ContainerName]
|
||
if !ok {
|
||
log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod ！", c.ContainerName)
|
||
continue
|
||
}
|
||
}
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) {
|
||
for _, c := range podList {
|
||
if strings.Contains(c.ContainerName, "redis") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "emqxs") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "rabbitmq") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "nacos") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "oss") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "minio") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "nfs") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "operator") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "proxy") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "cleanlog") {
|
||
continue
|
||
}
|
||
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiNodeSoft(nodeList []CmiiNodeInterface) (result []CmiiNodeInterface) {
|
||
|
||
for _, nodeInterface := range nodeList {
|
||
|
||
if strings.HasPrefix(nodeInterface.Name, "ai") {
|
||
continue
|
||
}
|
||
|
||
if strings.HasPrefix(nodeInterface.Name, "35") {
|
||
continue
|
||
}
|
||
|
||
result = append(result, nodeInterface)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func AppNameBelongsToCmiiImage(appName string) bool {
|
||
_, ok := CmiiBackendAppMap[appName]
|
||
if !ok {
|
||
_, ok = CmiiFrontendAppMap[appName]
|
||
if !ok {
|
||
log.WarnF("[AppNameBelongsToCmiiImage] - [%s] not cmii app ！", appName)
|
||
return false
|
||
} else {
|
||
return true
|
||
}
|
||
} else {
|
||
return true
|
||
}
|
||
|
||
}
|