[ CMII ] [ Operator ] - 抽离为单独的项目

This commit is contained in:
zeaslity
2024-01-22 14:33:55 +08:00
parent f1fb7e8309
commit 9aaabc0337
21 changed files with 911 additions and 42 deletions

View File

@@ -0,0 +1,571 @@
package cmii_operator
import (
"bufio"
"strings"
"time"
"wdd.io/agent-go/executor"
"wdd.io/agent-go/utils"
)
var CmiiOperator = CmiiK8sOperator{}
var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt"
// FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment
func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface {
//podInterface := CmiiPodInterface{}
// get all pods
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
// restart map
restartMap := make(map[string]int32, len(podAll))
// restart count
for _, pod := range podAll {
if pod.RestartCount > restartCount {
restart, exists := restartMap[pod.ContainerName]
if exists {
restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart)
} else {
restartMap[pod.ContainerName] = pod.RestartCount
}
}
// unhealthy
if !pod.PodStatus {
restartMap[pod.ContainerName] = pod.RestartCount
}
}
result := make([]CmiiDeploymentInterface, len(restartMap))
index := 0
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap)
// find deployment convert to interface
for key, value := range restartMap {
// container Name must equals deployment name
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deployment != nil {
// deployment exists
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
result[index] = *deployment
index++
}
}
return result[:index]
}
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
// get all deployments
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
// filter
for _, deploymentInterface := range cmiiDeploymentInterfaces {
if deploymentInterface.Replicas <= replicasMin {
deploymentList = append(deploymentList, deploymentInterface)
}
}
// convert
return deploymentList
}
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
if !podInterface.PodStatus {
// unhealthy pod
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
if deploymentInterface != nil {
deploymentList = append(deploymentList, *deploymentInterface)
}
}
}
return deploymentList
}
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
// dev-cluster
devNodeList := CmiiOperator.NodeAllInterface("dev")
// core-cluster
coreNodeList := CmiiOperator.NodeAllInterface("uat")
// append
coreNodeList = append(coreNodeList, devNodeList...)
// filter
for _, node := range coreNodeList {
if node.Unschedulable {
nodeList = append(nodeList, node)
continue
}
if !node.NodeStatus {
nodeList = append(nodeList, node)
continue
}
if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure {
nodeList = append(nodeList, node)
continue
}
}
return nodeList
}
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
if !podInterface.PodStatus {
// unhealthy pod
podList = append(podList, podInterface)
}
}
return podList
}
func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) {
// get app
podList := CmiiOperator.PodByAppName(cmiiEnv, appName)
// get pod
if podList == nil || len(podList) == 0 {
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - get app pod error [%s] [%s]", cmiiEnv, appName)
return "", ""
}
// exec env
stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
errLog := stderr.String()
if errLog != "" {
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - pod Exec error %s", errLog)
return "", ""
}
scanner := bufio.NewScanner(stdout)
for scanner.Scan() {
// Get the current line
line := scanner.Text()
// Check if the current line contains the search term
if strings.HasPrefix(line, "GIT_BRANCH") {
gitBranch = strings.Split(line, "=")[1]
}
if strings.HasPrefix(line, "GIT_COMMIT") {
gitCommit = strings.Split(line, "=")[1]
}
}
// get out git info
return gitBranch, gitCommit
}
func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
for _, podInterface := range cmiiPodInterfaces {
for key := range CmiiMiddlewareNameMap {
if strings.Contains(podInterface.Name, key) {
podList = append(podList, podInterface)
}
}
}
return podList
}
func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
result := true
for _, deployment := range deploymentList {
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
if !result {
return result
}
time.Sleep(time.Second)
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
if !result {
return result
}
}
return result
}
func RestartCmiiBackendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
_, ok := CmiiBackendAppMap[deploymentInterface.Name]
if ok {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
}
}
}
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
}
func RestartCmiiFrontendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
if ok {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
}
}
}
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
}
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
deployment := CmiiOperator.DeploymentExist(cmiiEnv, appName)
if deployment == nil {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] not exists !", cmiiEnv, appName)
return false
}
deploymentInterface := CmiiDeploymentInterface{}
cmiiDeploymentInterface := deploymentInterface.Convert(*deployment)
// check if need to update
if cmiiDeploymentInterface.ImageTag == newTag {
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
// restart
if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
return true
} else {
return false
}
}
content := executor.BasicWordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 45)
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.Name, 85)
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
content = content + newTag + "\n"
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
// update
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
if !tag {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
return false
}
// append log
executor.BasicAppendContentToFile(content, updateLogPath)
// re-get from env
time.Sleep(time.Second)
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if deploy == nil {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
return false
}
// log
log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] \n update tag [%s] success ! ", deploy.Image, content)
return true
}
func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
result = make(map[string]string, len(nameTagMap))
for appName, newTag := range nameTagMap {
if AppNameBelongsToCmiiImage(appName) {
if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
result[appName] = newTag
} else {
result[appName] = "false"
}
}
}
return result
}
func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
if !executor.BasicFindContentInFile(updateLog, updateLogPath) {
log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog)
return false
}
split := strings.Split(updateLog, " ")
index := 0
cmiiEnv := ""
appName := ""
fromTag := ""
newTag := ""
for _, s := range split {
if s != "" {
if index == 1 {
cmiiEnv = s
} else if index == 2 {
appName = s
} else if index == 3 {
fromTag = s
} else if index == 4 {
newTag = s
}
index++
}
}
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
return rollback
}
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
// must filter
allInterface = FilterAllCmiiAppSoft(allInterface)
filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
firstCol := 0
secondCol := 0
thirdCol := 0
fourthCol := 0
for _, deploymentInterface := range allInterface {
firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol)
secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol)
thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol)
fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol)
}
firstCol += 2
secondCol += 2
secondCol += firstCol
thirdCol += 2
thirdCol += secondCol
fourthCol += 2
fourthCol += thirdCol
for _, deploymentInterface := range allInterface {
if deploymentInterface.GitBranch == "" {
branch, commit := GetDeploymentGitInfoFromInnerEnv(deploymentInterface.Namespace, deploymentInterface.Name)
deploymentInterface.GitBranch = branch
deploymentInterface.GitCommit = commit
}
content := executor.BasicWordSpaceCompletion(deploymentInterface.Name, firstCol)
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol)
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol)
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol)
content += "\n"
if !executor.BasicAppendContentToFile(content, filePath) {
log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content)
return false
}
}
return true
}
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[string]string) {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface = FilterAllCmiiAppSoft(allInterface)
backendMap = make(map[string]string, len(allInterface))
frontendMap = make(map[string]string, len(allInterface))
for _, deploymentInterface := range allInterface {
if strings.Contains(deploymentInterface.Name, "platform") {
frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
} else {
backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
}
}
return backendMap, frontendMap
}
func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
for _, c := range source {
_, ok := CmiiBackendAppMap[c.ContainerName]
if !ok {
_, ok = CmiiFrontendAppMap[c.ContainerName]
if !ok {
log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod ", c.ContainerName)
continue
}
}
result = append(result, c)
}
return result
}
func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
for _, c := range source {
if strings.Contains(c.ContainerName, "redis") {
continue
}
if strings.Contains(c.ContainerName, "emqxs") {
continue
}
if strings.Contains(c.ContainerName, "rabbitmq") {
continue
}
if strings.Contains(c.ContainerName, "nacos") {
continue
}
if strings.Contains(c.ContainerName, "oss") {
continue
}
if strings.Contains(c.ContainerName, "minio") {
continue
}
if strings.HasPrefix(c.ContainerName, "nfs") {
continue
}
if strings.HasPrefix(c.ContainerName, "operator") {
continue
}
if strings.HasPrefix(c.ContainerName, "proxy") {
continue
}
result = append(result, c)
}
return result
}
func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) {
for _, c := range podList {
_, ok := CmiiBackendAppMap[c.ContainerName]
if !ok {
_, ok = CmiiFrontendAppMap[c.ContainerName]
if !ok {
log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod ", c.ContainerName)
continue
}
}
result = append(result, c)
}
return result
}
func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) {
for _, c := range podList {
if strings.Contains(c.ContainerName, "redis") {
continue
}
if strings.Contains(c.ContainerName, "emqxs") {
continue
}
if strings.Contains(c.ContainerName, "rabbitmq") {
continue
}
if strings.Contains(c.ContainerName, "nacos") {
continue
}
if strings.Contains(c.ContainerName, "oss") {
continue
}
if strings.Contains(c.ContainerName, "minio") {
continue
}
if strings.HasPrefix(c.ContainerName, "nfs") {
continue
}
if strings.HasPrefix(c.ContainerName, "operator") {
continue
}
if strings.HasPrefix(c.ContainerName, "proxy") {
continue
}
if strings.HasPrefix(c.ContainerName, "cleanlog") {
continue
}
result = append(result, c)
}
return result
}
func FilterAllCmiiNodeSoft(nodeList []CmiiNodeInterface) (result []CmiiNodeInterface) {
for _, nodeInterface := range nodeList {
if strings.HasPrefix(nodeInterface.Name, "ai") {
continue
}
if strings.HasPrefix(nodeInterface.Name, "35") {
continue
}
result = append(result, nodeInterface)
}
return result
}
func AppNameBelongsToCmiiImage(appName string) bool {
_, ok := CmiiBackendAppMap[appName]
if !ok {
_, ok = CmiiFrontendAppMap[appName]
if !ok {
log.WarnF("[AppNameBelongsToCmiiImage] - [%s] not cmii app ", appName)
return false
} else {
return true
}
} else {
return true
}
}