[ CMII ] [ Operator ] - 抽离为单独的项目
This commit is contained in:
571
cmii_operator/CmiiK8sOperator.go
Normal file
571
cmii_operator/CmiiK8sOperator.go
Normal file
@@ -0,0 +1,571 @@
|
||||
package cmii_operator
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"strings"
|
||||
"time"
|
||||
"wdd.io/agent-go/executor"
|
||||
"wdd.io/agent-go/utils"
|
||||
)
|
||||
|
||||
var CmiiOperator = CmiiK8sOperator{}
|
||||
var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt"
|
||||
|
||||
// FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment
|
||||
func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface {
|
||||
|
||||
//podInterface := CmiiPodInterface{}
|
||||
|
||||
// get all pods
|
||||
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
|
||||
|
||||
// restart map
|
||||
restartMap := make(map[string]int32, len(podAll))
|
||||
|
||||
// restart count
|
||||
for _, pod := range podAll {
|
||||
if pod.RestartCount > restartCount {
|
||||
restart, exists := restartMap[pod.ContainerName]
|
||||
if exists {
|
||||
restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart)
|
||||
} else {
|
||||
restartMap[pod.ContainerName] = pod.RestartCount
|
||||
}
|
||||
}
|
||||
|
||||
// unhealthy
|
||||
if !pod.PodStatus {
|
||||
restartMap[pod.ContainerName] = pod.RestartCount
|
||||
}
|
||||
}
|
||||
|
||||
result := make([]CmiiDeploymentInterface, len(restartMap))
|
||||
index := 0
|
||||
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap)
|
||||
// find deployment convert to interface
|
||||
for key, value := range restartMap {
|
||||
// container Name must equals deployment name
|
||||
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
|
||||
if deployment != nil {
|
||||
// deployment exists
|
||||
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
|
||||
|
||||
result[index] = *deployment
|
||||
index++
|
||||
}
|
||||
}
|
||||
|
||||
return result[:index]
|
||||
}
|
||||
|
||||
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
|
||||
|
||||
// get all deployments
|
||||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||||
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
|
||||
|
||||
// filter
|
||||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||||
if deploymentInterface.Replicas <= replicasMin {
|
||||
deploymentList = append(deploymentList, deploymentInterface)
|
||||
}
|
||||
}
|
||||
|
||||
// convert
|
||||
return deploymentList
|
||||
}
|
||||
|
||||
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
|
||||
|
||||
// all unhealthy pods
|
||||
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
|
||||
|
||||
// find the deployments
|
||||
for _, podInterface := range allInterface {
|
||||
if !podInterface.PodStatus {
|
||||
// unhealthy pod
|
||||
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
|
||||
if deploymentInterface != nil {
|
||||
deploymentList = append(deploymentList, *deploymentInterface)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return deploymentList
|
||||
}
|
||||
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
|
||||
|
||||
// dev-cluster
|
||||
devNodeList := CmiiOperator.NodeAllInterface("dev")
|
||||
|
||||
// core-cluster
|
||||
coreNodeList := CmiiOperator.NodeAllInterface("uat")
|
||||
|
||||
// append
|
||||
coreNodeList = append(coreNodeList, devNodeList...)
|
||||
|
||||
// filter
|
||||
for _, node := range coreNodeList {
|
||||
if node.Unschedulable {
|
||||
nodeList = append(nodeList, node)
|
||||
continue
|
||||
}
|
||||
|
||||
if !node.NodeStatus {
|
||||
nodeList = append(nodeList, node)
|
||||
continue
|
||||
}
|
||||
|
||||
if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure {
|
||||
nodeList = append(nodeList, node)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return nodeList
|
||||
}
|
||||
|
||||
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
|
||||
// all unhealthy pods
|
||||
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
|
||||
|
||||
// find the deployments
|
||||
for _, podInterface := range allInterface {
|
||||
if !podInterface.PodStatus {
|
||||
// unhealthy pod
|
||||
podList = append(podList, podInterface)
|
||||
}
|
||||
}
|
||||
|
||||
return podList
|
||||
}
|
||||
|
||||
func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) {
|
||||
|
||||
// get app
|
||||
podList := CmiiOperator.PodByAppName(cmiiEnv, appName)
|
||||
|
||||
// get pod
|
||||
if podList == nil || len(podList) == 0 {
|
||||
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - get app pod error [%s] [%s]", cmiiEnv, appName)
|
||||
return "", ""
|
||||
}
|
||||
|
||||
// exec env
|
||||
stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
|
||||
|
||||
errLog := stderr.String()
|
||||
if errLog != "" {
|
||||
log.ErrorF("[GetDeploymentGitInfoFromInnerEnv] - pod Exec error %s", errLog)
|
||||
return "", ""
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
// Get the current line
|
||||
line := scanner.Text()
|
||||
|
||||
// Check if the current line contains the search term
|
||||
if strings.HasPrefix(line, "GIT_BRANCH") {
|
||||
gitBranch = strings.Split(line, "=")[1]
|
||||
}
|
||||
if strings.HasPrefix(line, "GIT_COMMIT") {
|
||||
gitCommit = strings.Split(line, "=")[1]
|
||||
}
|
||||
}
|
||||
|
||||
// get out git info
|
||||
return gitBranch, gitCommit
|
||||
}
|
||||
|
||||
func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
|
||||
|
||||
cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
|
||||
|
||||
for _, podInterface := range cmiiPodInterfaces {
|
||||
for key := range CmiiMiddlewareNameMap {
|
||||
if strings.Contains(podInterface.Name, key) {
|
||||
podList = append(podList, podInterface)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return podList
|
||||
}
|
||||
|
||||
func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
|
||||
|
||||
result := true
|
||||
|
||||
for _, deployment := range deploymentList {
|
||||
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
|
||||
if !result {
|
||||
return result
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
|
||||
if !result {
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func RestartCmiiBackendDeployment(cmiiEnv string) {
|
||||
|
||||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||||
|
||||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||||
_, ok := CmiiBackendAppMap[deploymentInterface.Name]
|
||||
if ok {
|
||||
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
|
||||
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||||
} else {
|
||||
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
|
||||
}
|
||||
|
||||
func RestartCmiiFrontendDeployment(cmiiEnv string) {
|
||||
|
||||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||||
|
||||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||||
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
|
||||
if ok {
|
||||
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
|
||||
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||||
} else {
|
||||
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
|
||||
}
|
||||
|
||||
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
|
||||
|
||||
deployment := CmiiOperator.DeploymentExist(cmiiEnv, appName)
|
||||
if deployment == nil {
|
||||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] not exists !", cmiiEnv, appName)
|
||||
return false
|
||||
}
|
||||
|
||||
deploymentInterface := CmiiDeploymentInterface{}
|
||||
cmiiDeploymentInterface := deploymentInterface.Convert(*deployment)
|
||||
// check if need to update
|
||||
if cmiiDeploymentInterface.ImageTag == newTag {
|
||||
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
|
||||
|
||||
// restart
|
||||
if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
content := executor.BasicWordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 45)
|
||||
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.Name, 85)
|
||||
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
|
||||
content = content + newTag + "\n"
|
||||
|
||||
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
|
||||
|
||||
// update
|
||||
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
|
||||
if !tag {
|
||||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
|
||||
return false
|
||||
}
|
||||
|
||||
// append log
|
||||
executor.BasicAppendContentToFile(content, updateLogPath)
|
||||
|
||||
// re-get from env
|
||||
time.Sleep(time.Second)
|
||||
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
|
||||
if deploy == nil {
|
||||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
|
||||
return false
|
||||
}
|
||||
|
||||
// log
|
||||
log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] \n update tag [%s] success ! ", deploy.Image, content)
|
||||
return true
|
||||
}
|
||||
|
||||
func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
|
||||
|
||||
result = make(map[string]string, len(nameTagMap))
|
||||
for appName, newTag := range nameTagMap {
|
||||
if AppNameBelongsToCmiiImage(appName) {
|
||||
if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
|
||||
result[appName] = newTag
|
||||
} else {
|
||||
result[appName] = "false"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
|
||||
|
||||
if !executor.BasicFindContentInFile(updateLog, updateLogPath) {
|
||||
log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog)
|
||||
return false
|
||||
}
|
||||
|
||||
split := strings.Split(updateLog, " ")
|
||||
index := 0
|
||||
cmiiEnv := ""
|
||||
appName := ""
|
||||
fromTag := ""
|
||||
newTag := ""
|
||||
for _, s := range split {
|
||||
if s != "" {
|
||||
if index == 1 {
|
||||
cmiiEnv = s
|
||||
} else if index == 2 {
|
||||
appName = s
|
||||
} else if index == 3 {
|
||||
fromTag = s
|
||||
} else if index == 4 {
|
||||
newTag = s
|
||||
}
|
||||
index++
|
||||
}
|
||||
}
|
||||
|
||||
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
|
||||
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
|
||||
|
||||
return rollback
|
||||
}
|
||||
|
||||
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
|
||||
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
|
||||
|
||||
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||||
|
||||
// must filter
|
||||
allInterface = FilterAllCmiiAppSoft(allInterface)
|
||||
|
||||
filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
|
||||
|
||||
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
|
||||
|
||||
firstCol := 0
|
||||
secondCol := 0
|
||||
thirdCol := 0
|
||||
fourthCol := 0
|
||||
|
||||
for _, deploymentInterface := range allInterface {
|
||||
|
||||
firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol)
|
||||
secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol)
|
||||
thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol)
|
||||
fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol)
|
||||
}
|
||||
|
||||
firstCol += 2
|
||||
secondCol += 2
|
||||
secondCol += firstCol
|
||||
thirdCol += 2
|
||||
thirdCol += secondCol
|
||||
fourthCol += 2
|
||||
fourthCol += thirdCol
|
||||
|
||||
for _, deploymentInterface := range allInterface {
|
||||
|
||||
if deploymentInterface.GitBranch == "" {
|
||||
branch, commit := GetDeploymentGitInfoFromInnerEnv(deploymentInterface.Namespace, deploymentInterface.Name)
|
||||
deploymentInterface.GitBranch = branch
|
||||
deploymentInterface.GitCommit = commit
|
||||
}
|
||||
|
||||
content := executor.BasicWordSpaceCompletion(deploymentInterface.Name, firstCol)
|
||||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol)
|
||||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol)
|
||||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol)
|
||||
content += "\n"
|
||||
|
||||
if !executor.BasicAppendContentToFile(content, filePath) {
|
||||
log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content)
|
||||
return false
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[string]string) {
|
||||
|
||||
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||||
allInterface = FilterAllCmiiAppSoft(allInterface)
|
||||
|
||||
backendMap = make(map[string]string, len(allInterface))
|
||||
frontendMap = make(map[string]string, len(allInterface))
|
||||
|
||||
for _, deploymentInterface := range allInterface {
|
||||
if strings.Contains(deploymentInterface.Name, "platform") {
|
||||
frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
|
||||
} else {
|
||||
backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
|
||||
}
|
||||
}
|
||||
|
||||
return backendMap, frontendMap
|
||||
}
|
||||
|
||||
func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
|
||||
|
||||
for _, c := range source {
|
||||
_, ok := CmiiBackendAppMap[c.ContainerName]
|
||||
if !ok {
|
||||
_, ok = CmiiFrontendAppMap[c.ContainerName]
|
||||
if !ok {
|
||||
log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod !", c.ContainerName)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, c)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
|
||||
|
||||
for _, c := range source {
|
||||
if strings.Contains(c.ContainerName, "redis") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "emqxs") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "rabbitmq") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "nacos") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "oss") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "minio") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "nfs") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "operator") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "proxy") {
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, c)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) {
|
||||
for _, c := range podList {
|
||||
_, ok := CmiiBackendAppMap[c.ContainerName]
|
||||
if !ok {
|
||||
_, ok = CmiiFrontendAppMap[c.ContainerName]
|
||||
if !ok {
|
||||
log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod !", c.ContainerName)
|
||||
continue
|
||||
}
|
||||
}
|
||||
result = append(result, c)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) {
|
||||
for _, c := range podList {
|
||||
if strings.Contains(c.ContainerName, "redis") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "emqxs") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "rabbitmq") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "nacos") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "oss") {
|
||||
continue
|
||||
}
|
||||
if strings.Contains(c.ContainerName, "minio") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "nfs") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "operator") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "proxy") {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(c.ContainerName, "cleanlog") {
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, c)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func FilterAllCmiiNodeSoft(nodeList []CmiiNodeInterface) (result []CmiiNodeInterface) {
|
||||
|
||||
for _, nodeInterface := range nodeList {
|
||||
|
||||
if strings.HasPrefix(nodeInterface.Name, "ai") {
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.HasPrefix(nodeInterface.Name, "35") {
|
||||
continue
|
||||
}
|
||||
|
||||
result = append(result, nodeInterface)
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
func AppNameBelongsToCmiiImage(appName string) bool {
|
||||
_, ok := CmiiBackendAppMap[appName]
|
||||
if !ok {
|
||||
_, ok = CmiiFrontendAppMap[appName]
|
||||
if !ok {
|
||||
log.WarnF("[AppNameBelongsToCmiiImage] - [%s] not cmii app !", appName)
|
||||
return false
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user