[agent-operator] - update dltu part

This commit is contained in:
zeaslity
2024-04-26 17:51:11 +08:00
parent 318a5a8026
commit 021690f5c4
12 changed files with 338 additions and 197 deletions

View File

@@ -8,7 +8,7 @@ import (
"wdd.io/agent-common/utils"
)
var CmiiOperator = CmiiK8sOperator{}
var DefaultCmiiOperator = CmiiK8sOperator{}
// var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt"
var updateLogPath = "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log/cmii-update-log.txt"
@@ -19,7 +19,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in
//podInterface := CmiiPodInterface{}
// get all pods
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
podAll := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
// restart map
restartMap := make(map[string]int32, len(podAll))
@@ -47,7 +47,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in
// find deployment convert to interface
for key, value := range restartMap {
// container Name must equals deployment name
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deployment := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deployment != nil {
// deployment exists
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
@@ -63,7 +63,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
// get all deployments
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
// filter
@@ -80,13 +80,13 @@ func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (depl
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
if !podInterface.PodStatus {
// unhealthy pod
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
deploymentInterface := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
if deploymentInterface != nil {
deploymentList = append(deploymentList, *deploymentInterface)
}
@@ -98,10 +98,10 @@ func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentIn
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
// dev-cluster
devNodeList := CmiiOperator.NodeAllInterface("dev")
devNodeList := DefaultCmiiOperator.NodeAllInterface("dev")
// core-cluster
coreNodeList := CmiiOperator.NodeAllInterface("uat")
coreNodeList := DefaultCmiiOperator.NodeAllInterface("uat")
// append
coreNodeList = append(coreNodeList, devNodeList...)
@@ -129,7 +129,7 @@ func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
@@ -145,7 +145,7 @@ func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) {
// get app
podList := CmiiOperator.PodByAppName(cmiiEnv, appName)
podList := DefaultCmiiOperator.PodByAppName(cmiiEnv, appName)
// get pod
if podList == nil || len(podList) == 0 {
@@ -154,7 +154,7 @@ func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCo
}
// exec env
stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
stdout, stderr := DefaultCmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
errLog := stderr.String()
if errLog != "" {
@@ -182,7 +182,7 @@ func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCo
func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
cmiiPodInterfaces := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
for _, podInterface := range cmiiPodInterfaces {
for key := range CmiiMiddlewareNameMap {
@@ -219,9 +219,9 @@ func ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv string, nameReplicasMap map
// 遍历
for appName, replica := range nameReplicasMap {
exists := CmiiOperator.DeploymentExist(cmiiEnv, appName)
exists := DefaultCmiiOperator.DeploymentExist(cmiiEnv, appName)
if exists != nil {
scale := CmiiOperator.DeploymentScale(cmiiEnv, appName, replica)
scale := DefaultCmiiOperator.DeploymentScale(cmiiEnv, appName, replica)
if !scale {
errorUpdateMap[appName] = replica
}
@@ -238,12 +238,12 @@ func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
result := true
for _, deployment := range deploymentList {
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
result = DefaultCmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
if !result {
return result
}
time.Sleep(time.Second)
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
result = DefaultCmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
if !result {
return result
}
@@ -254,11 +254,11 @@ func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
func RestartCmiiBackendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
if AppNameBelongsToCmiiImage(deploymentInterface.Name) {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
if !DefaultCmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
@@ -267,17 +267,17 @@ func RestartCmiiBackendDeployment(cmiiEnv string) {
}
}
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", DefaultCmiiOperator.CurrentNamespace)
}
func RestartCmiiFrontendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
if ok {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
if !DefaultCmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
@@ -285,38 +285,39 @@ func RestartCmiiFrontendDeployment(cmiiEnv string) {
}
}
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", DefaultCmiiOperator.CurrentNamespace)
}
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) (updateOK bool, oldImageTag, newImageTag string) {
cmiiDeploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
cmiiDeploymentInterface := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if cmiiDeploymentInterface == nil {
return false
return updateOK, oldImageTag, newImageTag
}
// check if need to update
if cmiiDeploymentInterface.ImageTag == newTag {
oldImageTag = cmiiDeploymentInterface.ImageTag
if oldImageTag == newTag {
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
// restart
if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
return true
if DefaultCmiiOperator.DeploymentRestart(cmiiEnv, appName) {
return true, oldImageTag, oldImageTag
} else {
return false
return false, oldImageTag, oldImageTag
}
}
content := utils.WordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 35)
content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.Name, 75)
content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
content = utils.WordSpaceCompletion(content+oldImageTag, 105)
content = content + newTag + "\n"
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
// update
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
tag := DefaultCmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
if !tag {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
return false
return false, oldImageTag, newImageTag
}
// append log
@@ -324,15 +325,13 @@ func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
// re-get from env
time.Sleep(time.Second)
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if deploy == nil {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
return false
return false, oldImageTag, newImageTag
}
// log
//log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] update tag [%s] success ! ", deploy.Image, content)
return true
return true, oldImageTag, deploy.ImageTag
}
func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
@@ -340,12 +339,12 @@ func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]stri
result = make(map[string]string, len(nameTagMap))
for appName, newTag := range nameTagMap {
if AppNameBelongsToCmiiImage(appName) {
if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
ok, oldImageTag, newImageTag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
if ok {
log.InfoF("[UpdateCmiiImageTagFromNameTagMap] - %s %s to %s", cmiiEnv, appName, newTag)
result[appName] = newTag
result[appName] = newImageTag
} else {
result[appName] = "false"
result[appName] = oldImageTag
}
}
}
@@ -396,23 +395,23 @@ func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
}
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
ok, _, _ := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
return rollback
return ok
}
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
// must filter
allInterface = FilterAllCmiiAppSoft(allInterface)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", DefaultCmiiOperator.CurrentNamespace, filePath)
firstCol := 0
secondCol := 0
@@ -462,7 +461,7 @@ func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
// BackupAllCmiiDeploymentToMap 备份DEMO环境全部的镜像名称为Map 包括SRS 前端 后端
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsMap map[string]string) {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface = FilterAllCmiiAppSoft(allInterface)
backendMap = make(map[string]string, len(allInterface))
@@ -481,7 +480,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM
for key, value := range CmiiSrsAppMap {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
split := strings.Split(imageName, ":")
@@ -492,7 +491,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM
}
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
split := strings.Split(imageName, ":")
@@ -531,9 +530,9 @@ func BackupAllCmiiDeploymentToList(cmiiEnv string, completePrefix bool) (allCmii
func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
CmiiOperator.changeOperatorEnv(cmiiEnv)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
DefaultCmiiOperator.changeOperatorEnv(cmiiEnv)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
only := make(map[string]string, 150)
// front
@@ -541,7 +540,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
for key, value := range CmiiFrontendAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
@@ -552,7 +551,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
for key, value := range CmiiBackendAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
@@ -565,7 +564,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
for key, value := range CmiiGISAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
@@ -579,13 +578,13 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
if !ok {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if app != nil {
only[key] = value
utils.AppendContentToFile(app.Image+"\n", filePath)
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
if app != nil {
only[key] = value
for _, imageName := range app.ContainerImageMap {