452 lines
13 KiB
Go
452 lines
13 KiB
Go
package k8s_exec
|
||
|
||
import (
|
||
"agent-go/executor"
|
||
"agent-go/utils"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
var CmiiOperator = CmiiK8sOperator{}
|
||
var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\agent-go\\k8s_exec\\log\\cmii-update-log.txt"
|
||
|
||
// FindAppNotHealthyOrRestartCountGreaterThanN 重启次数大于N的所有Deployment
|
||
func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount int32) []CmiiDeploymentInterface {
|
||
|
||
//podInterface := CmiiPodInterface{}
|
||
|
||
// get all pods
|
||
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
// restart map
|
||
restartMap := make(map[string]int32, len(podAll))
|
||
|
||
// restart count
|
||
for _, pod := range podAll {
|
||
if pod.RestartCount > restartCount {
|
||
restart, exists := restartMap[pod.ContainerName]
|
||
if exists {
|
||
restartMap[pod.ContainerName] = utils.MaxInt32(pod.RestartCount, restart)
|
||
} else {
|
||
restartMap[pod.ContainerName] = pod.RestartCount
|
||
}
|
||
}
|
||
|
||
// unhealthy
|
||
if !pod.PodStatus {
|
||
restartMap[pod.ContainerName] = pod.RestartCount
|
||
}
|
||
}
|
||
|
||
result := make([]CmiiDeploymentInterface, len(restartMap))
|
||
index := 0
|
||
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart map is => %v", restartMap)
|
||
// find deployment convert to interface
|
||
for key, value := range restartMap {
|
||
// container Name must equals deployment name
|
||
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
|
||
if deployment != nil {
|
||
// deployment exists
|
||
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
|
||
|
||
result[index] = *deployment
|
||
index++
|
||
}
|
||
}
|
||
|
||
return result[:index]
|
||
}
|
||
|
||
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
|
||
|
||
// get all deployments
|
||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
|
||
|
||
// filter
|
||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||
if deploymentInterface.Replicas <= replicasMin {
|
||
deploymentList = append(deploymentList, deploymentInterface)
|
||
}
|
||
}
|
||
|
||
// convert
|
||
return deploymentList
|
||
}
|
||
|
||
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
|
||
|
||
// all unhealthy pods
|
||
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
// find the deployments
|
||
for _, podInterface := range allInterface {
|
||
if !podInterface.PodStatus {
|
||
// unhealthy pod
|
||
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
|
||
if deploymentInterface != nil {
|
||
deploymentList = append(deploymentList, *deploymentInterface)
|
||
}
|
||
}
|
||
}
|
||
|
||
return deploymentList
|
||
}
|
||
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
|
||
|
||
// dev-cluster
|
||
devNodeList := CmiiOperator.NodeAllInterface("dev")
|
||
|
||
// core-cluster
|
||
coreNodeList := CmiiOperator.NodeAllInterface("uat")
|
||
|
||
// append
|
||
coreNodeList = append(coreNodeList, devNodeList...)
|
||
|
||
// filter
|
||
for _, node := range coreNodeList {
|
||
if node.Unschedulable {
|
||
nodeList = append(nodeList, node)
|
||
continue
|
||
}
|
||
|
||
if !node.NodeStatus {
|
||
nodeList = append(nodeList, node)
|
||
continue
|
||
}
|
||
|
||
if node.MemoryPressure || node.PIDPressure || node.NetworkUnavailable || node.DiskPressure {
|
||
nodeList = append(nodeList, node)
|
||
continue
|
||
}
|
||
}
|
||
|
||
return nodeList
|
||
}
|
||
|
||
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
|
||
// all unhealthy pods
|
||
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
|
||
|
||
// find the deployments
|
||
for _, podInterface := range allInterface {
|
||
if !podInterface.PodStatus {
|
||
// unhealthy pod
|
||
podList = append(podList, podInterface)
|
||
}
|
||
}
|
||
|
||
return podList
|
||
}
|
||
|
||
func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
|
||
|
||
result := true
|
||
|
||
for _, deployment := range deploymentList {
|
||
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
|
||
if !result {
|
||
return result
|
||
}
|
||
time.Sleep(time.Second)
|
||
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
|
||
if !result {
|
||
return result
|
||
}
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func RestartCmiiBackendDeployment(cmiiEnv string) {
|
||
|
||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
|
||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||
_, ok := CmiiBackendAppMap[deploymentInterface.Name]
|
||
if ok {
|
||
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
|
||
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
} else {
|
||
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
}
|
||
}
|
||
}
|
||
|
||
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
|
||
}
|
||
|
||
func RestartCmiiFrontendDeployment(cmiiEnv string) {
|
||
|
||
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
|
||
for _, deploymentInterface := range cmiiDeploymentInterfaces {
|
||
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
|
||
if ok {
|
||
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
|
||
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
} else {
|
||
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
|
||
}
|
||
}
|
||
}
|
||
|
||
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
|
||
}
|
||
|
||
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
|
||
|
||
deployment := CmiiOperator.DeploymentExist(cmiiEnv, appName)
|
||
if deployment == nil {
|
||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] not exists !", cmiiEnv, appName)
|
||
return false
|
||
}
|
||
|
||
deploymentInterface := CmiiDeploymentInterface{}
|
||
cmiiDeploymentInterface := deploymentInterface.Convert(*deployment)
|
||
// check if need to update
|
||
if cmiiDeploymentInterface.ImageTag == newTag {
|
||
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
|
||
return true
|
||
}
|
||
|
||
content := executor.BasicWordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 45)
|
||
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.Name, 85)
|
||
content = executor.BasicWordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
|
||
content = content + newTag + "\n"
|
||
|
||
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
|
||
|
||
// update
|
||
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
|
||
if !tag {
|
||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
|
||
return false
|
||
}
|
||
|
||
// append log
|
||
executor.BasicAppendContentToFile(content, updateLogPath)
|
||
|
||
// re-get from env
|
||
time.Sleep(time.Second)
|
||
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
|
||
if deploy == nil {
|
||
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
|
||
return false
|
||
}
|
||
|
||
// log
|
||
log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] \n update tag [%s] success ! ", deploy.Image, content)
|
||
return true
|
||
}
|
||
|
||
func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
|
||
|
||
if !executor.BasicFindContentInFile(updateLog, updateLogPath) {
|
||
log.ErrorF("[RollBackCmiiDeploymentFromUpdateLog] - [%s] no this update log ! use update instead ! => ", updateLog)
|
||
return false
|
||
}
|
||
|
||
split := strings.Split(updateLog, " ")
|
||
index := 0
|
||
cmiiEnv := ""
|
||
appName := ""
|
||
fromTag := ""
|
||
newTag := ""
|
||
for _, s := range split {
|
||
if s != "" {
|
||
if index == 1 {
|
||
cmiiEnv = s
|
||
} else if index == 2 {
|
||
appName = s
|
||
} else if index == 3 {
|
||
fromTag = s
|
||
} else if index == 4 {
|
||
newTag = s
|
||
}
|
||
index++
|
||
}
|
||
}
|
||
|
||
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
|
||
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
|
||
|
||
return rollback
|
||
}
|
||
|
||
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
|
||
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
|
||
|
||
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
|
||
filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\agent-go\\k8s_exec\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
|
||
|
||
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
|
||
|
||
firstCol := 0
|
||
secondCol := 0
|
||
thirdCol := 0
|
||
fourthCol := 0
|
||
|
||
for _, deploymentInterface := range allInterface {
|
||
firstCol = utils.MaxInt(len(deploymentInterface.Name), firstCol)
|
||
secondCol = utils.MaxInt(len(deploymentInterface.ImageTag), secondCol)
|
||
thirdCol = utils.MaxInt(len(deploymentInterface.GitBranch), thirdCol)
|
||
fourthCol = utils.MaxInt(len(deploymentInterface.GitCommit), fourthCol)
|
||
}
|
||
|
||
firstCol += 2
|
||
secondCol += 2
|
||
secondCol += firstCol
|
||
thirdCol += 2
|
||
thirdCol += secondCol
|
||
fourthCol += 2
|
||
fourthCol += thirdCol
|
||
|
||
for _, deploymentInterface := range allInterface {
|
||
|
||
content := executor.BasicWordSpaceCompletion(deploymentInterface.Name, firstCol)
|
||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.ImageTag, secondCol)
|
||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitBranch, thirdCol)
|
||
content = executor.BasicWordSpaceCompletion(content+deploymentInterface.GitCommit, fourthCol)
|
||
content += "\n"
|
||
|
||
if !executor.BasicAppendContentToFile(content, filePath) {
|
||
log.ErrorF("[BackupAllDeploymentFromEnv] - write to file %s error with contend %s", filePath, content)
|
||
return false
|
||
}
|
||
|
||
}
|
||
|
||
return true
|
||
}
|
||
|
||
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap map[string]string) {
|
||
|
||
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
|
||
allInterface = FilterAllCmiiAppSoft(allInterface)
|
||
|
||
backendMap = make(map[string]string, len(allInterface))
|
||
frontendMap = make(map[string]string, len(allInterface))
|
||
|
||
for _, deploymentInterface := range allInterface {
|
||
if strings.Contains(deploymentInterface.Name, "platform") {
|
||
frontendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
|
||
} else {
|
||
backendMap[deploymentInterface.Name] = deploymentInterface.ImageTag
|
||
}
|
||
}
|
||
|
||
return backendMap, frontendMap
|
||
}
|
||
|
||
func FilterAllCmiiAppStrict(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
|
||
|
||
for _, c := range source {
|
||
_, ok := CmiiBackendAppMap[c.ContainerName]
|
||
if !ok {
|
||
_, ok = CmiiFrontendAppMap[c.ContainerName]
|
||
if !ok {
|
||
log.WarnF("[FilterAllCmiiAppStrict] - [%s] not cmii pod ！", c.ContainerName)
|
||
continue
|
||
}
|
||
}
|
||
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiAppSoft(source []CmiiDeploymentInterface) (result []CmiiDeploymentInterface) {
|
||
|
||
for _, c := range source {
|
||
if strings.Contains(c.ContainerName, "redis") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "emqxs") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "rabbitmq") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "nacos") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "oss") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "minio") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "nfs") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "operator") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "proxy") {
|
||
continue
|
||
}
|
||
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiPodStrict(podList []CmiiPodInterface) (result []CmiiPodInterface) {
|
||
for _, c := range podList {
|
||
_, ok := CmiiBackendAppMap[c.ContainerName]
|
||
if !ok {
|
||
_, ok = CmiiFrontendAppMap[c.ContainerName]
|
||
if !ok {
|
||
log.WarnF("[FilterAllCmiiPodStrict] - [%s] not cmii pod ！", c.ContainerName)
|
||
continue
|
||
}
|
||
}
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|
||
|
||
func FilterAllCmiiPodSoft(podList []CmiiPodInterface) (result []CmiiPodInterface) {
|
||
for _, c := range podList {
|
||
if strings.Contains(c.ContainerName, "redis") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "emqxs") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "rabbitmq") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "nacos") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "oss") {
|
||
continue
|
||
}
|
||
if strings.Contains(c.ContainerName, "minio") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "nfs") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "operator") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "proxy") {
|
||
continue
|
||
}
|
||
if strings.HasPrefix(c.ContainerName, "cleanlog") {
|
||
continue
|
||
}
|
||
|
||
result = append(result, c)
|
||
}
|
||
|
||
return result
|
||
}
|