diff --git a/agent-common/utils/FileUtils.go b/agent-common/utils/FileUtils.go index ae6553e..8e0f794 100644 --- a/agent-common/utils/FileUtils.go +++ b/agent-common/utils/FileUtils.go @@ -120,6 +120,15 @@ func WordSpaceCompletion(source string, totalLength int) string { return source } +// IsFileOrDir 如果是目录则返回true,是文件则返回false +func IsFileOrDir(path string) bool { + info, err := os.Stat(path) + if err != nil { + return false + } + return info.IsDir() +} + // FileExists 文件存在返回true,不存在返回false,如果文件是一个目录,也返回false func FileExists(fileFullPath string) bool { _, err := os.Stat(fileFullPath) diff --git a/agent-go/a_status/Disk.go b/agent-go/a_status/Disk.go index 6d56701..9ea8f12 100644 --- a/agent-go/a_status/Disk.go +++ b/agent-go/a_status/Disk.go @@ -3,6 +3,8 @@ package a_status import ( "fmt" "github.com/shirou/gopsutil/v3/disk" + "os" + "path/filepath" "regexp" "time" ) @@ -115,3 +117,45 @@ func MatchNeededDisk(deviceName string) bool { return true } + +func dirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} + +//func DiskUsages() { +// +// +// root := "/" +// maxDepth := 3 +// +// fmt.Println("Scanning directories...") +// err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { +// if err != nil { +// return err +// } +// if info.IsDir() && filepath.DirCount(path) <= maxDepth { +// size, err := dirSize(path) +// if err != nil { +// fmt.Printf("Error: %v\n", err) +// return nil +// } +// fmt.Printf("%s: %d bytes\n", path, size) +// } +// return nil +// }) +// if err != nil { +// fmt.Println(err) +// } +// +// +//} diff --git a/agent-go/tmp/test.go b/agent-go/tmp/test.go index e8898c6..6fa9755 100644 --- a/agent-go/tmp/test.go +++ b/agent-go/tmp/test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "regexp" "strings" ) @@ -25,6 +26,17 @@ func splitTest() { func main() { - splitTest() + inputList := []string{ + "4.1.6-xxx", + "5.1.0", + "3.2.0-0123-123", + } + r, _ := regexp.Compile(`\d+.+\d+.+\d+`) + for _, input := range inputList { + matches := r.FindAllString(input, -1) + for _, match := range matches { + fmt.Println(match) // Output: 4.1.6 5.1.0 3.2.0 + } + } } diff --git a/agent-operator/CmiiK8sOperator.go b/agent-operator/CmiiK8sOperator.go index cb12a82..7a1f901 100644 --- a/agent-operator/CmiiK8sOperator.go +++ b/agent-operator/CmiiK8sOperator.go @@ -8,7 +8,7 @@ import ( "wdd.io/agent-common/utils" ) -var CmiiOperator = CmiiK8sOperator{} +var DefaultCmiiOperator = CmiiK8sOperator{} // var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt" var updateLogPath = "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log/cmii-update-log.txt" @@ -19,7 +19,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in //podInterface := CmiiPodInterface{} // get all pods - podAll := CmiiOperator.PodAllInterface(cmiiEnv) + podAll := DefaultCmiiOperator.PodAllInterface(cmiiEnv) // restart map restartMap := make(map[string]int32, len(podAll)) @@ -47,7 +47,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in // find deployment convert to interface for key, value := range restartMap { // container Name must equals deployment name - deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) + deployment := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deployment != nil { // deployment exists log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value) @@ -63,7 +63,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) { // get all deployments - cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) + cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv) cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces) // filter @@ -80,13 +80,13 @@ func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (depl func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) { // all unhealthy pods - allInterface := CmiiOperator.PodAllInterface(cmiiEnv) + allInterface := DefaultCmiiOperator.PodAllInterface(cmiiEnv) // find the deployments for _, podInterface := range allInterface { if !podInterface.PodStatus { // unhealthy pod - deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName) + deploymentInterface := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName) if deploymentInterface != nil { deploymentList = append(deploymentList, *deploymentInterface) } @@ -98,10 +98,10 @@ func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentIn func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) { // dev-cluster - devNodeList := CmiiOperator.NodeAllInterface("dev") + devNodeList := DefaultCmiiOperator.NodeAllInterface("dev") // core-cluster - coreNodeList := CmiiOperator.NodeAllInterface("uat") + coreNodeList := DefaultCmiiOperator.NodeAllInterface("uat") // append coreNodeList = append(coreNodeList, devNodeList...) @@ -129,7 +129,7 @@ func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) { func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) { // all unhealthy pods - allInterface := CmiiOperator.PodAllInterface(cmiiEnv) + allInterface := DefaultCmiiOperator.PodAllInterface(cmiiEnv) // find the deployments for _, podInterface := range allInterface { @@ -145,7 +145,7 @@ func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) { func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) { // get app - podList := CmiiOperator.PodByAppName(cmiiEnv, appName) + podList := DefaultCmiiOperator.PodByAppName(cmiiEnv, appName) // get pod if podList == nil || len(podList) == 0 { @@ -154,7 +154,7 @@ func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCo } // exec env - stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"}) + stdout, stderr := DefaultCmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"}) errLog := stderr.String() if errLog != "" { @@ -182,7 +182,7 @@ func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCo func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) { - cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv) + cmiiPodInterfaces := DefaultCmiiOperator.PodAllInterface(cmiiEnv) for _, podInterface := range cmiiPodInterfaces { for key := range CmiiMiddlewareNameMap { @@ -219,9 +219,9 @@ func ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv string, nameReplicasMap map // 遍历 for appName, replica := range nameReplicasMap { - exists := CmiiOperator.DeploymentExist(cmiiEnv, appName) + exists := DefaultCmiiOperator.DeploymentExist(cmiiEnv, appName) if exists != nil { - scale := CmiiOperator.DeploymentScale(cmiiEnv, appName, replica) + scale := DefaultCmiiOperator.DeploymentScale(cmiiEnv, appName, replica) if !scale { errorUpdateMap[appName] = replica } @@ -238,12 +238,12 @@ func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool { result := true for _, deployment := range deploymentList { - result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0) + result = DefaultCmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0) if !result { return result } time.Sleep(time.Second) - result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas) + result = DefaultCmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas) if !result { return result } @@ -254,11 +254,11 @@ func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool { func RestartCmiiBackendDeployment(cmiiEnv string) { - cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) + cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { if AppNameBelongsToCmiiImage(deploymentInterface.Name) { - if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { + if !DefaultCmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) } else { log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) @@ -267,17 +267,17 @@ func RestartCmiiBackendDeployment(cmiiEnv string) { } } - log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) + log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", DefaultCmiiOperator.CurrentNamespace) } func RestartCmiiFrontendDeployment(cmiiEnv string) { - cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv) + cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv) for _, deploymentInterface := range cmiiDeploymentInterfaces { _, ok := CmiiFrontendAppMap[deploymentInterface.Name] if ok { - if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { + if !DefaultCmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) { log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name) } else { log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name) @@ -285,38 +285,39 @@ func RestartCmiiFrontendDeployment(cmiiEnv string) { } } - log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace) + log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", DefaultCmiiOperator.CurrentNamespace) } -func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool { +func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) (updateOK bool, oldImageTag, newImageTag string) { - cmiiDeploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName) + cmiiDeploymentInterface := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName) if cmiiDeploymentInterface == nil { - return false + return updateOK, oldImageTag, newImageTag } // check if need to update - if cmiiDeploymentInterface.ImageTag == newTag { + oldImageTag = cmiiDeploymentInterface.ImageTag + if oldImageTag == newTag { log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName) // restart - if CmiiOperator.DeploymentRestart(cmiiEnv, appName) { - return true + if DefaultCmiiOperator.DeploymentRestart(cmiiEnv, appName) { + return true, oldImageTag, oldImageTag } else { - return false + return false, oldImageTag, oldImageTag } } content := utils.WordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 35) content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.Name, 75) - content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105) + content = utils.WordSpaceCompletion(content+oldImageTag, 105) content = content + newTag + "\n" log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content) // update - tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag) + tag := DefaultCmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag) if !tag { log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content) - return false + return false, oldImageTag, newImageTag } // append log @@ -324,15 +325,13 @@ func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool { // re-get from env time.Sleep(time.Second) - deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName) + deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName) if deploy == nil { log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName) - return false + return false, oldImageTag, newImageTag } - // log - //log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] update tag [%s] success ! ", deploy.Image, content) - return true + return true, oldImageTag, deploy.ImageTag } func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) { @@ -340,12 +339,12 @@ func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]stri result = make(map[string]string, len(nameTagMap)) for appName, newTag := range nameTagMap { if AppNameBelongsToCmiiImage(appName) { - - if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) { + ok, oldImageTag, newImageTag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) + if ok { log.InfoF("[UpdateCmiiImageTagFromNameTagMap] - %s %s to %s", cmiiEnv, appName, newTag) - result[appName] = newTag + result[appName] = newImageTag } else { - result[appName] = "false" + result[appName] = oldImageTag } } } @@ -396,23 +395,23 @@ func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool { } log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag) - rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag) + ok, _, _ := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag) - return rollback + return ok } // BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用 func BackupAllDeploymentFromEnv(cmiiEnv string) bool { - allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) + allInterface := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv) // must filter allInterface = FilterAllCmiiAppSoft(allInterface) - //filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" - filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" + //filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" + filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" - log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath) + log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", DefaultCmiiOperator.CurrentNamespace, filePath) firstCol := 0 secondCol := 0 @@ -462,7 +461,7 @@ func BackupAllDeploymentFromEnv(cmiiEnv string) bool { // BackupAllCmiiDeploymentToMap 备份DEMO环境全部的镜像名称为Map 包括SRS 前端 后端 func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsMap map[string]string) { - allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv) + allInterface := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv) allInterface = FilterAllCmiiAppSoft(allInterface) backendMap = make(map[string]string, len(allInterface)) @@ -481,7 +480,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM for key, value := range CmiiSrsAppMap { var app *CmiiDeploymentInterface if strings.Contains(value, "deployment") { - app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key) + app = DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key) if app != nil { for _, imageName := range app.ContainerImageMap { split := strings.Split(imageName, ":") @@ -492,7 +491,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM } } } else if strings.Contains(value, "state") { - app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key) + app = DefaultCmiiOperator.StatefulSetOneInterface(cmiiEnv, key) if app != nil { for _, imageName := range app.ContainerImageMap { split := strings.Split(imageName, ":") @@ -531,9 +530,9 @@ func BackupAllCmiiDeploymentToList(cmiiEnv string, completePrefix bool) (allCmii func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) { - CmiiOperator.changeOperatorEnv(cmiiEnv) - //filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" - filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" + DefaultCmiiOperator.changeOperatorEnv(cmiiEnv) + //filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" + filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt" only := make(map[string]string, 150) // front @@ -541,7 +540,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) { for key, value := range CmiiFrontendAppMap { _, ok := only[key] if !ok { - deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) + deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deploy != nil { only[key] = value utils.AppendContentToFile(deploy.Image+"\n", filePath) @@ -552,7 +551,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) { for key, value := range CmiiBackendAppMap { _, ok := only[key] if !ok { - deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) + deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deploy != nil { only[key] = value utils.AppendContentToFile(deploy.Image+"\n", filePath) @@ -565,7 +564,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) { for key, value := range CmiiGISAppMap { _, ok := only[key] if !ok { - deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key) + deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key) if deploy != nil { only[key] = value utils.AppendContentToFile(deploy.Image+"\n", filePath) @@ -579,13 +578,13 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) { if !ok { var app *CmiiDeploymentInterface if strings.Contains(value, "deployment") { - app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key) + app = DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key) if app != nil { only[key] = value utils.AppendContentToFile(app.Image+"\n", filePath) } } else if strings.Contains(value, "state") { - app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key) + app = DefaultCmiiOperator.StatefulSetOneInterface(cmiiEnv, key) if app != nil { only[key] = value for _, imageName := range app.ContainerImageMap { diff --git a/agent-operator/CmiiK8sOperator_test.go b/agent-operator/CmiiK8sOperator_test.go index 2b96fd7..5f55246 100644 --- a/agent-operator/CmiiK8sOperator_test.go +++ b/agent-operator/CmiiK8sOperator_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "strconv" "testing" "time" "wdd.io/agent-common/assert" @@ -110,7 +111,7 @@ func TestFindPodNotHealthy_And_Delete(t *testing.T) { for _, podInterface := range podNotHealthy { t.Logf("[%s] [%s]", podInterface.Name, podInterface.PodPhase) - podDelete := CmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name) + podDelete := DefaultCmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name) assert.Equal(t, podDelete, true, "delete of ", podInterface.Namespace, podInterface.Name, " failed !") } @@ -118,12 +119,12 @@ func TestFindPodNotHealthy_And_Delete(t *testing.T) { } func TestFilterAllCmiiAppStrict(t *testing.T) { - allInterface := CmiiOperator.DeploymentAllInterface("devflight") + allInterface := DefaultCmiiOperator.DeploymentAllInterface("devflight") FilterAllCmiiAppStrict(allInterface) } func TestRestartDeploymentFromList(t *testing.T) { - allInterface := CmiiOperator.DeploymentAllInterface("devflight") + allInterface := DefaultCmiiOperator.DeploymentAllInterface("devflight") allInterface = FilterAllCmiiAppSoft(allInterface) RestartDeploymentFromList(allInterface) @@ -240,10 +241,10 @@ func TestRestartCmiiDeployment(t *testing.T) { cmiiEnv := integration appName := "cmii-uav-platform" - kill := CmiiOperator.DeploymentRestartByKill(cmiiEnv, appName) + kill := DefaultCmiiOperator.DeploymentRestartByKill(cmiiEnv, appName) assert.Equal(t, kill, true, "have unhealthy pod !") - check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) + check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) assert.Equal(t, check, true, "DeploymentStatusCheck failed !") } @@ -278,20 +279,22 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) { } for appName, newTag := range appNameTagMap { - tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) - assert.Equal(t, tag, true, "update image tag failed !") + ok, oldImageTag, newImageTag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) + assert.Equal(t, ok, true, "update image tag failed !") utils.SplitLinePrint() - check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 300) + check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 300) assert.Equal(t, check, true, "deployment run failed!") + deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName) + // push message message := pusher.CmiiUpdateMessage{ Namespace: cmiiEnv, AppName: appName, - FromTag: "", - ToTag: newTag, - Replicas: "", + FromTag: oldImageTag, + ToTag: newImageTag, + Replicas: strconv.FormatInt(int64(deploy.Replicas), 10), DeployStatus: check, } message.SendMessage() diff --git a/agent-operator/CmiiOperator.go b/agent-operator/CmiiOperator.go index 334953e..b7bd3f0 100644 --- a/agent-operator/CmiiOperator.go +++ b/agent-operator/CmiiOperator.go @@ -1,12 +1,8 @@ package main import ( - "bufio" "errors" - "fmt" - "io/fs" "os" - "path/filepath" "slices" "strings" @@ -59,13 +55,14 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul // compress if sync.ProjectVersion != "" { - // get version images + // 获取特定版本的镜像 + errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio) gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion } else { - // get demo images + // 获取DEMO的镜像 errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio) gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName @@ -147,9 +144,13 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu log.Info("COMPRESS START") for _, imageFullName := range fullNameList { - if !image.SaveToTarGZ(imageFullName, gzipFolderFullPath) { + ok, gzipImageFileFullPath := image.SaveToGzipFile(imageFullName, gzipFolderFullPath) + if !ok { errorGzipImageList = append(errorGzipImageList, imageFullName) + continue } + // 压缩成功 + allGzipFileNameList = append(allGzipFileNameList, gzipImageFileFullPath) } // remove failed fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { @@ -162,28 +163,17 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu //uploadGzipFileToDemoMinio() // get gzip file name list log.Info("UPLOAD OSS START !") - err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error { - if err != nil { - log.ErrorF("error getting gzip file name list 1! %s", err.Error()) - } - if !d.IsDir() { - allGzipFileNameList = append(allGzipFileNameList, d.Name()) - } - return nil - }) - if err != nil { - log.ErrorF("error getting gzip file name list 2! %s", err.Error()) - } // start to upload // extract demo oss location suffix from gzipFolderFullPath trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix) - bucketName := "cmlc-installation/" + trimPrefix - log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName) + bucketNameWithPrefix := "cmlc-installation/" + trimPrefix + log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix) for _, gzipFileName := range allGzipFileNameList { - - if !DefaultCmiiMinioOperator.UploadToDemo(bucketName, gzipFolderFullPath, gzipFileName) { + // SaveToGzipFile 返回的是全路径 归一化处理 gzip file name + gzipFileName = strings.TrimPrefix(gzipFileName, gzipFolderFullPath) + if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileName) { log.ErrorF("upload of %s to demo oss error !", gzipFolderFullPath+gzipFileName) } } @@ -193,59 +183,75 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileNameList } -// DownloadLoadTagPush DLTU procedure ImageSync的另外一般流程,需要支持 堡垒机(纯离线)的模式 +// DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程,需要支持 堡垒机(纯离线)的模式 // 2. Gzip文件目录,RKE MIDDLE CMII三个文件目录 - 约定目录 // 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/ // 3. 读取本机的IP地址 - 参数传递 // 4. OSS地址 - ossUrlPrefix传空 则使用默认值 // 5. ossFileName - 如果结尾为txt,则为文件的形式,如果为tar.gz,则为gzip文件夹的形式 -func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolder string, targetHarborFullName string) []string { +func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) { + + // 支持单文件的形式 + if !utils.IsFileOrDir(localGzipFolderOrGzipFile) { + // 单个压缩文件 + if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") { + log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile) + return nil + } + + // load + image.LoadFromGzipFilePath(localGzipFolderOrGzipFile) + } separator := os.PathSeparator - if !strings.HasSuffix(localGzipFolder, string(separator)) { - localGzipFolder += string(separator) + if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) { + localGzipFolderOrGzipFile += string(separator) } // download if downloadFromOss { - if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder) { + if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) { log.ErrorF("download from oss error !") return nil } } // load - loadAllGzipImageFromLocalFolder(localGzipFolder) + loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile) // tag // push - allFileInFolder, err := utils.ListAllFileInFolder(localGzipFolder) + allFileInFolder, err := utils.ListAllFileInFolder(localGzipFolderOrGzipFile) if err != nil { return nil } for _, gzipFileName := range allFileInFolder { + // 过滤非.tar.gz结尾的文件 + if !strings.HasSuffix(gzipFileName, ".tar.gz") { + continue + } + log.DebugF("gzip file name is %s", gzipFileName) + + // gzip to image full name 拿到镜像的原始名称 imageFullName := image2.GzipFileNameToImageFullName(gzipFileName) if imageFullName == "" { log.ErrorF("gzip file %s to image full name error !", gzipFileName) continue } + + // tag 拿到目标名称 然后重新Tag targetImageFullName := image2.ImageNameToTargetImageFullName(imageFullName, targetHarborFullName) - - // tag image.TagFromSourceToTarget(imageFullName, targetImageFullName) - //push - pushResult := image.PushToOctopusKindHarbor(targetImageFullName) - defer pushResult.Close() - scanner := bufio.NewScanner(pushResult) - for scanner.Scan() { + // uploadToHarbor 上传到目标Harbor + if image.UploadToHarbor(targetImageFullName) { + targetImageFullNameList = append(targetImageFullNameList, targetHarborFullName) + } else { + log.ErrorF("upload to harbor error of %s", targetImageFullName) } - fmt.Println() - fmt.Printf("%s to %s push success !", gzipFileName, targetImageFullName) - fmt.Println() } - return nil + return targetImageFullNameList } func loadAllGzipImageFromLocalFolder(localGzipFolder string) { @@ -381,12 +387,12 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string { for key, value := range CmiiSrsAppMap { var app *CmiiDeploymentInterface if strings.Contains(value, "deployment") { - app = CmiiOperator.DeploymentOneInterface(demo, key) + app = DefaultCmiiOperator.DeploymentOneInterface(demo, key) if app != nil { realCmiiImageName = append(realCmiiImageName, app.Image) } } else if strings.Contains(value, "state") { - app = CmiiOperator.StatefulSetOneInterface(demo, key) + app = DefaultCmiiOperator.StatefulSetOneInterface(demo, key) if app != nil { for _, imageName := range app.ContainerImageMap { realCmiiImageName = append(realCmiiImageName, imageName) diff --git a/agent-operator/K8sOperator.go b/agent-operator/K8sOperator.go index add3dc4..2522ada 100644 --- a/agent-operator/K8sOperator.go +++ b/agent-operator/K8sOperator.go @@ -386,6 +386,25 @@ func (op *CmiiK8sOperator) DeploymentScale(cmiiEnv, appName string, scaleCount i return true } +func (op *CmiiK8sOperator) DeploymentUpdateTagByImageFullName(cmiiEnv, imageFullName string) bool { + + split := strings.Split(imageFullName, ":") + // harbor + // 192.168.6.6:8033/rancher/k8s-dns-sidecar:v1.0.2 + newTag := split[1] + appName := strings.Split(split[0], "/")[len(strings.Split(split[0], "/"))-1] + + if strings.Contains(imageFullName, "8033") { + newTag = split[2] + + appName = strings.Split(split[1], "/")[len(strings.Split(split[1], "/"))-1] + } + + // 拿到AppName + log.DebugF("DeploymentUpdateTagByImageFullName - appName => %s, newTag => %s", appName, newTag) + + return op.DeploymentUpdateTag(cmiiEnv, appName, newTag) +} func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string) bool { @@ -400,37 +419,55 @@ func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string) } containers := deployment.Spec.Template.Spec.Containers - if len(containers) == 1 { - // only update this kind - container := containers[0] - - oldName := container.Image - - split := strings.Split(container.Image, ":") - if strings.HasPrefix(container.Image, image2.CmiiHarborPrefix) { - // harbor - container.Image = split[0] + ":" + newTag - } else if strings.Contains(container.Image, "8033") { - // 192.168.6.6:8033/rancher/k8s-dns-sidecar:v1.0.2 - container.Image = split[0] + ":" + split[1] + ":" + newTag - } - - log.DebugF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s]", op.CurrentNamespace, appName, oldName, container.Image) - - // re assign - deployment.Spec.Template.Spec.Containers[0] = container - - // update - _, err := op.CurrentClient.AppsV1().Deployments(deployment.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) - if err != nil { - log.ErrorF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s] error ! %s", op.CurrentNamespace, appName, split[1], container.Image, err.Error()) - return false - } - } else if len(containers) == 2 { + if len(containers) >= 2 { log.ErrorF("[DeploymentUpdateTag] - cant update app with 2 containers !") return false } + // 只支持container的数量为1的形式 + container := containers[0] + + oldName := container.Image + split := strings.Split(container.Image, ":") + if strings.HasPrefix(container.Image, image2.CmiiHarborPrefix) { + // harbor + container.Image = split[0] + ":" + newTag + } else if strings.Contains(container.Image, "8033") { + // 192.168.6.6:8033/rancher/k8s-dns-sidecar:v1.0.2 + // 重新拼接 + container.Image = split[0] + ":" + split[1] + ":" + newTag + } + log.DebugF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s]", op.CurrentNamespace, appName, oldName, container.Image) + + // 更新Cmii BIZ_GROUP + tagVersion := newTag + if strings.Contains(newTag, "-") { + tagVersion = strings.Split(newTag, "-")[0] + } + envList := container.Env + for _, envVar := range envList { + if envVar.Name == "IMAGE_VERSION" { + envVar.Value = tagVersion + } + if envVar.Name == "BIZ_CONFIG_GROUP" { + envVar.Value = tagVersion + } + if envVar.Name == "SYS_CONFIG_GROUP" { + envVar.Value = tagVersion + } + } + log.DebugF("[DeploymentUpdateTag] - update env IMAGE_VERSION to [%s]", tagVersion) + + // 赋值回去 很关键 + deployment.Spec.Template.Spec.Containers[0] = container + + // update + _, err := op.CurrentClient.AppsV1().Deployments(deployment.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{}) + if err != nil { + log.ErrorF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s] error ! %s", op.CurrentNamespace, appName, split[1], container.Image, err.Error()) + return false + } + return true } diff --git a/agent-operator/K8sOperator_test.go b/agent-operator/K8sOperator_test.go index 1867ba2..b2ee624 100644 --- a/agent-operator/K8sOperator_test.go +++ b/agent-operator/K8sOperator_test.go @@ -12,7 +12,7 @@ import ( func TestCmiiK8sOperator_DeploymentAll(t *testing.T) { start := time.Now() - deploymentList := CmiiOperator.DeploymentAll("devflight") + deploymentList := DefaultCmiiOperator.DeploymentAll("devflight") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -39,7 +39,7 @@ func TestCmiiK8sOperator_DeploymentAll(t *testing.T) { func TestCmiiK8sOperator_DeploymentAllInterface(t *testing.T) { start := time.Now() - deploymentList := CmiiOperator.DeploymentAllInterface("devflight") + deploymentList := DefaultCmiiOperator.DeploymentAllInterface("devflight") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -54,7 +54,7 @@ func TestCmiiK8sOperator_DeploymentAllInterface(t *testing.T) { func TestCmiiK8sOperator_DeploymentFizz(t *testing.T) { start := time.Now() - deploymentFizz := CmiiOperator.DeploymentFizz("int", "cmii-suav-supervision") + deploymentFizz := DefaultCmiiOperator.DeploymentFizz("int", "cmii-suav-supervision") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -75,7 +75,7 @@ func TestCmiiK8sOperator_DeploymentFizz(t *testing.T) { func TestCmiiK8sOperator_DeploymentScale(t *testing.T) { start := time.Now() - CmiiOperator.DeploymentScale(demo, "cmii-uav-industrial-portfolio", 1) + DefaultCmiiOperator.DeploymentScale(demo, "cmii-uav-industrial-portfolio", 1) elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -84,7 +84,7 @@ func TestCmiiK8sOperator_DeploymentScale(t *testing.T) { func TestCmiiK8sOperator_DeploymentUpdateTag(t *testing.T) { start := time.Now() - CmiiOperator.DeploymentUpdateTag("demo", "cmii-uav-platform", "5.2.0-011001") + DefaultCmiiOperator.DeploymentUpdateTag("demo", "cmii-uav-platform", "5.2.0-011001") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) } @@ -94,11 +94,11 @@ func TestCmiiK8sOperator_DeploymentRestart(t *testing.T) { cmiiEnv := integration appName := "cmii-uav-data-post-process" - CmiiOperator.DeploymentRestart(cmiiEnv, appName) + DefaultCmiiOperator.DeploymentRestart(cmiiEnv, appName) utils.SplitLinePrint() - check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) + check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) assert.Equal(t, check, true, "deployment run failed!") } @@ -106,19 +106,19 @@ func TestCmiiK8sOperator_DeploymentRestartByKill(t *testing.T) { cmiiEnv := "demo" appName := "cmii-uav-platform" - kill := CmiiOperator.DeploymentRestartByKill(cmiiEnv, appName) + kill := DefaultCmiiOperator.DeploymentRestartByKill(cmiiEnv, appName) assert.Equal(t, kill, true, "deployment restart by kill failed !") utils.SplitLinePrint() - check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) + check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) assert.Equal(t, check, true, "deployment run failed!") } func TestCmiiK8sOperator_DeploymentOneInterface(t *testing.T) { start := time.Now() - deploy := CmiiOperator.DeploymentOneInterface("devflight", "cmii-uav-depotautoreturn") + deploy := DefaultCmiiOperator.DeploymentOneInterface("devflight", "cmii-uav-depotautoreturn") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -130,7 +130,7 @@ func TestCmiiK8sOperator_ReplicaSetExists(t *testing.T) { cmiiEnv := "uavcloud-devflight" appName := "cmii-admin-data-bf8f87cb7" - exists := CmiiOperator.ReplicaSetExists(cmiiEnv, appName) + exists := DefaultCmiiOperator.ReplicaSetExists(cmiiEnv, appName) utils.BeautifulPrint(*exists) } @@ -140,7 +140,7 @@ func TestCmiiK8sOperator_ReplicaSetByAppName(t *testing.T) { cmiiEnv := "uavcloud-devflight" appName := "cmii-admin-data" - exists := CmiiOperator.ReplicaSetByAppName(cmiiEnv, appName) + exists := DefaultCmiiOperator.ReplicaSetByAppName(cmiiEnv, appName) for _, replicaSet := range exists { utils.BeautifulPrint(replicaSet) @@ -149,7 +149,7 @@ func TestCmiiK8sOperator_ReplicaSetByAppName(t *testing.T) { func TestCmiiK8sOperator_PodAll(t *testing.T) { start := time.Now() - podList := CmiiOperator.PodAll("devflight") + podList := DefaultCmiiOperator.PodAll("devflight") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -163,7 +163,7 @@ func TestCmiiK8sOperator_PodAll(t *testing.T) { func TestCmiiK8sOperator_PodFizz(t *testing.T) { start := time.Now() - podList := CmiiOperator.PodFizz("devflight", "cmii-uav-data-post-process") + podList := DefaultCmiiOperator.PodFizz("devflight", "cmii-uav-data-post-process") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) t.Logf("pod list lenght is => %d", len(podList)) @@ -183,7 +183,7 @@ func TestCmiiK8sOperator_PodByAppName(t *testing.T) { cmiiEnv := "uat" appName := "cmii-admin-data" - exists := CmiiOperator.PodByNodeName(cmiiEnv, appName) + exists := DefaultCmiiOperator.PodByNodeName(cmiiEnv, appName) for _, podInterface := range exists { utils.BeautifulPrint(podInterface) @@ -193,7 +193,7 @@ func TestCmiiK8sOperator_PodByAppName(t *testing.T) { func TestCmiiK8sOperator_PodFizz2(t *testing.T) { start := time.Now() - podList := CmiiOperator.PodFizz("devflight", "notice") + podList := DefaultCmiiOperator.PodFizz("devflight", "notice") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) t.Logf("pod list lenght is => %d", len(podList)) @@ -215,13 +215,13 @@ func TestCmiiK8sOperator_PodByNodeName(t *testing.T) { cmiiEnv := "uat" nodeName := "test-03.ecs.io" - exists := CmiiOperator.PodByNodeName(cmiiEnv, nodeName) + exists := DefaultCmiiOperator.PodByNodeName(cmiiEnv, nodeName) exists = FilterAllCmiiPodSoft(exists) for _, podInterface := range exists { utils.BeautifulPrint(podInterface) if !podInterface.PodStatus { - podDelete := CmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name) + podDelete := DefaultCmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name) assert.Equal(t, podDelete, true, "delete pod failed !") } } @@ -229,9 +229,9 @@ func TestCmiiK8sOperator_PodByNodeName(t *testing.T) { func TestCmiiK8sOperator_PodExec(t *testing.T) { - podList := CmiiOperator.PodByAppName(devFlight, "cmii-uav-gateway") + podList := DefaultCmiiOperator.PodByAppName(devFlight, "cmii-uav-gateway") - stdout, stderr := CmiiOperator.PodExec(devFlight, podList[0], []string{ + stdout, stderr := DefaultCmiiOperator.PodExec(devFlight, podList[0], []string{ "env", }) @@ -262,14 +262,14 @@ func TestCmiiK8sOperator_DeploymentStatusCheck(t *testing.T) { cmiiEnv := "devflight" appName := "cmii-uav-gateway" - check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) + check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180) assert.Equal(t, check, true, "deployment run failed!") } func TestCmiiK8sOperator_NodeAll(t *testing.T) { start := time.Now() - nodeList := CmiiOperator.NodeAll("dev") + nodeList := DefaultCmiiOperator.NodeAll("dev") elapsed := time.Since(start).Milliseconds() fmt.Printf("执行耗时: %d ms\n", elapsed) @@ -282,7 +282,7 @@ func TestCmiiK8sOperator_NodeAll(t *testing.T) { } func TestCmiiK8sOperator_NodeAllInterface(t *testing.T) { - cmiiNodeInterfaces := CmiiOperator.NodeAllInterface("uat") + cmiiNodeInterfaces := DefaultCmiiOperator.NodeAllInterface("uat") for _, nodeInterface := range cmiiNodeInterfaces { println() diff --git a/agent-operator/image/CmiiImageSync.go b/agent-operator/image/CmiiImageSync.go index 3af2e48..ed62097 100644 --- a/agent-operator/image/CmiiImageSync.go +++ b/agent-operator/image/CmiiImageSync.go @@ -181,7 +181,7 @@ func TagFromSourceToTarget(sourceImageName, targetImageName string) bool { return true } -func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) { +func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) { if GetByName(targetImageName) == nil { log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName) @@ -202,6 +202,17 @@ func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) return pushResult } +func UploadToHarbor(targetImageName string) (uploadOK bool) { + pushResult := UploadToOctopusKindHarbor(targetImageName) + defer pushResult.Close() + scanner := bufio.NewScanner(pushResult) + for scanner.Scan() { + } + fmt.Println() + log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName) + fmt.Println() +} + // TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) { @@ -239,7 +250,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s fmt.Println(targetImageName) //if TagFromSourceToTarget(cmiiImageFullName, targetImageName) { - // pushResult := PushToOctopusKindHarbor(targetImageName) + // pushResult := UploadToOctopusKindHarbor(targetImageName) // if pushResult == nil { // errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName) // log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName) @@ -360,11 +371,13 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) for _, image := range fullImageNameList { - if !SaveToTarGZ(image, gzipFolder) { + ok, path := SaveToGzipFile(image, gzipFolder) + if !ok { errorGzipImageList = append(errorGzipImageList, image) continue } - tarGzipFileNameList = append(tarGzipFileNameList, image2.ImageFullNameToGzipFileName(image)) + + tarGzipFileNameList = append(tarGzipFileNameList, path) } utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list") @@ -372,20 +385,20 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) } // LoadFromGzipFilePath 根据Gzip文件的全名称进行Load操作 -func LoadFromGzipFilePath(gzipFullPath string) bool { - openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm) +func LoadFromGzipFilePath(gzipFileNameFullPath string) bool { + openFile, err := os.OpenFile(gzipFileNameFullPath, 0, fs.ModePerm) if err != nil { - log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error()) + log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFileNameFullPath, err.Error()) return false } loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true) if err != nil { - log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFullPath, err.Error()) + log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFileNameFullPath, err.Error()) return false } - log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFullPath, strconv.FormatBool(loadResponse.JSON)) + log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFileNameFullPath, strconv.FormatBool(loadResponse.JSON)) scanner := bufio.NewScanner(loadResponse.Body) for scanner.Scan() { @@ -420,18 +433,19 @@ func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) { return errorLoadImageNameList } -func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { +// SaveToGzipFile 根据目标镜像的全名称 将镜像压缩为Gzip文件 +func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipImageFileFullPath string) { - imageGetByName := GetByName(targetImageName) + imageGetByName := GetByName(imageFullName) if imageGetByName == nil { - log.ErrorF("[ImageSaveToTarGZ] - %s not exits", targetImageName) - return false + log.ErrorF("[ImageSaveToTarGZ] - %s not exits", imageFullName) + return false, "" } imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags) if err != nil { log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error()) - return false + return false, "" } var realImageTag string @@ -442,19 +456,25 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { } } - gzipImageFile := image2.ImageFullNameToGzipFileName(realImageTag) + gzipImageFileFullPath = image2.ImageFullNameToGzipFileName(realImageTag) if !strings.HasSuffix(folderPathPrefix, "/") { folderPathPrefix += "/" } _ = os.MkdirAll(folderPathPrefix, os.ModeDir) - gzipImageFile = folderPathPrefix + gzipImageFile - log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFile) - _ = os.Remove(gzipImageFile) - tarFile, err := os.Create(gzipImageFile) + // 生成gzip压缩文件的全路径名称 + gzipImageFileFullPath = folderPathPrefix + gzipImageFileFullPath + + log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFileFullPath) + + // 删除掉旧的Gzip文件 + _ = os.Remove(gzipImageFileFullPath) + + // 创建 + tarFile, err := os.Create(gzipImageFileFullPath) if err != nil { - log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFile, err.Error()) - return false + log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFileFullPath, err.Error()) + return false, "" } defer tarFile.Close() @@ -468,10 +488,11 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { // Copy the tar archive to the gzip writer. if _, err := io.Copy(gw, imageSaveTarStream); err != nil { log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error()) - return false + return false, "" } - return true + // 成功 + return true, gzipImageFileFullPath } func CmiiImageMapToFullNameList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { diff --git a/agent-operator/image/CmiiImageSync_test.go b/agent-operator/image/CmiiImageSync_test.go index 333840e..bdf7b67 100644 --- a/agent-operator/image/CmiiImageSync_test.go +++ b/agent-operator/image/CmiiImageSync_test.go @@ -88,7 +88,7 @@ func TestImagePushToOctopusKindHarbor(t *testing.T) { assert.Equal(t, target, true, "image re-tag error !") // push - pushResult := PushToOctopusKindHarbor(newTag) + pushResult := UploadToOctopusKindHarbor(newTag) defer pushResult.Close() scanner := bufio.NewScanner(pushResult) @@ -109,7 +109,7 @@ func TestImageLoadFromFile(t *testing.T) { func TestImageSaveToTarGZ(t *testing.T) { image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta" - imageSaveToTarGZ := SaveToTarGZ(image, "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log") + imageSaveToTarGZ, _ := SaveToGzipFile(image, "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log") assert.Equal(t, imageSaveToTarGZ, true, "image save to tar gz file error !") } @@ -149,7 +149,7 @@ func TestImageTagFromSourceToTarget(t *testing.T) { targetImageName := "harbor.wdd.io:8033/cmii/srs:v5.0.195" if TagFromSourceToTarget(sourceImageName, targetImageName) { - pushResult := PushToOctopusKindHarbor(targetImageName) + pushResult := UploadToOctopusKindHarbor(targetImageName) defer pushResult.Close() scanner := bufio.NewScanner(pushResult) @@ -188,7 +188,7 @@ func TestSaveSpecificImageToGzipFile(t *testing.T) { // image pull success log.InfoF("image should have pulled successful ! => %s", imageFullName) - if !SaveToTarGZ(imageFullName, imageGzipFilePathPrefix) { + if !SaveToGzipFile(imageFullName, imageGzipFilePathPrefix) { log.ErrorF("image save to gzip file error ! => %s", imageFullName) return } diff --git a/agent-operator/main.go b/agent-operator/main.go index 0d1da3f..fc57af4 100644 --- a/agent-operator/main.go +++ b/agent-operator/main.go @@ -33,7 +33,7 @@ func RealProjectRunner() { op := CmiiK8sOperator{} op.BuildCurrentClientFromConfig(readFile) - CmiiOperator = op + DefaultCmiiOperator = op // ops @@ -77,7 +77,7 @@ func CmiiRunner() { var DLTUHelp = ` DLTUHelp - dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName] + dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName] [namespace] ` func main() { @@ -113,7 +113,7 @@ func main() { result = append(result, text) } - if len(result) != 5 { + if len(result) != 6 { fmt.Println("input error!") fmt.Printf(DLTUHelp) return @@ -123,11 +123,13 @@ func main() { ossFileName := result[2] localGzipFolder := result[3] harborHostFullName := result[4] + namespace := result[5] fmt.Println("ossUrlPrefix: ", ossUrlPrefix) fmt.Println("ossFileName: ", ossFileName) fmt.Println("localGzipFolder: ", localGzipFolder) fmt.Println("harborHostFullName: ", harborHostFullName) + fmt.Println("namespace: ", namespace) fmt.Println() downloadFromOss := true @@ -135,9 +137,17 @@ func main() { downloadFromOss = false } - DownloadLoadTagPush(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName) + // DLTU + targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName) - // 下载 + // 是否需要更新 + if namespace != "" { + for _, targetImageFullName := range targetImageFullNameList { + if !DefaultCmiiOperator.DeploymentUpdateTagByImageFullName(namespace, targetImageFullName) { + fmt.Printf("[Update] update [%s] [%s] failed", namespace, targetImageFullName) + } + } + } fmt.Println() } diff --git a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java index 8774be7..83442a6 100644 --- a/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java +++ b/server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java @@ -38,13 +38,13 @@ public class TestImageSyncScheduler { public void runImageSync() { ArrayList CmiiAppNameList = new ArrayList<>(List.of( -// "cmii-uav-process:5.4.0-041901" +// "cmii-uav-process:5.4.0-041901"1 )); ArrayList ImageFullNameList = new ArrayList<>(List.of( // "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0", // "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195" - "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042601" + "harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042601" )); Boolean downloadAndCompressOnly = false;