From 84455a88496d0670b428577d28fa244a109215ee Mon Sep 17 00:00:00 2001 From: zeaslity Date: Thu, 25 Jan 2024 12:07:17 +0800 Subject: [PATCH] [ Cmii ] [ Operator ] - optimize Image pull --- agent-go/utils/PrintUtils.go | 11 +++++ cmii_operator/CmiiImageSync.go | 65 +++++++++++++----------------- cmii_operator/CmiiOperator.go | 29 ++++++++----- cmii_operator/CmiiOperator_test.go | 16 ++++++-- cmii_operator/DependencyConfig.go | 1 + 5 files changed, 71 insertions(+), 51 deletions(-) diff --git a/agent-go/utils/PrintUtils.go b/agent-go/utils/PrintUtils.go index 288d570..80755ed 100644 --- a/agent-go/utils/PrintUtils.go +++ b/agent-go/utils/PrintUtils.go @@ -31,6 +31,17 @@ func BeautifulPrintToString(object interface{}) string { return string(bytes) } +func BeautifulPrintListWithTitle(contend []string, title string) { + + fmt.Println() + fmt.Println(fmt.Sprintf("content tile is => %s", title)) + for _, line := range contend { + bytes, _ := json.MarshalIndent(line, "", " ") + fmt.Println(string(bytes)) + } + fmt.Println("---------- end -----------") +} + func SplitLinePrint() { fmt.Println() fmt.Println() diff --git a/cmii_operator/CmiiImageSync.go b/cmii_operator/CmiiImageSync.go index e7edbd1..eaf768a 100644 --- a/cmii_operator/CmiiImageSync.go +++ b/cmii_operator/CmiiImageSync.go @@ -6,15 +6,13 @@ import ( "encoding/json" "errors" "fmt" + "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" "github.com/klauspost/pgzip" "io" "os" "strings" - "wdd.io/cmii_operator/tools" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/client" ) var apiClient = newClient() @@ -158,35 +156,10 @@ func ImagePullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) { return pullResult } -func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) { +func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (errorPullImageList []string) { - var fs uintptr - - for image, tag := range imageVersionMap { - s := CmiiHarborPrefix + image + ":" + tag - pullResult := ImagePullFromCmiiHarbor(s) - if pullResult == nil { - continue - } - if silentMode { - scanner := bufio.NewScanner(pullResult) - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "\"status\":\"Pulling from") { - fmt.Println(line) - } - if strings.Contains(line, "Status: Image is up to date for") { - fmt.Println(line) - } - } - } else { - _ = tools.DisplayJSONMessagesStream(pullResult, os.Stdout, fs, true, func(message tools.JSONMessage) { - - }) - } - fmt.Println() - fmt.Println() - } + fullImageNameList := convertCMiiImageMapToList(imageVersionMap) + return ImagePullFromFullNameList(fullImageNameList) } @@ -225,13 +198,14 @@ func ImagePullFromFileJson(filePathName string) { } -func ImagePullFromList(depContainerList []string) { - for _, dep := range depContainerList { +func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) { + for _, dep := range fullImageNameList { loginToDockerHub() pullResult := ImagePullFromCmiiHarbor(dep) if pullResult == nil { + errorPullImageList = append(errorPullImageList, dep) continue } scanner := bufio.NewScanner(pullResult) @@ -246,11 +220,13 @@ func ImagePullFromList(depContainerList []string) { } fmt.Println() } + + return errorPullImageList } -func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string) { +func ImagePullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) { - ImagePullFromList(imageNameList) + errorPullImageList = ImagePullFromFullNameList(fullImageNameList) // generate a project folder err := os.MkdirAll(gzipFolder, os.ModeDir) @@ -260,10 +236,13 @@ func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string } } - for _, image := range imageNameList { - ImageSaveToTarGZ(image, gzipFolder) + for _, image := range fullImageNameList { + if !ImageSaveToTarGZ(image, gzipFolder) { + errorGzipImageList = append(errorGzipImageList, image) + } } + return errorPullImageList, errorGzipImageList } func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool { @@ -350,6 +329,16 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) { return gzipFileName } +func convertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { + + for image, tag := range cmiiImageVersionMap { + s := CmiiHarborPrefix + image + ":" + tag + fullImageNameList = append(fullImageNameList, s) + } + + return fullImageNameList +} + func loginToDockerHub() { login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{ diff --git a/cmii_operator/CmiiOperator.go b/cmii_operator/CmiiOperator.go index 0d62a5c..78db913 100644 --- a/cmii_operator/CmiiOperator.go +++ b/cmii_operator/CmiiOperator.go @@ -9,14 +9,14 @@ import ( const OfflineImageGzipFolderPrefix = "/root/octopus_image/" -func FetchDemoImages(projectName string, gzipSplit bool) bool { +func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList []string) { // generate a project folder err := os.Mkdir(OfflineImageGzipFolderPrefix+projectName, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error()) - return false + return errorPullImageList, errorGzipImageList } } @@ -42,26 +42,33 @@ func FetchDemoImages(projectName string, gzipSplit bool) bool { ) // download image - ImagePullFromCmiiHarborByMap(backendMap, true) - ImagePullFromCmiiHarborByMap(frontendMap, true) + backendPull := ImagePullFromCmiiHarborByMap(backendMap, true) + frontendPull := ImagePullFromCmiiHarborByMap(frontendMap, true) // compress image if gzipSplit { for image, tag := range backendMap { - ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") + if !ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { + errorGzipImageList = append(errorGzipImageList, CmiiHarborPrefix+image+":"+tag) + } } for image, tag := range frontendMap { - ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") + if !ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") { + errorGzipImageList = append(errorGzipImageList, CmiiHarborPrefix+image+":"+tag) + } } } // upload to harbor // clean up images - return false + errorPullImageList = append(errorPullImageList, backendPull...) + errorPullImageList = append(errorPullImageList, frontendPull...) + + return errorPullImageList, errorGzipImageList } -func FetchDependencyRepos(gzipSplit bool) { +func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) { err := os.Mkdir(OfflineImageGzipFolderPrefix, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { @@ -69,8 +76,10 @@ func FetchDependencyRepos(gzipSplit bool) { } } - ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/") + errorPullImageList, errorGzipImageList = ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/") - ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/") + pull, gzipImageList := ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/") + + return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...) } diff --git a/cmii_operator/CmiiOperator_test.go b/cmii_operator/CmiiOperator_test.go index 88b1a3a..4f01631 100644 --- a/cmii_operator/CmiiOperator_test.go +++ b/cmii_operator/CmiiOperator_test.go @@ -1,11 +1,21 @@ package cmii_operator -import "testing" +import ( + "testing" + "wdd.io/agent-go/utils" +) func TestFetchDemoImages(t *testing.T) { - FetchDemoImages("cqga", true) + errorPullImageList, errorGzipImageList := FetchDemoImages("cqga", true) + + utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList") + utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList") } func TestFetchDependencyRepos(t *testing.T) { - FetchDependencyRepos(true) + errorPullImageList, errorGzipImageList := FetchDependencyRepos(true) + + utils.BeautifulPrintListWithTitle(errorPullImageList, "dep errorPullImageList") + utils.BeautifulPrintListWithTitle(errorGzipImageList, "dep errorGzipImageList") + } diff --git a/cmii_operator/DependencyConfig.go b/cmii_operator/DependencyConfig.go index 0d84ebc..53df232 100644 --- a/cmii_operator/DependencyConfig.go +++ b/cmii_operator/DependencyConfig.go @@ -84,6 +84,7 @@ var Rancher1204Amd64 = []string{ "rancher/rancher-webhook:v0.1.0-beta9", "rancher/rancher:v2.5.7", "rancher/rke-tools:v0.1.72", + "jerrychina2020/rke-tools:v0.175-linux", "rancher/security-scan:v0.1.14", "rancher/security-scan:v0.2.2", "rancher/shell:v0.1.6",