[ Cmii ] [ Operator ] - optimize Image pull

agent-go/utils/PrintUtils.go

@@ -31,6 +31,17 @@ func BeautifulPrintToString(object interface{}) string {
 	return string(bytes)
 }
 
+func BeautifulPrintListWithTitle(contend []string, title string) {
+
+	fmt.Println()
+	fmt.Println(fmt.Sprintf("content tile is => %s", title))
+	for _, line := range contend {
+		bytes, _ := json.MarshalIndent(line, "", "  ")
+		fmt.Println(string(bytes))
+	}
+	fmt.Println("---------- end -----------")
+}
+
 func SplitLinePrint() {
 	fmt.Println()
 	fmt.Println()

cmii_operator/CmiiImageSync.go

@@ -6,15 +6,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
 	"github.com/klauspost/pgzip"
 	"io"
 	"os"
 	"strings"
-	"wdd.io/cmii_operator/tools"
-
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
 )
 
 var apiClient = newClient()
@@ -158,35 +156,10 @@ func ImagePullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
 	return pullResult
 }
 
-func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) {
+func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (errorPullImageList []string) {
 
-	var fs uintptr
+	fullImageNameList := convertCMiiImageMapToList(imageVersionMap)
-
+	return ImagePullFromFullNameList(fullImageNameList)
-	for image, tag := range imageVersionMap {
-		s := CmiiHarborPrefix + image + ":" + tag
-		pullResult := ImagePullFromCmiiHarbor(s)
-		if pullResult == nil {
-			continue
-		}
-		if silentMode {
-			scanner := bufio.NewScanner(pullResult)
-			for scanner.Scan() {
-				line := scanner.Text()
-				if strings.Contains(line, "\"status\":\"Pulling from") {
-					fmt.Println(line)
-				}
-				if strings.Contains(line, "Status: Image is up to date for") {
-					fmt.Println(line)
-				}
-			}
-		} else {
-			_ = tools.DisplayJSONMessagesStream(pullResult, os.Stdout, fs, true, func(message tools.JSONMessage) {
-
-			})
-		}
-		fmt.Println()
-		fmt.Println()
-	}
 
 }
 
@@ -225,13 +198,14 @@ func ImagePullFromFileJson(filePathName string) {
 
 }
 
-func ImagePullFromList(depContainerList []string) {
+func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
-	for _, dep := range depContainerList {
 
+	for _, dep := range fullImageNameList {
 		loginToDockerHub()
 
 		pullResult := ImagePullFromCmiiHarbor(dep)
 		if pullResult == nil {
+			errorPullImageList = append(errorPullImageList, dep)
 			continue
 		}
 		scanner := bufio.NewScanner(pullResult)
@@ -246,11 +220,13 @@ func ImagePullFromList(depContainerList []string) {
 		}
 		fmt.Println()
 	}
+
+	return errorPullImageList
 }
 
-func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string) {
+func ImagePullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
 
-	ImagePullFromList(imageNameList)
+	errorPullImageList = ImagePullFromFullNameList(fullImageNameList)
 
 	// generate a project folder
 	err := os.MkdirAll(gzipFolder, os.ModeDir)
@@ -260,10 +236,13 @@ func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string
 		}
 	}
 
-	for _, image := range imageNameList {
+	for _, image := range fullImageNameList {
-		ImageSaveToTarGZ(image, gzipFolder)
+		if !ImageSaveToTarGZ(image, gzipFolder) {
+			errorGzipImageList = append(errorGzipImageList, image)
+		}
 	}
 
+	return errorPullImageList, errorGzipImageList
 }
 
 func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
@@ -350,6 +329,16 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
 	return gzipFileName
 }
 
+func convertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
+
+	for image, tag := range cmiiImageVersionMap {
+		s := CmiiHarborPrefix + image + ":" + tag
+		fullImageNameList = append(fullImageNameList, s)
+	}
+
+	return fullImageNameList
+}
+
 func loginToDockerHub() {
 
 	login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{

cmii_operator/CmiiOperator.go

@@ -9,14 +9,14 @@ import (
 
 const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
 
-func FetchDemoImages(projectName string, gzipSplit bool) bool {
+func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
 
 	// generate a project folder
 	err := os.Mkdir(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
 			log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
-			return false
+			return errorPullImageList, errorGzipImageList
 		}
 	}
 
@@ -42,26 +42,33 @@ func FetchDemoImages(projectName string, gzipSplit bool) bool {
 	)
 
 	// download image
-	ImagePullFromCmiiHarborByMap(backendMap, true)
+	backendPull := ImagePullFromCmiiHarborByMap(backendMap, true)
-	ImagePullFromCmiiHarborByMap(frontendMap, true)
+	frontendPull := ImagePullFromCmiiHarborByMap(frontendMap, true)
 
 	// compress image
 	if gzipSplit {
 		for image, tag := range backendMap {
-			ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
+			if !ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
+				errorGzipImageList = append(errorGzipImageList, CmiiHarborPrefix+image+":"+tag)
+			}
 		}
 		for image, tag := range frontendMap {
-			ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
+			if !ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
+				errorGzipImageList = append(errorGzipImageList, CmiiHarborPrefix+image+":"+tag)
+			}
 		}
 	}
 	// upload to harbor
 
 	// clean up images
 
-	return false
+	errorPullImageList = append(errorPullImageList, backendPull...)
+	errorPullImageList = append(errorPullImageList, frontendPull...)
+
+	return errorPullImageList, errorGzipImageList
 }
 
-func FetchDependencyRepos(gzipSplit bool) {
+func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
 	err := os.Mkdir(OfflineImageGzipFolderPrefix, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
@@ -69,8 +76,10 @@ func FetchDependencyRepos(gzipSplit bool) {
 		}
 	}
 
-	ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
+	errorPullImageList, errorGzipImageList = ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
 
-	ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
+	pull, gzipImageList := ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
+
+	return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
 
 }

cmii_operator/CmiiOperator_test.go

@@ -1,11 +1,21 @@
 package cmii_operator
 
-import "testing"
+import (
+	"testing"
+	"wdd.io/agent-go/utils"
+)
 
 func TestFetchDemoImages(t *testing.T) {
-	FetchDemoImages("cqga", true)
+	errorPullImageList, errorGzipImageList := FetchDemoImages("cqga", true)
+
+	utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList")
+	utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList")
 }
 
 func TestFetchDependencyRepos(t *testing.T) {
-	FetchDependencyRepos(true)
+	errorPullImageList, errorGzipImageList := FetchDependencyRepos(true)
+
+	utils.BeautifulPrintListWithTitle(errorPullImageList, "dep errorPullImageList")
+	utils.BeautifulPrintListWithTitle(errorGzipImageList, "dep errorGzipImageList")
+
 }

cmii_operator/DependencyConfig.go

@@ -84,6 +84,7 @@ var Rancher1204Amd64 = []string{
 	"rancher/rancher-webhook:v0.1.0-beta9",
 	"rancher/rancher:v2.5.7",
 	"rancher/rke-tools:v0.1.72",
+	"jerrychina2020/rke-tools:v0.175-linux",
 	"rancher/security-scan:v0.1.14",
 	"rancher/security-scan:v0.2.2",
 	"rancher/shell:v0.1.6",