zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Operator ] - optimize Image pull

Browse Source
This commit is contained in:
zeaslity
2024-01-25 12:07:17 +08:00
parent a09515c82d
commit a0369bda1e
5 changed files with 71 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
cmii_operator/CmiiImageSync.go
View File

@@ -6,15 +6,13 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/klauspost/pgzip"
"io"
"os"
"strings"
"wdd.io/cmii_operator/tools"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
)
var apiClient = newClient()
@@ -158,35 +156,10 @@ func ImagePullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
return pullResult
}
func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) {
func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (errorPullImageList []string) {
var fs uintptr
for image, tag := range imageVersionMap {
s := CmiiHarborPrefix + image + ":" + tag
pullResult := ImagePullFromCmiiHarbor(s)
if pullResult == nil {
continue
}
if silentMode {
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
} else {
_ = tools.DisplayJSONMessagesStream(pullResult, os.Stdout, fs, true, func(message tools.JSONMessage) {
})
}
fmt.Println()
fmt.Println()
}
fullImageNameList := convertCMiiImageMapToList(imageVersionMap)
return ImagePullFromFullNameList(fullImageNameList)
}
@@ -225,13 +198,14 @@ func ImagePullFromFileJson(filePathName string) {
}
func ImagePullFromList(depContainerList []string) {
for _, dep := range depContainerList {
func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
for _, dep := range fullImageNameList {
loginToDockerHub()
pullResult := ImagePullFromCmiiHarbor(dep)
if pullResult == nil {
errorPullImageList = append(errorPullImageList, dep)
continue
}
scanner := bufio.NewScanner(pullResult)
@@ -246,11 +220,13 @@ func ImagePullFromList(depContainerList []string) {
}
fmt.Println()
}
return errorPullImageList
}
func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string) {
func ImagePullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
ImagePullFromList(imageNameList)
errorPullImageList = ImagePullFromFullNameList(fullImageNameList)
// generate a project folder
err := os.MkdirAll(gzipFolder, os.ModeDir)
@@ -260,10 +236,13 @@ func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string
}
}
for _, image := range imageNameList {
ImageSaveToTarGZ(image, gzipFolder)
for _, image := range fullImageNameList {
if !ImageSaveToTarGZ(image, gzipFolder) {
errorGzipImageList = append(errorGzipImageList, image)
}
}
return errorPullImageList, errorGzipImageList
}
func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
@@ -350,6 +329,16 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
return gzipFileName
}
func convertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
for image, tag := range cmiiImageVersionMap {
s := CmiiHarborPrefix + image + ":" + tag
fullImageNameList = append(fullImageNameList, s)
}
return fullImageNameList
}
func loginToDockerHub() {
login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{

29
cmii_operator/CmiiOperator.go
View File

@@ -9,14 +9,14 @@ import (
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
func FetchDemoImages(projectName string, gzipSplit bool) bool {
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
// generate a project folder
err := os.Mkdir(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
return false
return errorPullImageList, errorGzipImageList
}
}
@@ -42,26 +42,33 @@ func FetchDemoImages(projectName string, gzipSplit bool) bool {
)
// download image
ImagePullFromCmiiHarborByMap(backendMap, true)
ImagePullFromCmiiHarborByMap(frontendMap, true)
backendPull := ImagePullFromCmiiHarborByMap(backendMap, true)
frontendPull := ImagePullFromCmiiHarborByMap(frontendMap, true)
// compress image
if gzipSplit {
for image, tag := range backendMap {
ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
if !ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, CmiiHarborPrefix+image+":"+tag)
}
}
for image, tag := range frontendMap {
ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
if !ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, CmiiHarborPrefix+image+":"+tag)
}
}
}
// upload to harbor
// clean up images
return false
errorPullImageList = append(errorPullImageList, backendPull...)
errorPullImageList = append(errorPullImageList, frontendPull...)
return errorPullImageList, errorGzipImageList
}
func FetchDependencyRepos(gzipSplit bool) {
func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
err := os.Mkdir(OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
@@ -69,8 +76,10 @@ func FetchDependencyRepos(gzipSplit bool) {
}
}
ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
errorPullImageList, errorGzipImageList = ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
pull, gzipImageList := ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
}

16
cmii_operator/CmiiOperator_test.go
View File

@@ -1,11 +1,21 @@
package cmii_operator
import "testing"
import (
"testing"
"wdd.io/agent-go/utils"
)
func TestFetchDemoImages(t *testing.T) {
FetchDemoImages("cqga", true)
errorPullImageList, errorGzipImageList := FetchDemoImages("cqga", true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList")
}
func TestFetchDependencyRepos(t *testing.T) {
FetchDependencyRepos(true)
errorPullImageList, errorGzipImageList := FetchDependencyRepos(true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "dep errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "dep errorGzipImageList")
}

1
cmii_operator/DependencyConfig.go
View File

@@ -84,6 +84,7 @@ var Rancher1204Amd64 = []string{
"rancher/rancher-webhook:v0.1.0-beta9",
"rancher/rancher:v2.5.7",
"rancher/rke-tools:v0.1.72",
"jerrychina2020/rke-tools:v0.175-linux",
"rancher/security-scan:v0.1.14",
"rancher/security-scan:v0.2.2",
"rancher/shell:v0.1.6",