zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Agent][Operator] - DLTU bug fix

Browse Source
This commit is contained in:
zeaslity
2024-05-08 17:03:08 +08:00
parent 25e0c904f9
commit f1b432fc3a
4 changed files with 86 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
agent-common/utils/FileUtils.go
View File

@@ -131,6 +131,15 @@ func IsDirOrFile(path string) bool {
// FileExists 文件存在返回true，不存在返回false，如果文件是一个目录，也返回false // FileExists 文件存在返回true，不存在返回false，如果文件是一个目录，也返回false
func FileExists(fileFullPath string) bool { func FileExists(fileFullPath string) bool {
info, err := os.Stat(fileFullPath)
if err != nil {
return false
}
return !info.IsDir()
}
// FileOrFolderExists 文件或者目录是否返回true，不存在返回false
func FileOrFolderExists(fileFullPath string) bool {
_, err := os.Stat(fileFullPath) _, err := os.Stat(fileFullPath)
return err == nil return err == nil
} }

2
agent-operator/CmiiMinioOperator.go
View File

@@ -28,7 +28,7 @@ func newInstance() *CmiiMinioOperator {
const ( const (
DefaultLocalEndpoint = "10.250.0.100:9000" DefaultLocalEndpoint = "10.250.0.100:9000"
DefaultPublicEndpoint = "42.192.52.227:9000" DefaultPublicEndpoint = "42.192.52.227:9000"
DefaultDemoEndpoint = "oss.demo.uavcmlc.com:18000" DefaultDemoEndpoint = "oss.demo.uavcmlc.com"
DefaultAccessKeyID = "cmii" DefaultAccessKeyID = "cmii"
DefaultSecretAccessKey = "B#923fC7mk" DefaultSecretAccessKey = "B#923fC7mk"
DefaultOctopusBucketName = "octopus" DefaultOctopusBucketName = "octopus"

85
agent-operator/CmiiOperator.go
View File

@@ -21,6 +21,7 @@ type ImageSyncEntity struct {
ProjectVersion string // 优先级2 ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式 CmiiNameTagList []string // 优先级1 appName:tag的形式
FullNameImageList []string // 优先级1 FullNameImageList []string // 优先级1
DownloadImage bool // 下载镜像
DownloadFromOss bool // 下载镜像 DownloadFromOss bool // 下载镜像
CompressImageToGzip bool // 压缩镜像 CompressImageToGzip bool // 压缩镜像
UploadToDemoMinio bool // 上传镜像 UploadToDemoMinio bool // 上传镜像
@@ -76,7 +77,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList) allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
// DCU // DCU
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio) errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(true, allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
} }
// 直接传输到目标Harbor仓库 // 直接传输到目标Harbor仓库
@@ -119,20 +120,43 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
} }
// DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode // DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode
func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
// write to file
localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt"
// Download // Download
log.Info("DOWNLOAD START !") log.Info("DOWNLOAD START !")
if downloadImage {
if fullNameList == nil || len(fullNameList) == 0 {
log.InfoF("no image name list !")
} else {
errorPullImageList = image.PullFromFullNameList(fullNameList) errorPullImageList = image.PullFromFullNameList(fullNameList)
}
}
// remove failed // remove failed
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool { fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
return slices.Contains(errorPullImageList, imageName) return slices.Contains(errorPullImageList, imageName)
}) })
var localGzipFileListTxt string
// Compress // Compress
if shouldGzip { if shouldGzip {
// remove file
_ = os.Remove(localGzipFileListTxt)
gzipFileAlready := make(map[string]bool)
if utils.FileOrFolderExists(gzipFolderFullPath) {
dir, _ := os.ReadDir(gzipFolderFullPath)
for _, entry := range dir {
if entry.IsDir() {
continue
}
gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipFolderFullPath)] = true
}
}
// mkdir folder // mkdir folder
err := os.MkdirAll(gzipFolderFullPath, os.ModeDir) err := os.MkdirAll(gzipFolderFullPath, os.ModeDir)
if err != nil { if err != nil {
@@ -146,11 +170,22 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
log.Info("COMPRESS START") log.Info("COMPRESS START")
for _, imageFullName := range fullNameList { for _, imageFullName := range fullNameList {
ok, gzipImageFileFullPath := image.SaveToGzipFile(imageFullName, gzipFolderFullPath)
// gzip image file already exists
gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName)
gzipImageFileFullPath := gzipFolderFullPath + gzipFileName
_, ok := gzipFileAlready[gzipFileName]
if len(gzipFileAlready) > 0 && ok {
log.DebugF("gzip file %s already exists !", gzipFileName)
} else {
ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipFolderFullPath)
if !ok { if !ok {
errorGzipImageList = append(errorGzipImageList, imageFullName) errorGzipImageList = append(errorGzipImageList, imageFullName)
continue continue
} }
}
// 压缩成功 // 压缩成功
allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath) allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath)
} }
@@ -159,12 +194,9 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
return slices.Contains(errorGzipImageList, imageName) return slices.Contains(errorGzipImageList, imageName)
}) })
// write to file
localGzipFileListTxt = gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt"
for _, gzipFileFullName := range allGzipFileFullNameList { for _, gzipFileFullName := range allGzipFileFullNameList {
utils.AppendContentToFile( utils.AppendContentToFile(
strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath), strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)+"\n",
localGzipFileListTxt, localGzipFileListTxt,
) )
} }
@@ -185,7 +217,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix) log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix)
// upload gzip file list txt to demo // upload gzip file list txt to demo
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, localGzipFileListTxt) { if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, strings.TrimPrefix(localGzipFileListTxt, gzipFolderFullPath)) {
log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt) log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt)
} }
@@ -309,7 +341,7 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
} }
// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像 // DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder // generate a project folder
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir) err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
@@ -325,7 +357,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO
// do work // do work
// DCU // DCU
return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) return DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
} }
func buildAllCmiiImageNameListFromDemo(projectName string) []string { func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -381,7 +413,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
// do work // do work
// DCU procedure // DCU procedure
return DownloadCompressUpload(realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) return DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
} }
@@ -422,7 +454,9 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
return realCmiiImageName return realCmiiImageName
} }
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
log.Info("DCU for middle and rke!")
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir) err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil { if err != nil {
if !errors.Is(err, os.ErrExist) { if !errors.Is(err, os.ErrExist) {
@@ -430,22 +464,21 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM
} }
} }
if downloadMiddle { var fulleImageNameList []string
var gzipFolderPrefix string
gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "middle/" if isRKE {
log.Info("DCU for rke!")
fulleImageNameList = image.Rancher1204Amd64
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "rke/"
} else {
log.Info("DCU for middle!")
// remove folder first fulleImageNameList = image.MiddlewareAmd64
utils.RemoveFolderComplete(gzipFolderPrefix) gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/"
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
} }
if downloadRke { return DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss)
gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "rke/"
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
}
return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
} }
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {

15
agent-operator/CmiiOperator_test.go
View File

@@ -9,9 +9,20 @@ import (
/* 拉取 /* 拉取
*/ */
func TestFetchDependencyRepos(t *testing.T) { func TestFetchDependencyRepos_Middle(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, false, true, false) errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, false)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
utils.BeautifulPrintListWithTitle(realCmiiImageName, "realCmiiImageName")
utils.BeautifulPrintListWithTitle(allGzipFileNameList, "allGzipFileNameList")
}
func TestFetchDependencyRepos_RKE(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")