zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Agent][Operator] - DLTU bug fix

Browse Source
This commit is contained in:
zeaslity
2024-05-08 17:03:08 +08:00
parent 25e0c904f9
commit f1b432fc3a
4 changed files with 86 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
agent-common/utils/FileUtils.go
View File

@@ -131,6 +131,15 @@ func IsDirOrFile(path string) bool {
// FileExists 文件存在返回true，不存在返回false，如果文件是一个目录，也返回false
func FileExists(fileFullPath string) bool {
info, err := os.Stat(fileFullPath)
if err != nil {
return false
}
return !info.IsDir()
}
// FileOrFolderExists 文件或者目录是否返回true，不存在返回false
func FileOrFolderExists(fileFullPath string) bool {
_, err := os.Stat(fileFullPath)
return err == nil
}

2
agent-operator/CmiiMinioOperator.go
View File

@@ -28,7 +28,7 @@ func newInstance() *CmiiMinioOperator {
const (
DefaultLocalEndpoint = "10.250.0.100:9000"
DefaultPublicEndpoint = "42.192.52.227:9000"
DefaultDemoEndpoint = "oss.demo.uavcmlc.com:18000"
DefaultDemoEndpoint = "oss.demo.uavcmlc.com"
DefaultAccessKeyID = "cmii"
DefaultSecretAccessKey = "B#923fC7mk"
DefaultOctopusBucketName = "octopus"

85
agent-operator/CmiiOperator.go
View File

@@ -21,6 +21,7 @@ type ImageSyncEntity struct {
ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式
FullNameImageList []string // 优先级1
DownloadImage bool // 下载镜像
DownloadFromOss bool // 下载镜像
CompressImageToGzip bool // 压缩镜像
UploadToDemoMinio bool // 上传镜像
@@ -76,7 +77,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
// DCU
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(true, allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
}
// 直接传输到目标Harbor仓库
@@ -119,20 +120,43 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
}
// DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode
func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
// write to file
localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt"
// Download
log.Info("DOWNLOAD START !")
if downloadImage {
if fullNameList == nil || len(fullNameList) == 0 {
log.InfoF("no image name list !")
} else {
errorPullImageList = image.PullFromFullNameList(fullNameList)
}
}
// remove failed
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
return slices.Contains(errorPullImageList, imageName)
})
var localGzipFileListTxt string
// Compress
if shouldGzip {
// remove file
_ = os.Remove(localGzipFileListTxt)
gzipFileAlready := make(map[string]bool)
if utils.FileOrFolderExists(gzipFolderFullPath) {
dir, _ := os.ReadDir(gzipFolderFullPath)
for _, entry := range dir {
if entry.IsDir() {
continue
}
gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipFolderFullPath)] = true
}
}
// mkdir folder
err := os.MkdirAll(gzipFolderFullPath, os.ModeDir)
if err != nil {
@@ -146,11 +170,22 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
log.Info("COMPRESS START")
for _, imageFullName := range fullNameList {
ok, gzipImageFileFullPath := image.SaveToGzipFile(imageFullName, gzipFolderFullPath)
// gzip image file already exists
gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName)
gzipImageFileFullPath := gzipFolderFullPath + gzipFileName
_, ok := gzipFileAlready[gzipFileName]
if len(gzipFileAlready) > 0 && ok {
log.DebugF("gzip file %s already exists !", gzipFileName)
} else {
ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipFolderFullPath)
if !ok {
errorGzipImageList = append(errorGzipImageList, imageFullName)
continue
}
}
// 压缩成功
allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath)
}
@@ -159,12 +194,9 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
return slices.Contains(errorGzipImageList, imageName)
})
// write to file
localGzipFileListTxt = gzipFolderFullPath + string(os.PathSeparator) + "all-gzip-image-file-name.txt"
for _, gzipFileFullName := range allGzipFileFullNameList {
utils.AppendContentToFile(
strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath),
strings.TrimPrefix(gzipFileFullName, gzipFolderFullPath)+"\n",
localGzipFileListTxt,
)
}
@@ -185,7 +217,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix)
// upload gzip file list txt to demo
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, localGzipFileListTxt) {
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, strings.TrimPrefix(localGzipFileListTxt, gzipFolderFullPath)) {
log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt)
}
@@ -309,7 +341,7 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
}
// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
@@ -325,7 +357,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO
// do work
// DCU
return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
return DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
}
func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -381,7 +413,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
// do work
// DCU procedure
return DownloadCompressUpload(realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
return DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
}
@@ -422,7 +454,9 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
return realCmiiImageName
}
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
log.Info("DCU for middle and rke!")
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
@@ -430,22 +464,21 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM
}
}
if downloadMiddle {
var fulleImageNameList []string
var gzipFolderPrefix string
gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "middle/"
if isRKE {
log.Info("DCU for rke!")
fulleImageNameList = image.Rancher1204Amd64
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "rke/"
} else {
log.Info("DCU for middle!")
// remove folder first
utils.RemoveFolderComplete(gzipFolderPrefix)
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
fulleImageNameList = image.MiddlewareAmd64
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/"
}
if downloadRke {
gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "rke/"
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
}
return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
return DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss)
}
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {

15
agent-operator/CmiiOperator_test.go
View File

@@ -9,9 +9,20 @@ import (
/* 拉取
*/
func TestFetchDependencyRepos(t *testing.T) {
func TestFetchDependencyRepos_Middle(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, false, true, false)
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, false)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
utils.BeautifulPrintListWithTitle(realCmiiImageName, "realCmiiImageName")
utils.BeautifulPrintListWithTitle(allGzipFileNameList, "allGzipFileNameList")
}
func TestFetchDependencyRepos_RKE(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")