[Agent][Deploy] - 修改ImageSync模块 大量修改
This commit is contained in:
zeaslity
@@ -87,7 +87,23 @@ type UploadResultEntity struct {
|
||||
// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中)
|
||||
func (syncCondition *ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult *ImageSyncResult) {
|
||||
|
||||
imageSyncResult = &ImageSyncResult{}
|
||||
imageSyncResult = &ImageSyncResult{
|
||||
ProcedureSuccessImageList: nil,
|
||||
DownloadResult: &DownloadResultEntity{
|
||||
ErrorPullImageList: nil,
|
||||
SuccessPullImageList: nil,
|
||||
SuccessPullTxtFileLocalFullPath: "",
|
||||
},
|
||||
CompressResult: &CompressResultEntity{
|
||||
ErrorGzipImageList: nil,
|
||||
SuccessGzipImageList: nil,
|
||||
GzipTxtFileLocalFullPath: "",
|
||||
},
|
||||
UploadResult: &UploadResultEntity{
|
||||
ErrorUploadImageList: nil,
|
||||
AllDownloadUrl: nil,
|
||||
},
|
||||
}
|
||||
|
||||
if (syncCondition.DownloadCondition.CmiiNameTagList == nil && syncCondition.DownloadCondition.FullNameImageList == nil) || (len(syncCondition.DownloadCondition.CmiiNameTagList) == 0 && len(syncCondition.DownloadCondition.FullNameImageList) == 0) {
|
||||
// 没有指定特定的镜像，那么根据 ProjectVersion 或者从DEMO拉取镜像
|
||||
@@ -101,7 +117,6 @@ func (syncCondition *ImageSyncEntity) PullFromEntityAndSyncConditionally() (imag
|
||||
} else {
|
||||
// 获取DEMO的镜像
|
||||
C_DownloadCompressUploadFromDemo(syncCondition, imageSyncResult)
|
||||
|
||||
}
|
||||
} else {
|
||||
// 根据列表拉取镜像
|
||||
@@ -171,11 +186,12 @@ func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSync
|
||||
|
||||
gzipLocalFolderPath := syncEntity.CompressCondition.GzipLocalFolder
|
||||
|
||||
localGzipFileListTxt := filepath.Join(gzipLocalFolderPath, AllGzipImageLocalFileName)
|
||||
|
||||
// Compress
|
||||
if syncEntity.CompressCondition.ShouldCompressImageToGzip {
|
||||
|
||||
// remove file
|
||||
localGzipFileListTxt := filepath.Join(gzipLocalFolderPath, AllGzipImageLocalFileName)
|
||||
_ = os.Remove(localGzipFileListTxt)
|
||||
|
||||
// 找到已经存在的压缩文件，跳过
|
||||
@@ -225,22 +241,24 @@ func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSync
|
||||
// 压缩成功
|
||||
allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath)
|
||||
|
||||
syncResult.CompressResult.SuccessGzipImageList = allGzipFileFullNameList
|
||||
syncResult.CompressResult.ErrorGzipImageList = errorGzipImageList
|
||||
|
||||
// remove failed
|
||||
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
|
||||
return slices.Contains(errorGzipImageList, imageName)
|
||||
})
|
||||
|
||||
// write all gzipped file name to file
|
||||
for _, gzipFileFullName := range allGzipFileFullNameList {
|
||||
utils.AppendContentToFile(
|
||||
strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath)+"\n",
|
||||
localGzipFileListTxt,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
syncResult.CompressResult.SuccessGzipImageList = allGzipFileFullNameList
|
||||
syncResult.CompressResult.ErrorGzipImageList = errorGzipImageList
|
||||
|
||||
// remove failed
|
||||
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
|
||||
return slices.Contains(errorGzipImageList, imageName)
|
||||
})
|
||||
|
||||
// write all gzipped file name to file
|
||||
for _, gzipFileFullName := range allGzipFileFullNameList {
|
||||
utils.AppendContentToFile(
|
||||
strings.TrimPrefix(strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath), "/")+"\n",
|
||||
localGzipFileListTxt,
|
||||
)
|
||||
}
|
||||
|
||||
} else {
|
||||
// 压缩为一个大的压缩包
|
||||
gzipFileName := generateMonolithicGzipFileName(syncEntity)
|
||||
@@ -251,10 +269,7 @@ func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSync
|
||||
}
|
||||
|
||||
// write all gzipped file name to file
|
||||
utils.AppendContentToFile(
|
||||
utils.BeautifulPrintToString(fullNameList),
|
||||
localGzipFileListTxt,
|
||||
)
|
||||
utils.AppendOverwriteListContentToFile(fullNameList, localGzipFileListTxt)
|
||||
|
||||
// remove failed
|
||||
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
|
||||
@@ -280,6 +295,21 @@ func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSync
|
||||
// get gzip file name list
|
||||
log.Info("[DCU] - UPLOAD OSS START !")
|
||||
|
||||
if !syncEntity.CompressCondition.ShouldCompressImageToGzip {
|
||||
// 没有压缩指令 直接上传已有的内容
|
||||
allFileInGzipFile, err := utils.ListAllFileInFolderWithFullPath(gzipLocalFolderPath)
|
||||
if err != nil {
|
||||
log.ErrorF("[DCU] - list all gzip file error !")
|
||||
return
|
||||
}
|
||||
for _, f := range allFileInGzipFile {
|
||||
if strings.HasSuffix(f, "tar.gz") {
|
||||
syncResult.CompressResult.SuccessGzipImageList = append(syncResult.CompressResult.SuccessGzipImageList, f)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var errorUploadOssGzipNameList []string
|
||||
var allDownloadUrl []string
|
||||
|
||||
@@ -288,20 +318,27 @@ func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSync
|
||||
// 根据本地保存Gzip的目录路径提取到 相应的后缀 项目代码
|
||||
// projectName / projectVersion
|
||||
projectUniqueName := strings.TrimPrefix(gzipLocalFolderPath, image.OfflineImageGzipFolderPrefix)
|
||||
projectUniqueName = strings.TrimSuffix(projectUniqueName, "/")
|
||||
|
||||
bucketNameWithPrefix := "cmlc-installation/" + projectUniqueName
|
||||
log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix)
|
||||
|
||||
// upload gzip file list txt to demo
|
||||
localGzipFileListTxt := syncResult.CompressResult.GzipTxtFileLocalFullPath
|
||||
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, strings.TrimPrefix(localGzipFileListTxt, gzipLocalFolderPath)) {
|
||||
log.ErrorF("upload of %s to demo oss error !", localGzipFileListTxt)
|
||||
// 上传所有的压缩文件名称
|
||||
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, AllGzipImageLocalFileName) {
|
||||
log.ErrorF("upload of %s to demo oss error !", AllGzipImageLocalFileName)
|
||||
}
|
||||
|
||||
// 上传所有的镜像名称
|
||||
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, AllCmiiImageListLocalFileName) {
|
||||
log.ErrorF("upload of %s to demo oss error !", AllCmiiImageListLocalFileName)
|
||||
}
|
||||
|
||||
log.InfoF("upload all gzip file to demo oss !")
|
||||
for _, gzipFileFullName := range syncResult.CompressResult.SuccessGzipImageList {
|
||||
// SaveToGzipFile 返回的是全路径 归一化处理 gzip file name
|
||||
gzipFileName := strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath)
|
||||
gzipFileName = strings.TrimPrefix(gzipFileName, "/")
|
||||
|
||||
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, gzipFileName) {
|
||||
log.ErrorF("upload of %s to demo oss error !", gzipFileName)
|
||||
errorUploadOssGzipNameList = append(errorUploadOssGzipNameList, gzipFileName)
|
||||
@@ -315,11 +352,12 @@ func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSync
|
||||
|
||||
}
|
||||
|
||||
utils.AppendContentToFile(utils.BeautifulPrintToString(syncResult), filepath.Join(gzipLocalFolderPath, utils.TimeSplitFormatString()+".json"))
|
||||
}
|
||||
|
||||
func generateMonolithicGzipFileName(syncEntity *ImageSyncEntity) string {
|
||||
|
||||
return strings.TrimPrefix(syncEntity.CompressCondition.GzipLocalFolder, image.OfflineImageGzipFolderPrefix)
|
||||
return strings.TrimPrefix(syncEntity.CompressCondition.GzipLocalFolder, image.OfflineImageGzipFolderPrefix) + ".tar.gz"
|
||||
}
|
||||
|
||||
// A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
|
||||
@@ -442,14 +480,15 @@ func C_DownloadCompressUploadFromDemo(syncEntity *ImageSyncEntity, syncResult *I
|
||||
}
|
||||
}
|
||||
|
||||
syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath
|
||||
|
||||
// get demo image version map
|
||||
allCmiiImageFullNameList := buildAllCmiiImageNameListFromDemo(projectName)
|
||||
|
||||
// save all cmii image to file
|
||||
allPullImageNameTxtFileName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName)
|
||||
utils.AppendOverwriteContentToFile(utils.BeautifulPrintToString(allCmiiImageFullNameList), allPullImageNameTxtFileName)
|
||||
utils.AppendOverwriteListContentToFile(allCmiiImageFullNameList, allPullImageNameTxtFileName)
|
||||
|
||||
syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath
|
||||
syncEntity.DownloadCondition.FullNameImageList = allCmiiImageFullNameList
|
||||
|
||||
// save to result
|
||||
syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allPullImageNameTxtFileName
|
||||
@@ -593,7 +632,7 @@ func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, isRKE b
|
||||
|
||||
syncEntity := &ImageSyncEntity{
|
||||
DownloadCondition: &DownloadEntity{
|
||||
ShouldDownloadImage: false,
|
||||
ShouldDownloadImage: true,
|
||||
ProjectName: "",
|
||||
ProjectVersion: "",
|
||||
CmiiNameTagList: nil,
|
||||
@@ -613,7 +652,25 @@ func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, isRKE b
|
||||
DirectHarborHost: "",
|
||||
}
|
||||
|
||||
syncResult := &ImageSyncResult{}
|
||||
syncResult := &ImageSyncResult{
|
||||
ProcedureSuccessImageList: nil,
|
||||
DownloadResult: &DownloadResultEntity{
|
||||
ErrorPullImageList: nil,
|
||||
SuccessPullImageList: nil,
|
||||
SuccessPullTxtFileLocalFullPath: "",
|
||||
},
|
||||
CompressResult: &CompressResultEntity{
|
||||
ErrorGzipImageList: nil,
|
||||
SuccessGzipImageList: nil,
|
||||
GzipTxtFileLocalFullPath: "",
|
||||
},
|
||||
UploadResult: &UploadResultEntity{
|
||||
ErrorUploadImageList: nil,
|
||||
AllDownloadUrl: nil,
|
||||
},
|
||||
}
|
||||
|
||||
utils.AppendOverwriteListContentToFile(fullImageNameList, filepath.Join(gzipFolderPrefix, AllCmiiImageListLocalFileName))
|
||||
|
||||
A_DownloadCompressUpload(syncEntity, syncResult)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user