初始化项目
This commit is contained in:
zeaslity
719
agent-operator/CmiiImageSyncOperator.go
Normal file
719
agent-operator/CmiiImageSyncOperator.go
Normal file
@@ -0,0 +1,719 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
image2 "wdd.io/agent-common/image"
|
||||
"wdd.io/agent-common/utils"
|
||||
"wdd.io/agent-deploy/d_app"
|
||||
"wdd.io/agent-operator/config"
|
||||
"wdd.io/agent-operator/image"
|
||||
"wdd.io/agent-operator/minio"
|
||||
)
|
||||
|
||||
const (
|
||||
OfflineDeployHarborHost = "harbor.wdd.io"
|
||||
PublicDeployHarborHost = "42.192.52.227"
|
||||
DirectPushDeployHarborHost = "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn"
|
||||
|
||||
AllCmiiImageListLocalFileName = "all-cmii-image-list.txt"
|
||||
AllGzipImageLocalFileName = "all-gzip-image-list.txt"
|
||||
)
|
||||
|
||||
type ImageSyncEntity struct {
|
||||
DownloadCondition *DownloadEntity // D的条件
|
||||
CompressCondition *CompressEntity // C的条件
|
||||
UploadCondition *UploadEntity // U的条件
|
||||
|
||||
ShouldDownloadFromOss bool // 下载镜像 DLTU中的D
|
||||
ShouldUpdateImageTag bool // 更新镜像 DLTU中的U
|
||||
|
||||
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机
|
||||
DirectHarborHost string // IP:Port or 域名:PORT 不带http前缀
|
||||
}
|
||||
|
||||
// DownloadEntity DCU中的D的条件
|
||||
type DownloadEntity struct {
|
||||
ShouldDownloadImage bool // 下载镜像 DCU中的D 实际无用
|
||||
|
||||
ProjectName string // 优先级3 优先级最低 从DEMO拉取镜像
|
||||
ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像
|
||||
|
||||
CmiiNameTagList []string // 优先级1 appName:tag 会被转换为FullNameImageList
|
||||
FullNameImageList []string // 优先级1 优先下载此类型
|
||||
|
||||
DownloadAuthUserName string // 下载需要认证的用户名
|
||||
DownloadAuthPassword string // 下载需要认证的密码
|
||||
}
|
||||
|
||||
// CompressEntity DCU中的C的条件
|
||||
type CompressEntity struct {
|
||||
ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C
|
||||
ShouldGzipSplit bool // 压缩镜像 是否应该分割存储 true=独立存储 false=整个存储
|
||||
GzipLocalFolder string // 压缩镜像 保存压缩镜像文件的本地目录
|
||||
}
|
||||
|
||||
// UploadEntity DCU中的U的条件
|
||||
type UploadEntity struct {
|
||||
ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U
|
||||
}
|
||||
|
||||
type DirectPushResultEntity struct {
|
||||
ErrorPushedImageList []string
|
||||
}
|
||||
|
||||
type ImageSyncResult struct {
|
||||
ProcedureSuccessImageList []string // 经过特定步骤之后成功的镜像
|
||||
|
||||
DownloadResult *DownloadResultEntity
|
||||
CompressResult *CompressResultEntity
|
||||
UploadResult *UploadResultEntity
|
||||
|
||||
DirectPushResult *DirectPushResultEntity
|
||||
}
|
||||
|
||||
type DownloadResultEntity struct {
|
||||
ErrorPullImageList []string // 下载镜像 DCU中的D 下载失败的镜像
|
||||
SuccessPullImageList []string // 下载镜像 DCU中的D 下载成功的镜像
|
||||
SuccessPullTxtFileLocalFullPath string // 下载镜像 DCU中的D 下载成功的镜像保存的文件地址 GzipLocalFolder + all
|
||||
}
|
||||
|
||||
type CompressResultEntity struct {
|
||||
ErrorGzipImageList []string // 压缩镜像 DCU中的C 压缩失败的镜像
|
||||
SuccessGzipImageList []string // 压缩镜像 DCU中的C 压缩成功的镜像
|
||||
GzipTxtFileLocalFullPath string // 压缩镜像 DCU中的C 压缩镜像保存的目录
|
||||
}
|
||||
|
||||
type UploadResultEntity struct {
|
||||
ErrorUploadImageList []string // 上传镜像 DCU中的U 上传失败的镜像
|
||||
AllDownloadUrl []string // 上传镜像 DCU中的U 正式的下载地址列表i
|
||||
}
|
||||
|
||||
// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器(或者上传的minio中)
|
||||
func (syncCondition *ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult *ImageSyncResult) {
|
||||
|
||||
// 初始化 imageSync
|
||||
imageSyncResult = &ImageSyncResult{
|
||||
ProcedureSuccessImageList: nil,
|
||||
DownloadResult: &DownloadResultEntity{
|
||||
ErrorPullImageList: nil,
|
||||
SuccessPullImageList: nil,
|
||||
SuccessPullTxtFileLocalFullPath: "",
|
||||
},
|
||||
CompressResult: &CompressResultEntity{
|
||||
ErrorGzipImageList: nil,
|
||||
SuccessGzipImageList: nil,
|
||||
GzipTxtFileLocalFullPath: "",
|
||||
},
|
||||
UploadResult: &UploadResultEntity{
|
||||
ErrorUploadImageList: nil,
|
||||
AllDownloadUrl: nil,
|
||||
},
|
||||
DirectPushResult: &DirectPushResultEntity{
|
||||
ErrorPushedImageList: nil,
|
||||
},
|
||||
}
|
||||
|
||||
if (syncCondition.DownloadCondition.CmiiNameTagList == nil && syncCondition.DownloadCondition.FullNameImageList == nil) || (len(syncCondition.DownloadCondition.CmiiNameTagList) == 0 && len(syncCondition.DownloadCondition.FullNameImageList) == 0) {
|
||||
// 没有指定特定的镜像，那么根据 ProjectVersion 或者从DEMO拉取镜像
|
||||
|
||||
// pull images
|
||||
// compress
|
||||
if syncCondition.DownloadCondition.ProjectVersion != "" {
|
||||
|
||||
// 获取特定版本的镜像
|
||||
C_DownloadCompressUploadFromVersion(syncCondition, imageSyncResult)
|
||||
|
||||
} else {
|
||||
// 获取DEMO的镜像
|
||||
C_DownloadCompressUploadFromDemo(syncCondition, imageSyncResult)
|
||||
}
|
||||
} else {
|
||||
// 根据列表拉取镜像
|
||||
|
||||
// 组装镜像名称
|
||||
syncCondition.DownloadCondition.FullNameImageList = concatAndUniformCmiiImage(syncCondition.DownloadCondition.FullNameImageList, syncCondition.DownloadCondition.CmiiNameTagList)
|
||||
|
||||
// gzip file folder path
|
||||
syncCondition.CompressCondition.GzipLocalFolder = filepath.Join(image.OfflineImageGzipFolderPrefix, "tmp")
|
||||
|
||||
// DCU
|
||||
A_DownloadCompressUpload(syncCondition, imageSyncResult)
|
||||
}
|
||||
|
||||
// 直接传输到目标Harbor仓库
|
||||
if syncCondition.ShouldDirectPushToHarbor {
|
||||
if syncCondition.DirectHarborHost == "" {
|
||||
log.ErrorF("DirectHarborHost is null ! can't push to target harbor !")
|
||||
}
|
||||
// push to
|
||||
imageSyncResult.DirectPushResult.ErrorPushedImageList = image.TagFromListAndPushToCHarbor(imageSyncResult.ProcedureSuccessImageList, syncCondition.DirectHarborHost)
|
||||
}
|
||||
|
||||
// build result
|
||||
|
||||
return imageSyncResult
|
||||
}
|
||||
|
||||
func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) []string {
|
||||
|
||||
if cmiiImageList != nil || len(cmiiImageList) > 0 {
|
||||
// cmiiImageList has content
|
||||
if fullImageList == nil {
|
||||
fullImageList = []string{}
|
||||
}
|
||||
|
||||
for _, cmiiImage := range cmiiImageList {
|
||||
fullImageList = append(fullImageList, image2.CmiiHarborPrefix+cmiiImage)
|
||||
}
|
||||
}
|
||||
|
||||
return fullImageList
|
||||
}
|
||||
|
||||
// A_DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode
|
||||
func A_DownloadCompressUpload(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) {
|
||||
|
||||
// all image full name list need to download
|
||||
fullNameList := syncEntity.DownloadCondition.FullNameImageList
|
||||
|
||||
// Download
|
||||
log.Info("[DCU] - DOWNLOAD START !")
|
||||
if syncEntity.DownloadCondition.ShouldDownloadImage && fullNameList != nil && len(fullNameList) > 0 {
|
||||
|
||||
syncResult.DownloadResult.ErrorPullImageList = image.PullFromFullNameList(fullNameList)
|
||||
|
||||
// remove failed download image from full name list
|
||||
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
|
||||
return slices.Contains(syncResult.DownloadResult.ErrorPullImageList, imageName)
|
||||
})
|
||||
|
||||
} else {
|
||||
log.Info("[DCU] - No Need To Download !")
|
||||
}
|
||||
|
||||
syncResult.ProcedureSuccessImageList = fullNameList
|
||||
|
||||
gzipLocalFolderPath := syncEntity.CompressCondition.GzipLocalFolder
|
||||
|
||||
localGzipFileListTxt := filepath.Join(gzipLocalFolderPath, AllGzipImageLocalFileName)
|
||||
|
||||
// Compress
|
||||
if syncEntity.CompressCondition.ShouldCompressImageToGzip {
|
||||
|
||||
// 移除Gzip的txt
|
||||
_ = os.Remove(localGzipFileListTxt)
|
||||
|
||||
// 找到已经存在的压缩文件，跳过
|
||||
gzipFileAlready := make(map[string]bool)
|
||||
if utils.FileOrFolderExists(gzipLocalFolderPath) {
|
||||
dir, _ := os.ReadDir(gzipLocalFolderPath)
|
||||
for _, entry := range dir {
|
||||
if entry.IsDir() {
|
||||
continue
|
||||
}
|
||||
gzipFileAlready[strings.TrimPrefix(entry.Name(), gzipLocalFolderPath)] = true
|
||||
}
|
||||
}
|
||||
|
||||
// mkdir folder
|
||||
err := os.MkdirAll(gzipLocalFolderPath, os.ModeDir)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
log.ErrorF("create folder error of %s", gzipLocalFolderPath)
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 循环遍历压缩
|
||||
log.Info("[DCU] - COMPRESS START")
|
||||
var errorGzipImageList []string
|
||||
var allGzipFileFullNameList []string
|
||||
|
||||
if syncEntity.CompressCondition.ShouldGzipSplit {
|
||||
// 独立压缩
|
||||
for _, imageFullName := range fullNameList {
|
||||
|
||||
// gzip image file already exists
|
||||
gzipFileName := image2.ImageFullNameToGzipFileName(imageFullName)
|
||||
gzipImageFileFullPath := gzipLocalFolderPath + gzipFileName
|
||||
_, ok := gzipFileAlready[gzipFileName]
|
||||
if len(gzipFileAlready) > 0 && ok {
|
||||
log.DebugF("gzip file %s already exists !", gzipFileName)
|
||||
} else {
|
||||
ok, gzipImageFileFullPath = image.SaveToGzipFile(imageFullName, gzipLocalFolderPath)
|
||||
if !ok {
|
||||
errorGzipImageList = append(errorGzipImageList, imageFullName)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// 压缩成功
|
||||
allGzipFileFullNameList = append(allGzipFileFullNameList, gzipImageFileFullPath)
|
||||
|
||||
}
|
||||
|
||||
syncResult.CompressResult.SuccessGzipImageList = allGzipFileFullNameList
|
||||
syncResult.CompressResult.ErrorGzipImageList = errorGzipImageList
|
||||
|
||||
// remove failed
|
||||
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
|
||||
return slices.Contains(errorGzipImageList, imageName)
|
||||
})
|
||||
|
||||
// write all gzipped file name to file
|
||||
for _, gzipFileFullName := range allGzipFileFullNameList {
|
||||
utils.AppendContentToFile(
|
||||
strings.TrimPrefix(strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath), "/")+"\n",
|
||||
localGzipFileListTxt,
|
||||
)
|
||||
}
|
||||
|
||||
} else {
|
||||
// 压缩为一个大的压缩包
|
||||
gzipFileName := generateMonolithicGzipFileName(syncEntity)
|
||||
|
||||
ok, gzipFileFullPath, errorGzipImageListTmp := image.SaveImageListToGzipFile(fullNameList, gzipLocalFolderPath, gzipFileName)
|
||||
if !ok {
|
||||
panic("[DCU] - gzip error to a monolithic file !")
|
||||
}
|
||||
|
||||
// write all gzipped file name to file
|
||||
utils.AppendOverwriteListContentToFile(fullNameList, localGzipFileListTxt)
|
||||
|
||||
// remove failed
|
||||
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
|
||||
return slices.Contains(errorGzipImageListTmp, imageName)
|
||||
})
|
||||
|
||||
syncResult.CompressResult.SuccessGzipImageList = fullNameList
|
||||
syncResult.CompressResult.ErrorGzipImageList = errorGzipImageListTmp
|
||||
|
||||
log.InfoF("[DCU] - gzip all image from list to monolithic file %s", gzipFileFullPath)
|
||||
}
|
||||
|
||||
syncResult.CompressResult.GzipTxtFileLocalFullPath = localGzipFileListTxt
|
||||
|
||||
log.InfoF("[DCU] - all gzip file name list is %s", localGzipFileListTxt)
|
||||
}
|
||||
|
||||
syncResult.ProcedureSuccessImageList = fullNameList
|
||||
|
||||
// Upload
|
||||
if syncEntity.UploadCondition.ShouldUploadToDemoMinio {
|
||||
//uploadGzipFileToDemoMinio()
|
||||
// get gzip file name list
|
||||
log.Info("[DCU] - UPLOAD OSS START !")
|
||||
|
||||
if !syncEntity.CompressCondition.ShouldCompressImageToGzip {
|
||||
// 没有压缩指令 直接上传已有的内容
|
||||
allFileInGzipFile, err := utils.ListAllFileInFolderWithFullPath(gzipLocalFolderPath)
|
||||
if err != nil {
|
||||
log.ErrorF("[DCU] - list all gzip file error !")
|
||||
return
|
||||
}
|
||||
for _, f := range allFileInGzipFile {
|
||||
if strings.HasSuffix(f, "tar.gz") {
|
||||
syncResult.CompressResult.SuccessGzipImageList = append(syncResult.CompressResult.SuccessGzipImageList, f)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
var errorUploadOssGzipNameList []string
|
||||
var allDownloadUrl []string
|
||||
|
||||
// start to upload
|
||||
// extract demo oss location suffix from gzipFolderFullPath
|
||||
// 根据本地保存Gzip的目录路径提取到 相应的后缀 项目代码
|
||||
// projectName / projectVersion
|
||||
projectUniqueName := strings.TrimPrefix(gzipLocalFolderPath, image.OfflineImageGzipFolderPrefix)
|
||||
projectUniqueName = strings.TrimSuffix(projectUniqueName, "/")
|
||||
|
||||
bucketNameWithPrefix := "cmlc-installation/" + projectUniqueName
|
||||
log.InfoF("gzip file location in demo oss is %s", minio.DefaultDemoEndpoint+"/"+bucketNameWithPrefix)
|
||||
|
||||
// 上传所有的压缩文件名称
|
||||
if !minio.DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, AllGzipImageLocalFileName) {
|
||||
log.ErrorF("upload of %s to demo oss error !", AllGzipImageLocalFileName)
|
||||
}
|
||||
|
||||
// 上传所有的镜像名称
|
||||
if !minio.DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, AllCmiiImageListLocalFileName) {
|
||||
log.ErrorF("upload of %s to demo oss error !", AllCmiiImageListLocalFileName)
|
||||
}
|
||||
|
||||
log.InfoF("upload all gzip file to demo oss !")
|
||||
for _, gzipFileFullName := range syncResult.CompressResult.SuccessGzipImageList {
|
||||
// SaveToGzipFile 返回的是全路径 归一化处理 gzip file name
|
||||
gzipFileName := strings.TrimPrefix(gzipFileFullName, gzipLocalFolderPath)
|
||||
gzipFileName = strings.TrimPrefix(gzipFileName, "/")
|
||||
|
||||
if !minio.DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipLocalFolderPath, gzipFileName) {
|
||||
log.ErrorF("upload of %s to demo oss error !", gzipFileName)
|
||||
errorUploadOssGzipNameList = append(errorUploadOssGzipNameList, gzipFileName)
|
||||
} else {
|
||||
allDownloadUrl = append(allDownloadUrl, minio.DefaultDemoEndpoint+"/"+bucketNameWithPrefix+"/"+gzipFileName)
|
||||
}
|
||||
}
|
||||
|
||||
syncResult.UploadResult.AllDownloadUrl = allDownloadUrl
|
||||
syncResult.UploadResult.ErrorUploadImageList = errorUploadOssGzipNameList
|
||||
|
||||
}
|
||||
|
||||
utils.AppendContentToFile(utils.BeautifulPrintToString(syncResult), filepath.Join(gzipLocalFolderPath, utils.TimeSplitFormatString()+".json"))
|
||||
}
|
||||
|
||||
func generateMonolithicGzipFileName(syncEntity *ImageSyncEntity) string {
|
||||
|
||||
return strings.TrimPrefix(syncEntity.CompressCondition.GzipLocalFolder, image.OfflineImageGzipFolderPrefix) + ".tar.gz"
|
||||
}
|
||||
|
||||
// A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
|
||||
// 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录
|
||||
// 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/
|
||||
// 3. 读取本机的IP地址 - 参数传递
|
||||
// 4. OSS地址 - ossUrlPrefix传空 则使用默认值
|
||||
// 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式
|
||||
func A_DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {
|
||||
|
||||
// 支持单文件的形式
|
||||
if !utils.IsDirOrFile(localGzipFolderOrGzipFile) {
|
||||
// 单个压缩文件 肯定是离线的形式
|
||||
if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") {
|
||||
log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
// load
|
||||
image.LoadFromGzipFilePath(localGzipFolderOrGzipFile)
|
||||
} else {
|
||||
separator := os.PathSeparator
|
||||
if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) {
|
||||
localGzipFolderOrGzipFile += string(separator)
|
||||
}
|
||||
|
||||
// download
|
||||
if downloadFromOss {
|
||||
if !parseAndDownloadFromOss(ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile) {
|
||||
log.ErrorF("download from oss error !")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// load
|
||||
loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile)
|
||||
}
|
||||
|
||||
// tag
|
||||
// push
|
||||
allFileInFolder, err := utils.ListAllFileInFolder(localGzipFolderOrGzipFile)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
for _, gzipFileName := range allFileInFolder {
|
||||
// 过滤非.tar.gz结尾的文件
|
||||
if !strings.HasSuffix(gzipFileName, ".tar.gz") {
|
||||
continue
|
||||
}
|
||||
|
||||
log.DebugF("gzip file name is %s", gzipFileName)
|
||||
|
||||
// gzip to image full name 拿到镜像的原始名称
|
||||
imageFullName := image2.GzipFileNameToImageFullName(gzipFileName)
|
||||
if imageFullName == "" {
|
||||
log.ErrorF("gzip file %s to image full name error !", gzipFileName)
|
||||
continue
|
||||
}
|
||||
|
||||
// tag 拿到目标名称 然后重新Tag
|
||||
targetImageFullName := image2.ImageNameToTargetImageFullName(imageFullName, targetHarborFullName)
|
||||
image.TagFromSourceToTarget(imageFullName, targetImageFullName)
|
||||
|
||||
// uploadToHarbor 上传到目标Harbor
|
||||
if image.UploadToHarbor(targetImageFullName) {
|
||||
targetImageFullNameList = append(targetImageFullNameList, targetImageFullName)
|
||||
} else {
|
||||
log.ErrorF("upload to harbor error of %s", targetImageFullName)
|
||||
}
|
||||
}
|
||||
|
||||
return targetImageFullNameList
|
||||
}
|
||||
|
||||
func loadAllGzipImageFromLocalFolder(localGzipFolder string) {
|
||||
image.LoadFromFolderPath(localGzipFolder)
|
||||
}
|
||||
|
||||
func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string) bool {
|
||||
|
||||
if ossUrlPrefix == "" {
|
||||
ossUrlPrefix = minio.DefaultOssUrlPrefix
|
||||
}
|
||||
if !strings.HasSuffix(ossUrlPrefix, "/") {
|
||||
ossUrlPrefix += "/"
|
||||
}
|
||||
|
||||
log.InfoF("prepare to download from %s%s", ossUrlPrefix, ossFileName)
|
||||
|
||||
if !minio.DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+ossFileName, localGzipFolder) {
|
||||
log.ErrorF("download %s from oss error !", ossUrlPrefix+ossFileName)
|
||||
return false
|
||||
}
|
||||
|
||||
if strings.HasSuffix(ossFileName, ".txt") {
|
||||
// download from gzip file list txt
|
||||
// download all files in the txt file
|
||||
result := utils.ReadAllContentFromFile(localGzipFolder + ossFileName)
|
||||
for _, gzipFileName := range result {
|
||||
minio.DefaultCmiiMinioOperator.DemoMinioOperator.DownloadFileFromOssFullUrl(ossUrlPrefix+gzipFileName, localGzipFolder)
|
||||
}
|
||||
}
|
||||
|
||||
// 解析
|
||||
return true
|
||||
}
|
||||
|
||||
// C_DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
|
||||
func C_DownloadCompressUploadFromDemo(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) {
|
||||
|
||||
// generate a project folder
|
||||
projectName := syncEntity.DownloadCondition.ProjectName
|
||||
gzipFolderLocalPath := filepath.Join(image.OfflineImageGzipFolderPrefix, projectName)
|
||||
|
||||
err := os.MkdirAll(gzipFolderLocalPath, os.ModeDir)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", gzipFolderLocalPath, err.Error())
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// get demo image version map
|
||||
allCmiiImageFullNameList := buildAllCmiiImageNameListFromDemo(projectName)
|
||||
|
||||
// save all cmii image to file
|
||||
allPullImageNameTxtFileName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName)
|
||||
utils.AppendOverwriteListContentToFile(allCmiiImageFullNameList, allPullImageNameTxtFileName)
|
||||
|
||||
syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath
|
||||
syncEntity.DownloadCondition.FullNameImageList = allCmiiImageFullNameList
|
||||
|
||||
// save to result
|
||||
syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allPullImageNameTxtFileName
|
||||
|
||||
// do work
|
||||
// DCU
|
||||
A_DownloadCompressUpload(syncEntity, syncResult)
|
||||
}
|
||||
|
||||
func buildAllCmiiImageNameListFromDemo(projectName string) []string {
|
||||
|
||||
var realCmiiImageName []string
|
||||
|
||||
backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(config.Demo)
|
||||
|
||||
// save map to file
|
||||
backendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
|
||||
frontendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
|
||||
srsMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-srs-app.json"
|
||||
_ = os.Remove(backendMapFile)
|
||||
_ = os.Remove(frontendMapFile)
|
||||
_ = os.Remove(srsMapFile)
|
||||
|
||||
//utils.AppendContentToFile(
|
||||
// utils.BeautifulPrintToString(backendMap),
|
||||
// backendMapFile,
|
||||
//)
|
||||
//utils.AppendContentToFile(
|
||||
// utils.BeautifulPrintToString(frontendMap),
|
||||
// frontendMapFile,
|
||||
//)
|
||||
//utils.AppendContentToFile(
|
||||
// utils.BeautifulPrintToString(srsMapFile),
|
||||
// srsMapFile,
|
||||
//)
|
||||
|
||||
realCmiiImageName = append(realCmiiImageName, image.CmiiImageMapToFullNameList(backendMap)...)
|
||||
realCmiiImageName = append(realCmiiImageName, image.CmiiImageMapToFullNameList(frontendMap)...)
|
||||
realCmiiImageName = append(realCmiiImageName, image.CmiiImageMapToFullNameList(srsMap)...)
|
||||
|
||||
utils.BeautifulPrintListWithTitle(realCmiiImageName, "Cmii Project Image => "+projectName)
|
||||
|
||||
return realCmiiImageName
|
||||
}
|
||||
|
||||
// C_DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像
|
||||
func C_DownloadCompressUploadFromVersion(syncEntity *ImageSyncEntity, syncResult *ImageSyncResult) {
|
||||
|
||||
// generate a project folder
|
||||
projectCmiiVersion := syncEntity.DownloadCondition.ProjectVersion
|
||||
|
||||
// gzip local path
|
||||
gzipFolderLocalPath := filepath.Join(image.OfflineImageGzipFolderPrefix, projectCmiiVersion)
|
||||
|
||||
err := os.MkdirAll(gzipFolderLocalPath, os.ModeDir)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", gzipFolderLocalPath, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
syncEntity.CompressCondition.GzipLocalFolder = gzipFolderLocalPath
|
||||
|
||||
// build all cmii image name list
|
||||
allCmiiImageFullNameList := buildAllCmiiImageNameListFromVersion(projectCmiiVersion)
|
||||
// assign
|
||||
syncEntity.DownloadCondition.FullNameImageList = allCmiiImageFullNameList
|
||||
|
||||
// save all cmii image to file
|
||||
allImageListTxtFileFullName := filepath.Join(gzipFolderLocalPath, AllCmiiImageListLocalFileName)
|
||||
utils.AppendOverwriteContentToFile(utils.BeautifulPrintToString(allCmiiImageFullNameList), allImageListTxtFileFullName)
|
||||
|
||||
// save to result
|
||||
syncResult.DownloadResult.SuccessPullTxtFileLocalFullPath = allImageListTxtFileFullName
|
||||
|
||||
// do work
|
||||
// DCU procedure
|
||||
A_DownloadCompressUpload(syncEntity, syncResult)
|
||||
}
|
||||
|
||||
// buildAllCmiiImageNameListFromVersion 根据VersionTag构建完整的应用名称列表
|
||||
func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
|
||||
|
||||
var realCmiiImageName []string
|
||||
|
||||
backendMap := d_app.CmiiBackendAppMap
|
||||
frontendMap := d_app.CmiiFrontendAppMap
|
||||
|
||||
for app := range backendMap {
|
||||
backendMap[app] = cmiiVersion
|
||||
}
|
||||
for app := range frontendMap {
|
||||
frontendMap[app] = cmiiVersion
|
||||
}
|
||||
|
||||
realCmiiImageName = append(realCmiiImageName, image.CmiiImageMapToFullNameList(backendMap)...)
|
||||
realCmiiImageName = append(realCmiiImageName, image.CmiiImageMapToFullNameList(frontendMap)...)
|
||||
|
||||
for key, value := range d_app.CmiiSrsAppMap {
|
||||
var app *config.CmiiDeploymentInterface
|
||||
if strings.Contains(value, "deployment") {
|
||||
app = DefaultCmiiOperator.DeploymentOneInterface(config.Demo, key)
|
||||
if app != nil {
|
||||
realCmiiImageName = append(realCmiiImageName, app.Image)
|
||||
}
|
||||
} else if strings.Contains(value, "state") {
|
||||
app = DefaultCmiiOperator.StatefulSetOneInterface(config.Demo, key)
|
||||
if app != nil {
|
||||
for _, imageName := range app.ContainerImageMap {
|
||||
realCmiiImageName = append(realCmiiImageName, imageName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
utils.BeautifulPrintListWithTitle(realCmiiImageName, "Cmii Version Image => "+cmiiVersion)
|
||||
return realCmiiImageName
|
||||
}
|
||||
|
||||
// C_DownloadCompressUploadDependency DCU所有的依赖镜像
|
||||
func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
|
||||
|
||||
log.Info("DCU for middle and rke!")
|
||||
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
|
||||
if err != nil {
|
||||
if !errors.Is(err, os.ErrExist) {
|
||||
log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
var fullImageNameList []string
|
||||
var gzipFolderPrefix string
|
||||
|
||||
if isRKE {
|
||||
log.Info("DCU for rke!")
|
||||
fullImageNameList = d_app.Rancher1204Amd64
|
||||
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "rke/"
|
||||
} else {
|
||||
log.Info("DCU for middle!")
|
||||
|
||||
fullImageNameList = d_app.MiddlewareAmd64
|
||||
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/"
|
||||
}
|
||||
|
||||
syncEntity := &ImageSyncEntity{
|
||||
DownloadCondition: &DownloadEntity{
|
||||
ShouldDownloadImage: true,
|
||||
ProjectName: "",
|
||||
ProjectVersion: "",
|
||||
CmiiNameTagList: nil,
|
||||
FullNameImageList: fullImageNameList,
|
||||
DownloadAuthUserName: "",
|
||||
DownloadAuthPassword: "",
|
||||
},
|
||||
CompressCondition: &CompressEntity{
|
||||
ShouldCompressImageToGzip: shouldGzip,
|
||||
ShouldGzipSplit: true,
|
||||
GzipLocalFolder: gzipFolderPrefix,
|
||||
},
|
||||
UploadCondition: &UploadEntity{ShouldUploadToDemoMinio: shouldOss},
|
||||
ShouldDownloadFromOss: false,
|
||||
ShouldUpdateImageTag: false,
|
||||
ShouldDirectPushToHarbor: false,
|
||||
DirectHarborHost: "",
|
||||
}
|
||||
|
||||
syncResult := &ImageSyncResult{
|
||||
ProcedureSuccessImageList: nil,
|
||||
DownloadResult: &DownloadResultEntity{
|
||||
ErrorPullImageList: nil,
|
||||
SuccessPullImageList: nil,
|
||||
SuccessPullTxtFileLocalFullPath: "",
|
||||
},
|
||||
CompressResult: &CompressResultEntity{
|
||||
ErrorGzipImageList: nil,
|
||||
SuccessGzipImageList: nil,
|
||||
GzipTxtFileLocalFullPath: "",
|
||||
},
|
||||
UploadResult: &UploadResultEntity{
|
||||
ErrorUploadImageList: nil,
|
||||
AllDownloadUrl: nil,
|
||||
},
|
||||
}
|
||||
|
||||
utils.AppendOverwriteListContentToFile(fullImageNameList, filepath.Join(gzipFolderPrefix, AllCmiiImageListLocalFileName))
|
||||
|
||||
A_DownloadCompressUpload(syncEntity, syncResult)
|
||||
|
||||
return syncResult.DownloadResult.ErrorPullImageList, syncResult.CompressResult.ErrorGzipImageList, syncResult.ProcedureSuccessImageList, syncResult.CompressResult.SuccessGzipImageList
|
||||
}
|
||||
|
||||
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
|
||||
|
||||
// list folder
|
||||
projectGzipFolder := image.OfflineImageGzipFolderPrefix + projectName
|
||||
errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...)
|
||||
// read from json
|
||||
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(d_app.Cmii520DemoImageList, targetHarborHost)...)
|
||||
|
||||
// re-tag
|
||||
// push
|
||||
|
||||
// todo clean host and harbor
|
||||
// check harbor exits
|
||||
|
||||
return errorLoadImageNameList, errorPushImageNameList
|
||||
}
|
||||
|
||||
func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) {
|
||||
|
||||
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(d_app.MiddlewareAmd64, targetHarborHost)...)
|
||||
//errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(d_app.Rancher1204Amd64, targetHarborHost)...)
|
||||
|
||||
return errorLoadImageNameList, errorPushImageNameList
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user