zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Operator ] - agent operator for DCU part

Browse Source
This commit is contained in:
zeaslity
2024-04-10 17:20:23 +08:00
parent a9a9ab859e
commit b840f0051e
6 changed files with 285 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

280
agent-operator/CmiiOperator.go
View File

@@ -18,11 +18,15 @@ const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.71.138"
type ImageSyncEntity struct {
ProjectName string // 3
ProjectVersion string // 2
CmiiImageList map[string]string // 1
DirectHarborHost string //此参数决定是否能够直连目标主机，如果有则代表直连，可以直接推送景象
PushToDemoMinio bool
ProjectName string // 优先级3
ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式
FullNameImageList []string // 优先级1
DownloadImage bool // 下载镜像
CompressImageToGzip bool // 压缩镜像
UploadToDemoMinio bool // 上传镜像
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 使用此部门
DirectHarborHost string // 目标Harbor仓库的Host，全名称带端口
}
type ImageSyncResult struct {
@@ -38,100 +42,170 @@ type ImageSyncResult struct {
func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) {
var realCmiiImageList []string
var errorPullImageList []string
var errorGzipImageList []string
var allCmiiImageNameList []string
var errorPullImageList []string
var allGzipFileNameList []string
var errorGzipImageList []string
var errorPushImageNameList []string
var gzipFolderFullPath string
// get all image name by Name or Version
// pull images
// compress
if sync.ProjectVersion != "" {
// get version
if sync.DirectHarborHost == "" {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, true)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion
if (sync.CmiiNameTagList == nil && sync.FullNameImageList == nil) || (len(sync.CmiiNameTagList) == 0 && len(sync.FullNameImageList) == 0) {
// 没有指定特定的镜像，那么根据 ProjectVersion 或者从DEMO拉取镜像
// pull images
// compress
if sync.ProjectVersion != "" {
// get version
if sync.DirectHarborHost == "" {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, true, sync.UploadToDemoMinio)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion
} else {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, false, sync.UploadToDemoMinio)
}
} else {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, false)
// get demo images
if sync.DirectHarborHost == "" {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName
} else {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = DownloadCompressUploadFromDemo(sync.ProjectName, false, sync.UploadToDemoMinio)
}
}
} else {
// get demo images
if sync.DirectHarborHost == "" {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, true)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName
} else {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, false)
}
// 拉取特定的镜像
gzipFolderFullPath = OfflineImageGzipFolderPrefix + "tmp"
// 组装镜像名称
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
// DCU
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUploadFromFullNameList(allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
}
backAllImageNameList := allCmiiImageNameList[:]
realCmiiImageList = slices.DeleteFunc(backAllImageNameList, func(imageName string) bool {
return slices.Contains(errorPullImageList, imageName)
})
realCmiiImageList = slices.DeleteFunc(backAllImageNameList, func(imageName string) bool {
return slices.Contains(errorGzipImageList, imageName)
})
// direct push if can
if sync.DirectHarborHost != "" {
// 直接传输到目标Harbor仓库
if sync.ShouldDirectPushToHarbor {
if sync.DirectHarborHost == "" {
log.ErrorF("DirectHarborHost is null ! can't push to target harbor !")
}
// push to
errorPushImageNameList = image.TagFromListAndPushToCHarbor(realCmiiImageList, sync.DirectHarborHost)
// build
imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
imageSyncResult.ErrorPullImageList = errorPullImageList
imageSyncResult.ErrorGzipImageList = errorGzipImageList
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
imageSyncResult.RealImageNameList = realCmiiImageList
// no gzip file
return imageSyncResult
}
// get gzip file name list
err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.ErrorF("error getting gzip file name list 1! %s", err.Error())
}
if !d.IsDir() {
allGzipFileNameList = append(allGzipFileNameList, d.Name())
}
return nil
})
if err != nil {
log.ErrorF("error getting gzip file name list 2! %s", err.Error())
}
// push to demo minio
if sync.PushToDemoMinio {
log.InfoF("pretend to push to minio !")
// create path
// push
}
// build
// build result
imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
imageSyncResult.ErrorPullImageList = errorPullImageList
imageSyncResult.ErrorGzipImageList = errorGzipImageList
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
imageSyncResult.RealGzipFileNameList = allGzipFileNameList
imageSyncResult.RealImageNameList = realCmiiImageList
// no gzip file
imageSyncResult.RealImageNameList = realCmiiImageList
imageSyncResult.ErrorPullImageList = errorPullImageList
imageSyncResult.RealGzipFileNameList = allGzipFileNameList
imageSyncResult.ErrorGzipImageList = errorGzipImageList
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
return imageSyncResult
}
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) []string {
if cmiiImageList != nil || len(cmiiImageList) > 0 {
// cmiiImageList has content
if fullImageList == nil {
fullImageList = []string{}
}
for _, cmiiImage := range cmiiImageList {
fullImageList = append(fullImageList, image2.CmiiHarborPrefix+cmiiImage)
}
}
return fullImageList
}
func DownloadCompressUploadFromFullNameList(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// Download
log.Info("DOWNLOAD START !")
errorPullImageList = image.PullFromFullNameList(fullNameList)
// remove failed
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
return slices.Contains(errorPullImageList, imageName)
})
// Compress
if shouldGzip {
// mkdir folder
err := os.MkdirAll(gzipFolderFullPath, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("create folder error of %s", gzipFolderFullPath)
panic(err)
}
}
// 循环遍历压缩
log.Info("COMPRESS START")
for _, imageFullName := range fullNameList {
if !image.SaveToTarGZ(imageFullName, gzipFolderFullPath) {
errorGzipImageList = append(errorGzipImageList, imageFullName)
}
}
// remove failed
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
return slices.Contains(errorGzipImageList, imageName)
})
}
// Upload
if shouldOss {
//uploadGzipFileToDemoMinio()
// get gzip file name list
log.Info("UPLOAD OSS START !")
err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.ErrorF("error getting gzip file name list 1! %s", err.Error())
}
if !d.IsDir() {
allGzipFileNameList = append(allGzipFileNameList, d.Name())
}
return nil
})
if err != nil {
log.ErrorF("error getting gzip file name list 2! %s", err.Error())
}
// start to upload
// extract demo oss location suffix from gzipFolderFullPath
trimPrefix := strings.TrimPrefix(gzipFolderFullPath, OfflineImageGzipFolderPrefix)
bucketName := "cmlc-installation/" + trimPrefix
log.InfoF("gzip file location in demo oss is %s", DemoEndpoint+"/"+bucketName)
minioOperator := CmiiMinioOperator{}
for _, gzipFileName := range allGzipFileNameList {
if !minioOperator.UploadToDemo(bucketName, gzipFolderFullPath, gzipFileName) {
log.ErrorF("upload of %s to demo oss error !", gzipFolderFullPath+gzipFileName)
}
}
}
return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileNameList
}
func uploadGzipFileToDemoMinio() {
}
// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName []string) {
// generate a project folder
err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
return errorPullImageList, errorGzipImageList, allCmiiImageName
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
return errorPullImageList, errorGzipImageList, realCmiiImageName
}
}
@@ -168,25 +242,25 @@ func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, er
frontendFullNameList, frontendPull := image.PullFromCmiiHarborByMap(frontendMap, true)
srsFullNameList, srsPull := image.PullFromCmiiHarborByMap(srsMap, true)
allCmiiImageName = append(allCmiiImageName, backendFullNameList...)
allCmiiImageName = append(allCmiiImageName, frontendFullNameList...)
allCmiiImageName = append(allCmiiImageName, srsFullNameList...)
realCmiiImageName = append(realCmiiImageName, backendFullNameList...)
realCmiiImageName = append(realCmiiImageName, frontendFullNameList...)
realCmiiImageName = append(realCmiiImageName, srsFullNameList...)
// compress image
if gzipSplit {
for image_name, tag := range backendMap {
if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag)
if shouldGzip {
for imageName, tag := range backendMap {
if !image.SaveToTarGZ(imageName+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+imageName+":"+tag)
}
}
for image_name, tag := range frontendMap {
if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag)
for imageName, tag := range frontendMap {
if !image.SaveToTarGZ(imageName+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+imageName+":"+tag)
}
}
for image_name, tag := range srsMap {
if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+image_name+":"+tag)
for imageName, tag := range srsMap {
if !image.SaveToTarGZ(imageName+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
errorGzipImageList = append(errorGzipImageList, image2.CmiiHarborPrefix+imageName+":"+tag)
}
}
}
@@ -198,17 +272,18 @@ func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, er
errorPullImageList = append(errorPullImageList, frontendPull...)
errorPullImageList = append(errorPullImageList, srsPull...)
return errorPullImageList, errorGzipImageList, allCmiiImageName
return errorPullImageList, errorGzipImageList, realCmiiImageName
}
func FetchVersionImages(cmiiVersion string, shouldGzip bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
// DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像
func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName []string) {
// generate a project folder
err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
return errorPullImageList, errorGzipImageList, allCmiiImageName
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
return errorPullImageList, errorGzipImageList, realCmiiImageName
}
}
@@ -222,39 +297,39 @@ func FetchVersionImages(cmiiVersion string, shouldGzip bool) (errorPullImageList
frontendMap[app] = cmiiVersion
}
allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
for key, value := range CmiiSrsAppMap {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(demo, key)
if app != nil {
allCmiiImageName = append(allCmiiImageName, app.Image)
realCmiiImageName = append(realCmiiImageName, app.Image)
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(demo, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
allCmiiImageName = append(allCmiiImageName, imageName)
realCmiiImageName = append(realCmiiImageName, imageName)
}
}
}
}
utils.BeautifulPrintListWithTitle(allCmiiImageName, "Cmii Version Image => "+cmiiVersion)
utils.BeautifulPrintListWithTitle(realCmiiImageName, "Cmii Version Image => "+cmiiVersion)
// do work
if shouldGzip {
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(allCmiiImageName, OfflineImageGzipFolderPrefix+cmiiVersion)
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(realCmiiImageName, OfflineImageGzipFolderPrefix+cmiiVersion)
} else {
errorPullImageList = image.PullFromFullNameList(allCmiiImageName)
errorPullImageList = image.PullFromFullNameList(realCmiiImageName)
}
return errorPullImageList, errorGzipImageList, allCmiiImageName
return errorPullImageList, errorGzipImageList, realCmiiImageName
}
func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList []string) {
err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
@@ -262,10 +337,13 @@ func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageLis
}
}
// pull middleware images
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(image.MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
// pull rke images
pull, gzipImageList := image.PullFromListAndCompressSplit(image.Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
// result
return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
}