314 lines
11 KiB
Go
314 lines
11 KiB
Go
package main
|
||
|
||
import (
|
||
"errors"
|
||
"io/fs"
|
||
"os"
|
||
"path/filepath"
|
||
"slices"
|
||
"strings"
|
||
"wdd.io/agent-common/utils"
|
||
"wdd.io/agent-operator/image"
|
||
)
|
||
|
||
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
|
||
const OfflineDeployHarborHost = "harbor.wdd.io"
|
||
const PublicDeployHarborHost = "42.192.52.227"
|
||
const DirectPushDeployHarborHost = "36.134.71.138"
|
||
|
||
type ImageSyncEntity struct {
|
||
ProjectName string
|
||
ProjectVersion string
|
||
DirectHarborHost string //此参数决定是否能够直连目标主机，如果有则代表直连，可以直接推送景象
|
||
PushToDemoMinio bool
|
||
}
|
||
|
||
type ImageSyncResult struct {
|
||
ErrorPullImageList []string
|
||
ErrorGzipImageList []string
|
||
ErrorPushImageNameList []string
|
||
RealImageNameList []string
|
||
RealGzipFileNameList []string
|
||
AllCmiiImageNameList []string
|
||
}
|
||
|
||
// PullFromEntityAndSyncConditionally 根据ImageSyncEntity拉取特定的镜像，然后上传到特定的目标机器
|
||
func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) {
|
||
|
||
var realCmiiImageList []string
|
||
var errorPullImageList []string
|
||
var errorGzipImageList []string
|
||
var allCmiiImageNameList []string
|
||
var allGzipFileNameList []string
|
||
var errorPushImageNameList []string
|
||
var gzipFolderFullPath string
|
||
// get all image name by Name or Version
|
||
// pull images
|
||
// compress
|
||
if sync.ProjectVersion != "" {
|
||
// get version
|
||
if sync.DirectHarborHost == "" {
|
||
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, true)
|
||
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion
|
||
} else {
|
||
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, false)
|
||
}
|
||
} else {
|
||
// get demo images
|
||
if sync.DirectHarborHost == "" {
|
||
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, true)
|
||
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName
|
||
} else {
|
||
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, false)
|
||
}
|
||
}
|
||
|
||
backAllImageNameList := allCmiiImageNameList[:]
|
||
realCmiiImageList = slices.DeleteFunc(backAllImageNameList, func(imageName string) bool {
|
||
return slices.Contains(errorPullImageList, imageName)
|
||
})
|
||
realCmiiImageList = slices.DeleteFunc(backAllImageNameList, func(imageName string) bool {
|
||
return slices.Contains(errorGzipImageList, imageName)
|
||
})
|
||
|
||
// direct push if can
|
||
if sync.DirectHarborHost != "" {
|
||
// push to
|
||
errorPushImageNameList = image.TagFromListAndPushToCHarbor(realCmiiImageList, sync.DirectHarborHost)
|
||
|
||
// build
|
||
imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
|
||
imageSyncResult.ErrorPullImageList = errorPullImageList
|
||
imageSyncResult.ErrorGzipImageList = errorGzipImageList
|
||
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
|
||
imageSyncResult.RealImageNameList = realCmiiImageList
|
||
|
||
// no gzip file
|
||
|
||
return imageSyncResult
|
||
}
|
||
|
||
// get gzip file name list
|
||
err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error {
|
||
if err != nil {
|
||
log.ErrorF("error getting gzip file name list 1! %s", err.Error())
|
||
}
|
||
if !d.IsDir() {
|
||
allGzipFileNameList = append(allGzipFileNameList, d.Name())
|
||
}
|
||
return nil
|
||
})
|
||
if err != nil {
|
||
log.ErrorF("error getting gzip file name list 2! %s", err.Error())
|
||
}
|
||
|
||
// push to demo minio
|
||
if sync.PushToDemoMinio {
|
||
log.InfoF("pretend to push to minio !")
|
||
// create path
|
||
// push
|
||
}
|
||
|
||
// build
|
||
imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
|
||
imageSyncResult.ErrorPullImageList = errorPullImageList
|
||
imageSyncResult.ErrorGzipImageList = errorGzipImageList
|
||
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
|
||
imageSyncResult.RealGzipFileNameList = allGzipFileNameList
|
||
imageSyncResult.RealImageNameList = realCmiiImageList
|
||
|
||
// no gzip file
|
||
|
||
return imageSyncResult
|
||
}
|
||
|
||
func remove(s1, s2 []string) []string {
|
||
m := make(map[string]struct{}, len(s2))
|
||
for _, v := range s2 {
|
||
m[v] = struct{}{}
|
||
}
|
||
res := make([]string, 0, len(s1))
|
||
for _, v := range s1 {
|
||
if _, ok := m[v]; !ok {
|
||
res = append(res, v)
|
||
}
|
||
}
|
||
return res
|
||
}
|
||
|
||
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
|
||
|
||
// generate a project folder
|
||
err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
|
||
if err != nil {
|
||
if !errors.Is(err, os.ErrExist) {
|
||
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
|
||
return errorPullImageList, errorGzipImageList, allCmiiImageName
|
||
}
|
||
}
|
||
|
||
// get demo image version map
|
||
backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(demo)
|
||
|
||
utils.BeautifulPrint(backendMap)
|
||
utils.BeautifulPrint(frontendMap)
|
||
utils.BeautifulPrint(srsMap)
|
||
|
||
// save map to file
|
||
backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
|
||
frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
|
||
srsMapFile := OfflineImageGzipFolderPrefix + projectName + "-srs-app.json"
|
||
_ = os.Remove(backendMapFile)
|
||
_ = os.Remove(frontendMapFile)
|
||
_ = os.Remove(srsMapFile)
|
||
|
||
utils.AppendContentToFile(
|
||
utils.BeautifulPrintToString(backendMap),
|
||
backendMapFile,
|
||
)
|
||
utils.AppendContentToFile(
|
||
utils.BeautifulPrintToString(frontendMap),
|
||
frontendMapFile,
|
||
)
|
||
utils.AppendContentToFile(
|
||
utils.BeautifulPrintToString(srsMapFile),
|
||
srsMapFile,
|
||
)
|
||
|
||
// download image
|
||
backendFullNameList, backendPull := image.PullFromCmiiHarborByMap(backendMap, true)
|
||
frontendFullNameList, frontendPull := image.PullFromCmiiHarborByMap(frontendMap, true)
|
||
srsFullNameList, srsPull := image.PullFromCmiiHarborByMap(srsMap, true)
|
||
|
||
allCmiiImageName = append(allCmiiImageName, backendFullNameList...)
|
||
allCmiiImageName = append(allCmiiImageName, frontendFullNameList...)
|
||
allCmiiImageName = append(allCmiiImageName, srsFullNameList...)
|
||
|
||
// compress image
|
||
if gzipSplit {
|
||
for image_name, tag := range backendMap {
|
||
if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
|
||
errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag)
|
||
}
|
||
}
|
||
for image_name, tag := range frontendMap {
|
||
if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
|
||
errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag)
|
||
}
|
||
}
|
||
for image_name, tag := range srsMap {
|
||
if !image.SaveToTarGZ(image_name+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/") {
|
||
errorGzipImageList = append(errorGzipImageList, image.CmiiHarborPrefix+image_name+":"+tag)
|
||
}
|
||
}
|
||
}
|
||
// upload to harbor
|
||
|
||
// clean up images
|
||
|
||
errorPullImageList = append(errorPullImageList, backendPull...)
|
||
errorPullImageList = append(errorPullImageList, frontendPull...)
|
||
errorPullImageList = append(errorPullImageList, srsPull...)
|
||
|
||
return errorPullImageList, errorGzipImageList, allCmiiImageName
|
||
}
|
||
|
||
func FetchVersionImages(cmiiVersion string, shouldGzip bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
|
||
|
||
// generate a project folder
|
||
err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
|
||
if err != nil {
|
||
if !errors.Is(err, os.ErrExist) {
|
||
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
|
||
return errorPullImageList, errorGzipImageList, allCmiiImageName
|
||
}
|
||
}
|
||
|
||
backendMap := CmiiBackendAppMap
|
||
frontendMap := CmiiFrontendAppMap
|
||
|
||
for app := range backendMap {
|
||
backendMap[app] = cmiiVersion
|
||
}
|
||
for app := range frontendMap {
|
||
frontendMap[app] = cmiiVersion
|
||
}
|
||
|
||
allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
|
||
allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
|
||
|
||
for key, value := range CmiiSrsAppMap {
|
||
var app *CmiiDeploymentInterface
|
||
if strings.Contains(value, "deployment") {
|
||
app = CmiiOperator.DeploymentOneInterface(demo, key)
|
||
if app != nil {
|
||
allCmiiImageName = append(allCmiiImageName, app.Image)
|
||
}
|
||
} else if strings.Contains(value, "state") {
|
||
app = CmiiOperator.StatefulSetOneInterface(demo, key)
|
||
if app != nil {
|
||
for _, imageName := range app.ContainerImageMap {
|
||
allCmiiImageName = append(allCmiiImageName, imageName)
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
utils.BeautifulPrintListWithTitle(allCmiiImageName, "Cmii Version Image => "+cmiiVersion)
|
||
|
||
// do work
|
||
if shouldGzip {
|
||
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(allCmiiImageName, OfflineImageGzipFolderPrefix+cmiiVersion)
|
||
} else {
|
||
errorPullImageList = image.PullFromFullNameList(allCmiiImageName)
|
||
}
|
||
|
||
return errorPullImageList, errorGzipImageList, allCmiiImageName
|
||
}
|
||
|
||
func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
|
||
err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir)
|
||
if err != nil {
|
||
if !errors.Is(err, os.ErrExist) {
|
||
log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error())
|
||
}
|
||
}
|
||
|
||
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(image.MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
|
||
|
||
pull, gzipImageList := image.PullFromListAndCompressSplit(image.Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
|
||
|
||
return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
|
||
}
|
||
|
||
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
|
||
|
||
// list folder
|
||
projectGzipFolder := OfflineImageGzipFolderPrefix + projectName
|
||
errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...)
|
||
// read from json
|
||
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Cmii520DemoImageList, targetHarborHost)...)
|
||
|
||
// re-tag
|
||
// push
|
||
|
||
// todo clean host and harbor
|
||
// check harbor exits
|
||
|
||
return errorLoadImageNameList, errorPushImageNameList
|
||
}
|
||
|
||
func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) {
|
||
|
||
//middle := OfflineImageGzipFolderPrefix + "middle/"
|
||
//rke := OfflineImageGzipFolderPrefix + "rke/"
|
||
//errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(middle)...)
|
||
//errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(rke)...)
|
||
|
||
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.MiddlewareAmd64, targetHarborHost)...)
|
||
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Rancher1204Amd64, targetHarborHost)...)
|
||
|
||
return errorLoadImageNameList, errorPushImageNameList
|
||
|
||
}
|