zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[agent-operator] - add tag from gzip folder

Browse Source
This commit is contained in:
zeaslity
2024-04-26 10:20:12 +08:00
parent 08e5a4d422
commit 7f4a4d877e
7 changed files with 101 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
agent-operator/CmiiOperator.go
View File

@@ -15,7 +15,6 @@ import (
"wdd.io/agent-operator/image"
)
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
const OfflineDeployHarborHost = "harbor.wdd.io"
const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.71.138"
@@ -63,19 +62,19 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
// get version images
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion
} else {
// get demo images
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName
}
} else {
// 拉取特定的镜像
gzipFolderFullPath = OfflineImageGzipFolderPrefix + "tmp"
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + "tmp"
// 组装镜像名称
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
@@ -178,7 +177,7 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
// start to upload
// extract demo oss location suffix from gzipFolderFullPath
trimPrefix := strings.TrimPrefix(gzipFolderFullPath, OfflineImageGzipFolderPrefix)
trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix)
bucketName := "cmlc-installation/" + trimPrefix
log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName)
@@ -290,10 +289,10 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder
err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+projectName, err.Error())
return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
}
}
@@ -303,7 +302,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO
// do work
// DCU
return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, OfflineImageGzipFolderPrefix+projectName, shouldOss)
return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
}
func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -313,9 +312,9 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
backendMap, frontendMap, srsMap := BackupAllCmiiDeploymentToMap(demo)
// save map to file
backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
srsMapFile := OfflineImageGzipFolderPrefix + projectName + "-srs-app.json"
backendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
frontendMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
srsMapFile := image.OfflineImageGzipFolderPrefix + projectName + "-srs-app.json"
_ = os.Remove(backendMapFile)
_ = os.Remove(frontendMapFile)
_ = os.Remove(srsMapFile)
@@ -333,9 +332,9 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
// srsMapFile,
//)
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(srsMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(backendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(frontendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(srsMap)...)
utils.BeautifulPrintListWithTitle(realCmiiImageName, "Cmii Project Image => "+projectName)
@@ -346,10 +345,10 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder
err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
log.ErrorF("[Download_Compress_Upload_From_Demo] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
}
}
@@ -359,7 +358,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
// do work
// DCU procedure
return DownloadCompressUpload(realCmiiImageName, shouldGzip, OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
return DownloadCompressUpload(realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
}
@@ -377,8 +376,8 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
frontendMap[app] = cmiiVersion
}
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(backendMap)...)
realCmiiImageName = append(realCmiiImageName, image.ConvertCmiiImageMapToList(frontendMap)...)
for key, value := range CmiiSrsAppMap {
var app *CmiiDeploymentInterface
@@ -401,16 +400,16 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
}
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir)
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error())
log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", image.OfflineImageGzipFolderPrefix, err.Error())
}
}
if downloadMiddle {
gzipFolderPrefix := OfflineImageGzipFolderPrefix + "middle/"
gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "middle/"
// remove folder first
utils.RemoveFolderComplete(gzipFolderPrefix)
@@ -419,7 +418,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM
}
if downloadRke {
gzipFolderPrefix := OfflineImageGzipFolderPrefix + "rke/"
gzipFolderPrefix := image.OfflineImageGzipFolderPrefix + "rke/"
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
}
@@ -429,7 +428,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadM
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
// list folder
projectGzipFolder := OfflineImageGzipFolderPrefix + projectName
projectGzipFolder := image.OfflineImageGzipFolderPrefix + projectName
errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...)
// read from json
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Cmii520DemoImageList, targetHarborHost)...)
@@ -445,8 +444,8 @@ func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string)
func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) {
//middle := OfflineImageGzipFolderPrefix + "middle/"
//rke := OfflineImageGzipFolderPrefix + "rke/"
//middle := image.OfflineImageGzipFolderPrefix + "middle/"
//rke := image.OfflineImageGzipFolderPrefix + "rke/"
//errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(middle)...)
//errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(rke)...)

26
agent-operator/deploy/c_app/DeployCmiiApp.go
View File

@@ -102,9 +102,18 @@ func (frontend *CmiiFrontendConfig) FrontendDeploy(common *z_dep.CommonEnvironme
return true
}
func SrsDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
return commonEnv.ParseCommonEnvToApplyFile(CmiiSrsTemplate, SRSApplyFilePath)
// copy
z_dep.CopySameFields(commonEnv, frontend)
// manual validate
if frontend.ShortName == "" || frontend.ClientId == "" {
log.ErrorF("short name or client id is empty !")
return false
}
return z_dep.ParseEnvToApplyFile(frontend, CmiiFrontendConfigMapTemplate, ConfigMapApplyFilePath)
}
func IngressDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
@@ -123,16 +132,7 @@ func IngressDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
return true
}
func (frontend *CmiiFrontendConfig) ConfigMapDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
func SrsDeploy(commonEnv *z_dep.CommonEnvironmentConfig) bool {
// copy
z_dep.CopySameFields(commonEnv, frontend)
// manual validate
if frontend.ShortName == "" || frontend.ClientId == "" {
log.ErrorF("short name or client id is empty !")
return false
}
return z_dep.ParseEnvToApplyFile(frontend, CmiiFrontendConfigMapTemplate, ConfigMapApplyFilePath)
return commonEnv.ParseCommonEnvToApplyFile(CmiiSrsTemplate, SRSApplyFilePath)
}

2
agent-operator/go.mod
View File

@@ -14,7 +14,6 @@ require (
k8s.io/api v0.29.1
k8s.io/apimachinery v0.29.1
k8s.io/client-go v0.29.1
sigs.k8s.io/yaml v1.4.0
wdd.io/agent-common v0.0.0
)
@@ -96,4 +95,5 @@ require (
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
)

23
agent-operator/image/CmiiImageSync.go
View File

@@ -24,6 +24,8 @@ import (
var apiClient = newClient()
var log = logger.Log
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
func newClient() *client.Client {
apiClient, err := client.NewClientWithOpts(client.FromEnv)
@@ -276,7 +278,7 @@ func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap)
fullImageNameList = ConvertCmiiImageMapToList(imageVersionMap)
return fullImageNameList, PullFromFullNameList(fullImageNameList)
}
@@ -472,7 +474,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
return true
}
func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
func ConvertCmiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
for image, tag := range cmiiImageVersionMap {
s := image2.CmiiHarborPrefix + image + ":" + tag
@@ -482,6 +484,23 @@ func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImage
return fullImageNameList
}
func ConvertCmiiImageMapFromGzipFolder(gzipFileFolder string) (cmiiImageVersionMap map[string]string) {
allFileInFolder, err := utils.ListAllFileInFolder(gzipFileFolder)
if err != nil {
return nil
}
cmiiImageVersionMap = make(map[string]string, len(allFileInFolder))
for _, gzipFileName := range allFileInFolder {
log.DebugF("gzip file name is %s", gzipFileName)
imageName, imageTag := image2.GzipFileNameToImageNameAndTag(gzipFileName)
cmiiImageVersionMap[imageName] = imageTag
}
return cmiiImageVersionMap
}
func loginToDockerHub(HarborFullHost string) {
if HarborFullHost == "" {

7
agent-operator/image/CmiiImageSync_test.go
View File

@@ -195,3 +195,10 @@ func TestSaveSpecificImageToGzipFile(t *testing.T) {
}
}
func TestConvertCmiiImageMapFromGzipFolder(t *testing.T) {
versionMap := ConvertCmiiImageMapFromGzipFolder(OfflineImageGzipFolderPrefix)
utils.BeautifulPrint(versionMap)
}