zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[agent-go] - image sync entity

Browse Source
This commit is contained in:
zeaslity
2024-07-03 16:58:43 +08:00
parent 657b771475
commit 09da29f57a
6 changed files with 57 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
agent-deploy/d_app/CmiiImageConfig.go
View File

@@ -122,6 +122,7 @@ var MiddlewareAmd64 = []string{
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"jerrychina2020/rke-tools:v0.175-linux",
"jerrychina2020/rke-tools:v0.175",
"busybox:latest",
}

60
agent-operator/CmiiOperator.go
View File

@@ -16,17 +16,21 @@ const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.71.138"
type ImageSyncEntity struct {
ProjectName string // 优先级3
ProjectFromDemoNameNotNull bool
ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式
FullNameImageList []string // 优先级1
DownloadImage bool // 下载镜像
DownloadFromOss bool // 下载镜像
CompressImageToGzip bool // 压缩镜像
UploadToDemoMinio bool // 上传镜像
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 使用此部门
DirectHarborHost string // 目标Harbor仓库的Host，全名称带端口 不带http前缀
ProjectName string // 优先级3 优先级最低 从DEMO拉取镜像
ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像
CmiiNameTagList []string // 优先级1 appName:tag 会被转换为FullNameImageList
FullNameImageList []string // 优先级1 优先下载此类型
ShouldDownloadImage bool // 下载镜像 DCU中的D
ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C
ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U
ShouldDownloadFromOss bool // 下载镜像 DLTU中的D
ShouldUpdateImageTag bool // 更新镜像 DLTU中的U
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机
DirectHarborHost string // IP:Port or 域名:PORT 不带http前缀
}
type ImageSyncResult struct {
@@ -57,15 +61,14 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
if sync.ProjectVersion != "" {
// 获取特定版本的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio)
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio)
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion
} else {
// 获取DEMO的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio)
// 获取DEMO的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromDemo(sync.ProjectName, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio)
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName
}
@@ -77,7 +80,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
// DCU
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(true, allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = A_DownloadCompressUpload(true, allCmiiImageNameList, sync.ShouldCompressImageToGzip, gzipFolderFullPath, sync.ShouldUploadToDemoMinio)
}
// 直接传输到目标Harbor仓库
@@ -119,8 +122,8 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
return fullImageList
}
// DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode
func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
// A_DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode
func A_DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
// write to file
localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt"
@@ -235,13 +238,13 @@ func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzi
return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList
}
// DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
// A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
// 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录
// 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/
// 3. 读取本机的IP地址 - 参数传递
// 4. OSS地址 - ossUrlPrefix传空 则使用默认值
// 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式
func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {
func A_DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {
// 支持单文件的形式
if !utils.IsDirOrFile(localGzipFolderOrGzipFile) {
@@ -340,8 +343,8 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
return true
}
// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// C_DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
func C_DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
@@ -357,7 +360,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bo
// do work
// DCU
return DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
return A_DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
}
func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -396,8 +399,8 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
return realCmiiImageName
}
// DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像
func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// C_DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像
func C_DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
@@ -413,7 +416,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
// do work
// DCU procedure
return DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
return A_DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
}
@@ -454,7 +457,8 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
return realCmiiImageName
}
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// C_DownloadCompressUploadDependency DCU所有的依赖镜像
func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
log.Info("DCU for middle and rke!")
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
@@ -478,7 +482,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDow
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/"
}
return DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss)
return A_DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss)
}
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {

19
agent-operator/CmiiOperator_test.go
View File

@@ -11,7 +11,7 @@ import (
func TestFetchDependencyRepos_Middle(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, false)
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, false)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
@@ -22,7 +22,7 @@ func TestFetchDependencyRepos_Middle(t *testing.T) {
func TestFetchDependencyRepos_RKE(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, true)
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
@@ -50,17 +50,14 @@ func TestPullFromEntityAndSyncConditionally(t *testing.T) {
sync := ImageSyncEntity{
CmiiNameTagList: []string{
"cmii-uav-mqtthandler:5.4.0-bjdyt-052102",
"cmii-uav-industrial-portfolio:5.4.0-bjdyt-052801",
"cmii-uav-platform-armypeople:5.4.0-bjdyt-052802",
"cmii-uav-platform:5.4.0-bjdyt-052901",
},
FullNameImageList: nil,
ProjectVersion: "",
ProjectName: "bjtg",
DirectHarborHost: "10.250.0.110",
DownloadImage: true,
CompressImageToGzip: false,
UploadToDemoMinio: false,
ProjectName: "cqsh",
DirectHarborHost: "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn",
ShouldDownloadImage: true,
ShouldCompressImageToGzip: false,
ShouldUploadToDemoMinio: false,
ShouldDirectPushToHarbor: true,
}
@@ -87,7 +84,7 @@ func TestDownloadLoadTagUpload(t *testing.T) {
for _, s := range localGzipFileList {
log.InfoF("start to sync => %s", s)
targetImageFullNameList := DownloadLoadTagUpload(false, "", "", s, "172.28.0.251:8033")
targetImageFullNameList := A_DownloadLoadTagUpload(false, "", "", s, "172.28.0.251:8033")
utils.BeautifulPrintListWithTitle(targetImageFullNameList, "targetImageFullNameList")

10
agent-operator/image/CmiiImageOperator.go
View File

@@ -204,20 +204,28 @@ func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser
}
func UploadToHarbor(targetImageName string) (uploadOK bool) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
fmt.Println()
log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
fmt.Println()
return true
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
targetHarborHost = strings.TrimPrefix(targetHarborHost, "http://")
targetHarborHost = strings.TrimPrefix(targetHarborHost, "https://")
for _, imageName := range referenceImageList {
// check image
@@ -247,7 +255,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
}
}
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName
targetImageName := targetHarborHost + targetProject + "/" + imageName
//fmt.Println(cmiiImageFullName)
//fmt.Println(targetImageName)

2
agent-operator/main.go
View File

@@ -150,7 +150,7 @@ func main() {
}
// DLTU
targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName)
targetImageFullNameList := A_DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName)
// 是否需要更新
if namespace != "" {

6
server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
View File

@@ -38,7 +38,7 @@ public class TestImageSyncScheduler {
public void runImageSync() {
ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
"cmii-uav-platform:5.3.0-061901"
"cmii-uav-integration:5.4.0-070301"
));
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
@@ -48,10 +48,10 @@ public class TestImageSyncScheduler {
Boolean downloadAndCompressOnly = false;
// String projectNamespace = "wdd"; // wdd
String projectNamespace = "cqlyj"; // 重庆林业局
// String projectNamespace = "cqlyj"; // 重庆林业局
// String projectNamespace = "jlyd"; // 吉林移动
// String projectNamespace = "jxyd"; // 吉林移动
// String projectNamespace = "xmyd"; // 厦门移动
String projectNamespace = "xmyd"; // 厦门移动
// String innerWorkerAgentName = "Chengdu-amd64-65-lapwdd"; //wdd
String innerWorkerAgentName = "Chengdu-amd64-071-ed6e2b7f"; //prod