zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[agent-go] - image sync entity

Browse Source
This commit is contained in:
zeaslity
2024-07-03 16:58:43 +08:00
parent 657b771475
commit 09da29f57a
6 changed files with 57 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
agent-deploy/d_app/CmiiImageConfig.go
View File

@@ -122,6 +122,7 @@ var MiddlewareAmd64 = []string{
"redis:6.0.20-alpine", "redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2", "dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"jerrychina2020/rke-tools:v0.175-linux", "jerrychina2020/rke-tools:v0.175-linux",
"jerrychina2020/rke-tools:v0.175",
"busybox:latest", "busybox:latest",
} }

60
agent-operator/CmiiOperator.go
View File

@@ -16,17 +16,21 @@ const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.71.138" const DirectPushDeployHarborHost = "36.134.71.138"
type ImageSyncEntity struct { type ImageSyncEntity struct {
ProjectName string // 优先级3 ProjectName string // 优先级3 优先级最低 从DEMO拉取镜像
ProjectFromDemoNameNotNull bool ProjectVersion string // 优先级2 高于ProjectName 优先拉取特定版本的镜像
ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式 CmiiNameTagList []string // 优先级1 appName:tag 会被转换为FullNameImageList
FullNameImageList []string // 优先级1 FullNameImageList []string // 优先级1 优先下载此类型
DownloadImage bool // 下载镜像
DownloadFromOss bool // 下载镜像 ShouldDownloadImage bool // 下载镜像 DCU中的D
CompressImageToGzip bool // 压缩镜像 ShouldCompressImageToGzip bool // 压缩镜像 DCU中的C
UploadToDemoMinio bool // 上传镜像 ShouldUploadToDemoMinio bool // 上传镜像 DCU中的U
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 使用此部门
DirectHarborHost string // 目标Harbor仓库的Host，全名称带端口 不带http前缀 ShouldDownloadFromOss bool // 下载镜像 DLTU中的D
ShouldUpdateImageTag bool // 更新镜像 DLTU中的U
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机
DirectHarborHost string // IP:Port or 域名:PORT 不带http前缀
} }
type ImageSyncResult struct { type ImageSyncResult struct {
@@ -57,15 +61,14 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
if sync.ProjectVersion != "" { if sync.ProjectVersion != "" {
// 获取特定版本的镜像 // 获取特定版本的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio)
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio)
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion
} else { } else {
// 获取DEMO的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio) // 获取DEMO的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = C_DownloadCompressUploadFromDemo(sync.ProjectName, sync.ShouldCompressImageToGzip, sync.ShouldUploadToDemoMinio)
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName
} }
@@ -77,7 +80,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList) allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
// DCU // DCU
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(true, allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio) errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = A_DownloadCompressUpload(true, allCmiiImageNameList, sync.ShouldCompressImageToGzip, gzipFolderFullPath, sync.ShouldUploadToDemoMinio)
} }
// 直接传输到目标Harbor仓库 // 直接传输到目标Harbor仓库
@@ -119,8 +122,8 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
return fullImageList return fullImageList
} }
// DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode // A_DownloadCompressUpload DCU 镜像同步的前半部分，通常在35.71 LapPro执行，无需Bastion Mode
func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) { func A_DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileFullNameList []string) {
// write to file // write to file
localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt" localGzipFileListTxt := gzipFolderFullPath + "all-gzip-image-file-name.txt"
@@ -235,13 +238,13 @@ func DownloadCompressUpload(downloadImage bool, fullNameList []string, shouldGzi
return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileFullNameList
} }
// DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式 // A_DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程，需要支持 堡垒机（纯离线）的模式
// 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录 // 2. Gzip文件目录，RKE MIDDLE CMII三个文件目录 - 约定目录
// 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/ // 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/
// 3. 读取本机的IP地址 - 参数传递 // 3. 读取本机的IP地址 - 参数传递
// 4. OSS地址 - ossUrlPrefix传空 则使用默认值 // 4. OSS地址 - ossUrlPrefix传空 则使用默认值
// 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式 // 5. ossFileName - 如果结尾为txt，则为文件的形式，如果为tar.gz，则为gzip文件夹的形式
func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) { func A_DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileNameOrGzipFileListTxt, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {
// 支持单文件的形式 // 支持单文件的形式
if !utils.IsDirOrFile(localGzipFolderOrGzipFile) { if !utils.IsDirOrFile(localGzipFolderOrGzipFile) {
@@ -340,8 +343,8 @@ func parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder string)
return true return true
} }
// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像 // C_DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { func C_DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder // generate a project folder
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir) err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
@@ -357,7 +360,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip, shouldOss bo
// do work // do work
// DCU // DCU
return DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss) return A_DownloadCompressUpload(true, allCmiiImageNameListFromDemo, shouldGzip, image.OfflineImageGzipFolderPrefix+projectName, shouldOss)
} }
func buildAllCmiiImageNameListFromDemo(projectName string) []string { func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -396,8 +399,8 @@ func buildAllCmiiImageNameListFromDemo(projectName string) []string {
return realCmiiImageName return realCmiiImageName
} }
// DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像 // C_DownloadCompressUploadFromVersion 根据版本下载全部的CMII镜像
func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { func C_DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// generate a project folder // generate a project folder
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir) err := os.MkdirAll(image.OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
@@ -413,7 +416,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
// do work // do work
// DCU procedure // DCU procedure
return DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss) return A_DownloadCompressUpload(true, realCmiiImageName, shouldGzip, image.OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
} }
@@ -454,7 +457,8 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
return realCmiiImageName return realCmiiImageName
} }
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) { // C_DownloadCompressUploadDependency DCU所有的依赖镜像
func C_DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDownload bool, isRKE bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
log.Info("DCU for middle and rke!") log.Info("DCU for middle and rke!")
err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir) err := os.MkdirAll(image.OfflineImageGzipFolderPrefix, os.ModeDir)
@@ -478,7 +482,7 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, shouldDow
gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/" gzipFolderPrefix = image.OfflineImageGzipFolderPrefix + "middle/"
} }
return DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss) return A_DownloadCompressUpload(shouldDownload, fulleImageNameList, shouldGzip, gzipFolderPrefix, shouldOss)
} }
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) { func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {

19
agent-operator/CmiiOperator_test.go
View File

@@ -11,7 +11,7 @@ import (
func TestFetchDependencyRepos_Middle(t *testing.T) { func TestFetchDependencyRepos_Middle(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, false) errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, false)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
@@ -22,7 +22,7 @@ func TestFetchDependencyRepos_Middle(t *testing.T) {
func TestFetchDependencyRepos_RKE(t *testing.T) { func TestFetchDependencyRepos_RKE(t *testing.T) {
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, true, false, true) errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := C_DownloadCompressUploadDependency(true, true, false, true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList") utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList") utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
@@ -50,17 +50,14 @@ func TestPullFromEntityAndSyncConditionally(t *testing.T) {
sync := ImageSyncEntity{ sync := ImageSyncEntity{
CmiiNameTagList: []string{ CmiiNameTagList: []string{
"cmii-uav-mqtthandler:5.4.0-bjdyt-052102", "cmii-uav-mqtthandler:5.4.0-bjdyt-052102",
"cmii-uav-industrial-portfolio:5.4.0-bjdyt-052801",
"cmii-uav-platform-armypeople:5.4.0-bjdyt-052802",
"cmii-uav-platform:5.4.0-bjdyt-052901",
}, },
FullNameImageList: nil, FullNameImageList: nil,
ProjectVersion: "", ProjectVersion: "",
ProjectName: "bjtg", ProjectName: "cqsh",
DirectHarborHost: "10.250.0.110", DirectHarborHost: "chongqingcis-9b4a3da9.ecis.chongqing-1.cmecloud.cn",
DownloadImage: true, ShouldDownloadImage: true,
CompressImageToGzip: false, ShouldCompressImageToGzip: false,
UploadToDemoMinio: false, ShouldUploadToDemoMinio: false,
ShouldDirectPushToHarbor: true, ShouldDirectPushToHarbor: true,
} }
@@ -87,7 +84,7 @@ func TestDownloadLoadTagUpload(t *testing.T) {
for _, s := range localGzipFileList { for _, s := range localGzipFileList {
log.InfoF("start to sync => %s", s) log.InfoF("start to sync => %s", s)
targetImageFullNameList := DownloadLoadTagUpload(false, "", "", s, "172.28.0.251:8033") targetImageFullNameList := A_DownloadLoadTagUpload(false, "", "", s, "172.28.0.251:8033")
utils.BeautifulPrintListWithTitle(targetImageFullNameList, "targetImageFullNameList") utils.BeautifulPrintListWithTitle(targetImageFullNameList, "targetImageFullNameList")

10
agent-operator/image/CmiiImageOperator.go
View File

@@ -204,20 +204,28 @@ func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser
} }
func UploadToHarbor(targetImageName string) (uploadOK bool) { func UploadToHarbor(targetImageName string) (uploadOK bool) {
pushResult := UploadToOctopusKindHarbor(targetImageName) pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close() defer pushResult.Close()
scanner := bufio.NewScanner(pushResult) scanner := bufio.NewScanner(pushResult)
for scanner.Scan() { for scanner.Scan() {
} }
fmt.Println() fmt.Println()
log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName) log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
fmt.Println() fmt.Println()
return true return true
} }
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest // TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) { func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
targetHarborHost = strings.TrimPrefix(targetHarborHost, "http://")
targetHarborHost = strings.TrimPrefix(targetHarborHost, "https://")
for _, imageName := range referenceImageList { for _, imageName := range referenceImageList {
// check image // check image
@@ -247,7 +255,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
} }
} }
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName targetImageName := targetHarborHost + targetProject + "/" + imageName
//fmt.Println(cmiiImageFullName) //fmt.Println(cmiiImageFullName)
//fmt.Println(targetImageName) //fmt.Println(targetImageName)

2
agent-operator/main.go
View File

@@ -150,7 +150,7 @@ func main() {
} }
// DLTU // DLTU
targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName) targetImageFullNameList := A_DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile, harborHostFullName)
// 是否需要更新 // 是否需要更新
if namespace != "" { if namespace != "" {

6
server/src/test/java/io/wdd/server/func/TestImageSyncScheduler.java
View File

@@ -38,7 +38,7 @@ public class TestImageSyncScheduler {
public void runImageSync() { public void runImageSync() {
ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of( ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
"cmii-uav-platform:5.3.0-061901" "cmii-uav-integration:5.4.0-070301"
)); ));
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of( ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
@@ -48,10 +48,10 @@ public class TestImageSyncScheduler {
Boolean downloadAndCompressOnly = false; Boolean downloadAndCompressOnly = false;
// String projectNamespace = "wdd"; // wdd // String projectNamespace = "wdd"; // wdd
String projectNamespace = "cqlyj"; // 重庆林业局 // String projectNamespace = "cqlyj"; // 重庆林业局
// String projectNamespace = "jlyd"; // 吉林移动 // String projectNamespace = "jlyd"; // 吉林移动
// String projectNamespace = "jxyd"; // 吉林移动 // String projectNamespace = "jxyd"; // 吉林移动
// String projectNamespace = "xmyd"; // 厦门移动 String projectNamespace = "xmyd"; // 厦门移动
// String innerWorkerAgentName = "Chengdu-amd64-65-lapwdd"; //wdd // String innerWorkerAgentName = "Chengdu-amd64-65-lapwdd"; //wdd
String innerWorkerAgentName = "Chengdu-amd64-071-ed6e2b7f"; //prod String innerWorkerAgentName = "Chengdu-amd64-071-ed6e2b7f"; //prod