zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[ Cmii ] [ Octopus ] - image sync refresh

Browse Source
This commit is contained in:
zeaslity
2024-03-18 16:38:54 +08:00
parent 5e80d7baad
commit df44d6eef9
31 changed files with 360 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmii_operator/CmiiK8sOperator_test.go
View File

@@ -67,7 +67,7 @@ func TestRestartCmiiFrontendDeployment(t *testing.T) {
func TestRestartCmiiDeployment(t *testing.T) {
cmiiEnv := integration
appName := "cmii-uav-cloud-live"
appName := "cmii-uav-device"
kill := CmiiOperator.DeploymentRestartByKill(cmiiEnv, appName)
assert.Equal(t, kill, true, "have unhealthy pod !")
@@ -237,7 +237,7 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
cmiiEnv := demo
appName := "cmii-uav-platform"
newTag := "5.4.0"
newTag := "5.4.0-25263-0311"
tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
assert.Equal(t, tag, true, "update image tag failed !")

157
cmii_operator/CmiiOperator.go
View File

@@ -2,7 +2,9 @@ package cmii_operator
import (
"errors"
"io/fs"
"os"
"path/filepath"
"strings"
"wdd.io/agent-go/executor"
"wdd.io/agent-go/utils"
@@ -12,16 +14,128 @@ import (
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
const OfflineDeployHarborHost = "harbor.wdd.io"
const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.28.60"
const DirectPushDeployHarborHost = "36.134.71.138"
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
type ImageSyncEntity struct {
ProjectName string
ProjectVersion string
DirectHarborHost string
PushToDemoMinio bool
}
type ImageSyncResult struct {
ErrorPullImageList []string
ErrorGzipImageList []string
ErrorPushImageNameList []string
RealImageNameList []string
RealGzipFileNameList []string
AllCmiiImageNameList []string
}
func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResult ImageSyncResult) {
var realCmiiImageList []string
var errorPullImageList []string
var errorGzipImageList []string
var allCmiiImageNameList []string
var allGzipFileNameList []string
var errorPushImageNameList []string
var gzipFolderFullPath string
// get all image name by Name or Version
// pull images
// compress
if sync.ProjectVersion == "" {
// get version
if sync.DirectHarborHost == "" {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, true)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectVersion
} else {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchVersionImages(sync.ProjectVersion, false)
}
} else {
// get demo images
if sync.DirectHarborHost == "" {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, true)
gzipFolderFullPath = OfflineImageGzipFolderPrefix + sync.ProjectName
} else {
errorPullImageList, errorGzipImageList, allCmiiImageNameList = FetchDemoImages(sync.ProjectName, false)
}
}
realCmiiImageList = append(realCmiiImageList, remove(allCmiiImageNameList, errorPullImageList)...)
realCmiiImageList = append(realCmiiImageList, remove(allCmiiImageNameList, errorGzipImageList)...)
// direct push if can
if sync.DirectHarborHost != "" {
// push to
errorPushImageNameList = image.TagFromListAndPushToCHarbor(realCmiiImageList, sync.DirectHarborHost)
// build
imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
imageSyncResult.ErrorPullImageList = errorPullImageList
imageSyncResult.ErrorGzipImageList = errorGzipImageList
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
// no gzip file
return imageSyncResult
}
// get gzip file name list
err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.ErrorF("error getting gzip file name list 1! %s", err.Error())
}
if !d.IsDir() {
allGzipFileNameList = append(allGzipFileNameList, d.Name())
}
return nil
})
if err != nil {
log.ErrorF("error getting gzip file name list 2! %s", err.Error())
}
// push to demo minio
if sync.PushToDemoMinio {
log.InfoF("pretend to push to minio !")
// create path
// push
}
// build
imageSyncResult.AllCmiiImageNameList = allCmiiImageNameList
imageSyncResult.ErrorPullImageList = errorPullImageList
imageSyncResult.ErrorGzipImageList = errorGzipImageList
imageSyncResult.ErrorPushImageNameList = errorPushImageNameList
imageSyncResult.RealGzipFileNameList = allGzipFileNameList
// no gzip file
return imageSyncResult
}
func remove(s1, s2 []string) []string {
m := make(map[string]struct{}, len(s2))
for _, v := range s2 {
m[v] = struct{}{}
}
res := make([]string, 0, len(s1))
for _, v := range s1 {
if _, ok := m[v]; !ok {
res = append(res, v)
}
}
return res
}
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
// generate a project folder
err := os.MkdirAll(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
return errorPullImageList, errorGzipImageList
return errorPullImageList, errorGzipImageList, allCmiiImageName
}
}
@@ -54,9 +168,13 @@ func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, er
)
// download image
backendPull := image.ImagePullFromCmiiHarborByMap(backendMap, true)
frontendPull := image.ImagePullFromCmiiHarborByMap(frontendMap, true)
srsPull := image.ImagePullFromCmiiHarborByMap(srsMap, true)
backendFullNameList, backendPull := image.PullFromCmiiHarborByMap(backendMap, true)
frontendFullNameList, frontendPull := image.PullFromCmiiHarborByMap(frontendMap, true)
srsFullNameList, srsPull := image.PullFromCmiiHarborByMap(srsMap, true)
allCmiiImageName = append(allCmiiImageName, backendFullNameList...)
allCmiiImageName = append(allCmiiImageName, frontendFullNameList...)
allCmiiImageName = append(allCmiiImageName, srsFullNameList...)
// compress image
if gzipSplit {
@@ -84,32 +202,30 @@ func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, er
errorPullImageList = append(errorPullImageList, frontendPull...)
errorPullImageList = append(errorPullImageList, srsPull...)
return errorPullImageList, errorGzipImageList
return errorPullImageList, errorGzipImageList, allCmiiImageName
}
func FetchVersionImages(cmiiVersion string, gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
func FetchVersionImages(cmiiVersion string, shouldGzip bool) (errorPullImageList, errorGzipImageList, allCmiiImageName []string) {
// generate a project folder
err := os.MkdirAll(OfflineImageGzipFolderPrefix+cmiiVersion, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+cmiiVersion, err.Error())
return errorPullImageList, errorGzipImageList
return errorPullImageList, errorGzipImageList, allCmiiImageName
}
}
backendMap := CmiiBackendAppMap
frontendMap := CmiiFrontendAppMap
for app, _ := range backendMap {
for app := range backendMap {
backendMap[app] = cmiiVersion
}
for app, _ := range frontendMap {
for app := range frontendMap {
frontendMap[app] = cmiiVersion
}
var allCmiiImageName []string
allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(backendMap)...)
allCmiiImageName = append(allCmiiImageName, image.ConvertCMiiImageMapToList(frontendMap)...)
@@ -130,12 +246,16 @@ func FetchVersionImages(cmiiVersion string, gzipSplit bool) (errorPullImageList,
}
}
utils.BeautifulPrint(allCmiiImageName)
utils.BeautifulPrintListWithTitle(allCmiiImageName, "Cmii Version Image => "+cmiiVersion)
// do work
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(allCmiiImageName, OfflineImageGzipFolderPrefix+cmiiVersion)
if shouldGzip {
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(allCmiiImageName, OfflineImageGzipFolderPrefix+cmiiVersion)
} else {
errorPullImageList = image.PullFromFullNameList(allCmiiImageName)
}
return errorPullImageList, errorGzipImageList
return errorPullImageList, errorGzipImageList, allCmiiImageName
}
func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
@@ -151,14 +271,13 @@ func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageLis
pull, gzipImageList := image.PullFromListAndCompressSplit(image.Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
}
func LoadSplitGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
// list folder
projectGzipFolder := OfflineImageGzipFolderPrefix + projectName
errorLoadImageNameList = append(errorLoadImageNameList, image.ImageLoadFromFolderPath(projectGzipFolder)...)
errorLoadImageNameList = append(errorLoadImageNameList, image.LoadFromFolderPath(projectGzipFolder)...)
// read from json
errorPushImageNameList = append(errorPushImageNameList, image.TagFromListAndPushToCHarbor(image.Cmii520DemoImageList, targetHarborHost)...)

22
cmii_operator/CmiiOperator_test.go
View File

@@ -7,18 +7,19 @@ import (
func TestFetchDemoImages(t *testing.T) {
errorPullImageList, errorGzipImageList := FetchDemoImages("shls", true)
errorPullImageList, errorGzipImageList, allCmiiImageName := FetchDemoImages("shls", true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList")
utils.BeautifulPrintListWithTitle(allCmiiImageName, "cmii allCmiiImageName")
}
func TestFetchVersionImages(t *testing.T) {
errorPullImageList, errorGzipImageList := FetchVersionImages("5.4.0", true)
errorPullImageList, errorGzipImageList, allCmiiImageName := FetchVersionImages("5.4.0", true)
utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList")
utils.BeautifulPrintListWithTitle(allCmiiImageName, "cmii allCmiiImageName")
}
func TestFetchDependencyRepos(t *testing.T) {
@@ -27,11 +28,10 @@ func TestFetchDependencyRepos(t *testing.T) {
utils.BeautifulPrintListWithTitle(errorPullImageList, "dep errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "dep errorGzipImageList")
}
func TestLoadSplitGzipImageToTargetHarbor(t *testing.T) {
errorLoadImageNameList, errorPushImageNameList := LoadSplitGzipImageToTargetHarbor("cqga", OfflineDeployHarborHost)
errorLoadImageNameList, errorPushImageNameList := LoadSplitCmiiGzipImageToTargetHarbor("xmyd", DirectPushDeployHarborHost)
utils.BeautifulPrintListWithTitle(errorLoadImageNameList, "errorLoadImageNameList")
utils.BeautifulPrintListWithTitle(errorPushImageNameList, "errorPushImageNameList")
@@ -43,3 +43,15 @@ func TestLoadSplitDepGzipImageToTargetHarbor(t *testing.T) {
utils.BeautifulPrintListWithTitle(errorLoadImageNameList, "errorLoadImageNameList")
utils.BeautifulPrintListWithTitle(errorPushImageNameList, "errorPushImageNameList")
}
func TestImageSyncEntity_PullFromEntityAndSyncConditionally(t *testing.T) {
imageSyncEntity := ImageSyncEntity{
ProjectVersion: "5.4.0",
DirectHarborHost: "36.134.71.138",
}
imageSyncResult := imageSyncEntity.PullFromEntityAndSyncConditionally()
utils.BeautifulPrint(imageSyncResult)
}

BIN
cmii_operator/actual_project/proxy_project/linux/socks5_linux_amd64
View File

Binary file not shown.

153
cmii_operator/actual_project/zyga/operator.go
View File

@@ -1,6 +1,7 @@
package main
import (
"wdd.io/agent-go/utils"
"wdd.io/cmii_operator"
)
@@ -37,87 +38,87 @@ func main() {
//}
// restart all backend
cmii_operator.RestartCmiiBackendDeployment(realNamespace)
//cmii_operator.RestartCmiiBackendDeployment(realNamespace)
//cmii_operator.RestartCmiiFrontendDeployment(realNamespace)
//backMap := map[string]string{
// "cmii-admin-data": "5.4.0",
// "cmii-admin-gateway": "5.4.0",
// "cmii-admin-user": "5.4.0",
// "cmii-app-release": "4.2.0-validation",
// "cmii-open-gateway": "5.4.0",
// "cmii-suav-supervision": "5.2.0",
// "cmii-uav-airspace": "5.4.0",
// "cmii-uav-alarm": "5.4.0",
// "cmii-uav-autowaypoint": "4.1.6-cm",
// "cmii-uav-brain": "5.4.0",
// "cmii-uav-cloud-live": "5.4.0",
// "cmii-uav-clusters": "5.2.0",
// "cmii-uav-cms": "5.3.0",
// "cmii-uav-data-post-process": "5.4.0",
// "cmii-uav-depotautoreturn": "5.4.0",
// "cmii-uav-developer": "5.4.0",
// "cmii-uav-device": "5.4.0-25916",
// "cmii-uav-emergency": "5.3.0",
// "cmii-uav-gateway": "5.4.0",
// "cmii-uav-gis-server": "5.4.0",
// "cmii-uav-grid-datasource": "5.2.0-24810",
// "cmii-uav-grid-engine": "5.1.0",
// "cmii-uav-grid-manage": "5.1.0",
// "cmii-uav-industrial-portfolio": "5.4.0-27348-1",
// "cmii-uav-integration": "5.4.0-25916",
// "cmii-uav-kpi-monitor": "5.4.0",
// "cmii-uav-logger": "5.4.0",
// "cmii-uav-material-warehouse": "5.4.0",
// "cmii-uav-mission": "5.4.0-26462-0307",
// "cmii-uav-mqtthandler": "5.4.0-25916",
// "cmii-uav-multilink": "5.4.0",
// "cmii-uav-notice": "5.4.0",
// "cmii-uav-oauth": "5.4.0",
// "cmii-uav-process": "5.4.0",
// "cmii-uav-surveillance": "5.4.0-25916",
// "cmii-uav-threedsimulation": "5.1.0",
// "cmii-uav-tower": "5.4.0",
// "cmii-uav-user": "5.4.0",
// "cmii-uav-waypoint": "5.4.0-26768",
//}
backMap := map[string]string{
//"cmii-admin-data": "5.4.0",
//"cmii-admin-gateway": "5.4.0",
//"cmii-admin-user": "5.4.0",
//"cmii-app-release": "4.2.0-validation",
//"cmii-open-gateway": "5.4.0",
//"cmii-suav-supervision": "5.2.0",
//"cmii-uav-airspace": "5.4.0",
//"cmii-uav-alarm": "5.4.0",
//"cmii-uav-autowaypoint": "4.1.6-cm",
//"cmii-uav-brain": "5.4.0",
//"cmii-uav-cloud-live": "5.4.0",
//"cmii-uav-clusters": "5.2.0",
//"cmii-uav-cms": "5.3.0",
//"cmii-uav-data-post-process": "5.4.0",
//"cmii-uav-depotautoreturn": "5.4.0",
//"cmii-uav-developer": "5.4.0",
//"cmii-uav-device": "5.4.0-25916",
//"cmii-uav-emergency": "5.3.0",
//"cmii-uav-gateway": "5.4.0",
//"cmii-uav-gis-server": "5.4.0",
//"cmii-uav-grid-datasource": "5.2.0-24810",
//"cmii-uav-grid-engine": "5.1.0",
//"cmii-uav-grid-manage": "5.1.0",
//"cmii-uav-industrial-portfolio": "5.4.0-27348-1",
//"cmii-uav-integration": "5.4.0-25916",
//"cmii-uav-kpi-monitor": "5.4.0",
//"cmii-uav-logger": "5.4.0",
//"cmii-uav-material-warehouse": "5.4.0",
//"cmii-uav-mission": "5.4.0-26462-0307",
//"cmii-uav-mqtthandler": "5.4.0-25916",
//"cmii-uav-multilink": "5.4.0",
//"cmii-uav-notice": "5.4.0",
//"cmii-uav-oauth": "5.4.0",
//"cmii-uav-process": "5.4.0",
"cmii-uav-surveillance": "5.4.0-leaflet",
//"cmii-uav-threedsimulation": "5.1.0",
//"cmii-uav-tower": "5.4.0",
//"cmii-uav-user": "5.4.0",
//"cmii-uav-waypoint": "5.4.0-26768",
}
//
//frontMap := map[string]string{
// "cmii-suav-platform-supervision": "5.4.0",
// "cmii-suav-platform-supervisionh5": "5.4.0",
// "cmii-uav-platform": "5.4.0-25263",
// "cmii-uav-platform-ai-brain": "5.4.0",
// "cmii-uav-platform-armypeople": "5.4.0",
// "cmii-uav-platform-base": "5.4.0",
// "cmii-uav-platform-cms-portal": "5.4.0",
// "cmii-uav-platform-detection": "5.4.0",
// "cmii-uav-platform-emergency-rescue": "5.2.0",
// "cmii-uav-platform-hljtt": "5.3.0-hjltt",
// "cmii-uav-platform-jiangsuwenlv": "4.1.3-jiangsu-0427",
// "cmii-uav-platform-logistics": "5.4.0",
// "cmii-uav-platform-media": "5.4.0",
// "cmii-uav-platform-multiterminal": "5.4.0",
// "cmii-uav-platform-mws": "5.4.0",
// "cmii-uav-platform-oms": "5.4.0",
// "cmii-uav-platform-open": "5.4.0",
// "cmii-uav-platform-qingdao": "4.1.6-24238-qingdao",
// "cmii-uav-platform-qinghaitourism": "4.1.0-21377-0508",
// "cmii-uav-platform-security": "4.1.6",
// "cmii-uav-platform-securityh5": "5.4.0",
// "cmii-uav-platform-seniclive": "5.2.0",
// "cmii-uav-platform-share": "5.4.0",
// "cmii-uav-platform-splice": "5.4.0",
// "cmii-uav-platform-threedsimulation": "5.2.0-21392",
// "cmii-uav-platform-visualization": "5.2.0",
//}
frontMap := map[string]string{
"cmii-uav-platform": "5.4.0-leaflet",
//"cmii-suav-platform-supervision": "5.4.0",
//"cmii-suav-platform-supervisionh5": "5.4.0",
//"cmii-uav-platform-ai-brain": "5.4.0",
//"cmii-uav-platform-armypeople": "5.4.0",
//"cmii-uav-platform-base": "5.4.0",
//"cmii-uav-platform-cms-portal": "5.4.0",
//"cmii-uav-platform-detection": "5.4.0",
//"cmii-uav-platform-emergency-rescue": "5.2.0",
//"cmii-uav-platform-hljtt": "5.3.0-hjltt",
//"cmii-uav-platform-jiangsuwenlv": "4.1.3-jiangsu-0427",
//"cmii-uav-platform-logistics": "5.4.0",
//"cmii-uav-platform-media": "5.4.0",
//"cmii-uav-platform-multiterminal": "5.4.0",
//"cmii-uav-platform-mws": "5.4.0",
//"cmii-uav-platform-oms": "5.4.0",
//"cmii-uav-platform-open": "5.4.0",
//"cmii-uav-platform-qingdao": "4.1.6-24238-qingdao",
//"cmii-uav-platform-qinghaitourism": "4.1.0-21377-0508",
//"cmii-uav-platform-security": "4.1.6",
//"cmii-uav-platform-securityh5": "5.4.0",
//"cmii-uav-platform-seniclive": "5.2.0",
//"cmii-uav-platform-share": "5.4.0",
//"cmii-uav-platform-splice": "5.4.0",
//"cmii-uav-platform-threedsimulation": "5.2.0-21392",
//"cmii-uav-platform-visualization": "5.2.0",
}
//
//cmii_operator.CmiiOperator = k8sOperator
cmii_operator.CmiiOperator = k8sOperator
//
//result := cmii_operator.UpdateCmiiImageTagFromNameTagMap(realNamespace, backMap)
//utils.BeautifulPrint(result)
//
//result = cmii_operator.UpdateCmiiImageTagFromNameTagMap(realNamespace, frontMap)
//utils.BeautifulPrint(result)
result := cmii_operator.UpdateCmiiImageTagFromNameTagMap(realNamespace, backMap)
utils.BeautifulPrint(result)
result = cmii_operator.UpdateCmiiImageTagFromNameTagMap(realNamespace, frontMap)
utils.BeautifulPrint(result)
}

2
cmii_operator/image/CmiiDependencyImageConfig.go
View File

@@ -25,6 +25,7 @@ var MiddlewareAmd64 = []string{
"nginx:1.21.3",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"jerrychina2020/rke-tools:v0.175-linux",
"busybox:latest",
}
@@ -85,7 +86,6 @@ var Rancher1204Amd64 = []string{
"rancher/rancher-webhook:v0.1.0-beta9",
"rancher/rancher:v2.5.7",
"rancher/rke-tools:v0.1.72",
"jerrychina2020/rke-tools:v0.175-linux",
"rancher/security-scan:v0.1.14",
"rancher/security-scan:v0.2.2",
"rancher/shell:v0.1.6",

79
cmii_operator/image/CmiiImageSync.go
View File

@@ -17,6 +17,7 @@ import (
"strings"
"wdd.io/agent-go/executor"
"wdd.io/agent-go/logger"
"wdd.io/agent-go/utils"
)
var apiClient = newClient()
@@ -109,7 +110,7 @@ func Delete(imageName string) []types.ImageDeleteResponseItem {
func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
apiClient.ImagesPrune(context.TODO(), filters.Args{})
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
imageGetAll := GetAll()
@@ -134,6 +135,8 @@ func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
}
}
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
return errorRemoveImageNameList
}
@@ -175,11 +178,15 @@ func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser)
return pushResult
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
for _, imageName := range referenceImageList {
// check image
// harbor.cdcyy.cn
cmiiImageFullName := imageName
if strings.HasPrefix(imageName, "cmii") {
cmiiImageFullName = CmiiHarborPrefix + imageName
}
@@ -190,7 +197,10 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
}
if strings.Contains(imageName, "/") {
//
imageName = strings.Split(imageName, "/")[1]
} else {
}
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName
@@ -232,14 +242,14 @@ func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
return pullResult
}
func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (errorPullImageList []string) {
func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
fullImageNameList := ConvertCMiiImageMapToList(imageVersionMap)
return ImagePullFromFullNameList(fullImageNameList)
fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap)
return fullImageNameList, PullFromFullNameList(fullImageNameList)
}
func ImagePullCMiiFromFileJson(filePathName string) {
func PullCmiiFromFileJson(filePathName string) {
readFile, err := os.ReadFile(filePathName)
if err != nil {
@@ -274,7 +284,8 @@ func ImagePullCMiiFromFileJson(filePathName string) {
}
func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
// PullFromFullNameList 根据镜像名列表拉取全部的镜像
func PullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
for _, dep := range fullImageNameList {
@@ -301,7 +312,7 @@ func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList [
func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
errorPullImageList = ImagePullFromFullNameList(fullImageNameList)
errorPullImageList = PullFromFullNameList(fullImageNameList)
// generate a project folder
err := os.MkdirAll(gzipFolder, os.ModeDir)
@@ -311,16 +322,22 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
}
}
var tarGzipFileNameList []string
for _, image := range fullImageNameList {
if !SaveToTarGZ(image, gzipFolder) {
errorGzipImageList = append(errorGzipImageList, image)
continue
}
tarGzipFileNameList = append(tarGzipFileNameList, convertImageGzipFileName(image))
}
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
return errorPullImageList, errorGzipImageList
}
func ImageLoadFromFile(gzipFullPath string) bool {
func LoadFromGzipFilePath(gzipFullPath string) bool {
openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error())
@@ -344,7 +361,7 @@ func ImageLoadFromFile(gzipFullPath string) bool {
return true
}
func ImageLoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
if !strings.HasSuffix(folderPath, "/") {
folderPath += "/"
}
@@ -358,7 +375,7 @@ func ImageLoadFromFolderPath(folderPath string) (errorLoadImageNameList []string
// load gzip file
for _, dirEntry := range dirEntries {
if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
if !ImageLoadFromFile(folderPath + dirEntry.Name()) {
if !LoadFromGzipFilePath(folderPath + dirEntry.Name()) {
errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
}
}
@@ -385,6 +402,7 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
for _, repoTag := range imageGetByName.RepoTags {
if !strings.Contains(repoTag, "8033") {
realImageTag = repoTag
break
}
}
@@ -393,9 +411,9 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
folderPathPrefix += "/"
}
_ = os.MkdirAll(folderPathPrefix, os.ModeDir)
gzipImageFile = folderPathPrefix + gzipImageFile
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFile)
_ = os.Remove(gzipImageFile)
tarFile, err := os.Create(gzipImageFile)
if err != nil {
@@ -420,9 +438,19 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
return true
}
// convertImageGzipFileName 必须输出长度完全一致的内容
// convertImageGzipFileName 必须输出长度为4的内容 =出现得次数为3
func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmlc=cmii=cmii-uav-platform=5.4.0.tar.gz
// rancher/fleet:v0.3.4
// ossr/srs:v5.0.1 ==> docker=cmii=srs=v5.0.1.tar.gz
// nginx:latest
// bitnami/minio:2022.5.4
// simonrupf/chronyd:0.4.3
// 10.1.1.1:8033/cmii/ok:1.2 不支持 不允许存在
split := strings.Split(imageRepoTag, ":")
//log.DebugF(" %s to %s", imageRepoTag, split)
if len(split) == 1 {
@@ -433,28 +461,23 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
first := strings.Split(split[0], "/")
//log.DebugF(" split[0] %s to %s", split[0], first)
if len(first) == 3 {
if strings.Contains(first[0], "cdcyy") {
gzipFileName += "cmlc="
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0
// docker.io/ossr/srs:v5.0.1
if strings.HasPrefix(split[0], CmiiHarborPrefix) {
gzipFileName += "cmlc=cmii="
} else {
gzipFileName += "docker="
gzipFileName += "docker=cmii="
}
gzipFileName += first[1]
gzipFileName += "="
gzipFileName += first[2]
gzipFileName += "="
} else if len(first) == 4 {
if strings.Contains(first[0], "cdcyy") {
gzipFileName += "cmlc="
} else {
gzipFileName += "docker="
// harbor.cdcyy.cn/cmii/ossr/srs:v5.0.1
if !strings.HasPrefix(split[0], CmiiHarborPrefix) {
return imageRepoTag
}
gzipFileName += first[1]
gzipFileName += "="
gzipFileName += first[2]
gzipFileName += "cmlc=cmii="
gzipFileName += first[3]
gzipFileName += "="
} else if len(first) == 2 {
// bitnami/redis
@@ -465,7 +488,7 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
gzipFileName += first[1]
gzipFileName += "="
} else if len(first) == 1 {
// nginx
// nginx:latest
return "docker=library=" + split[0] + "=" + split[1] + ".tar.gz"
}

10
cmii_operator/image/CmiiImageSync_test.go
View File

@@ -100,7 +100,7 @@ func TestImagePushToOctopusKindHarbor(t *testing.T) {
}
func TestImageLoadFromFile(t *testing.T) {
loadFromFile := ImageLoadFromFile("/root/octopus_image/cqga/cqga/cmlc=cmii=cmii-uav-user=5.2.0.tar.gz")
loadFromFile := LoadFromGzipFilePath("/root/octopus_image/cqga/cqga/cmlc=cmii=cmii-uav-user=5.2.0.tar.gz")
assert.Equal(t, loadFromFile, true, "image load error !")
}
@@ -117,20 +117,12 @@ func TestConvertImageGzipFileName(t *testing.T) {
test := []string{
"bitnami/redis:6.2.6-debian-10-r0",
"bitnami/mysql:8.1.0-debian-11-r42",
"simonrupf/chronyd:0.4.3",
"bitnami/bitnami-shell:11-debian-11-r136",
"harbor.cdcyy.com.cn/cmii/cmii-rtsp-operator:v4.1.0",
"harbor.cdcyy.com.cn/cmii/ossrs/srs:v4.0.136",
"bitnami/rabbitmq:3.9.12-debian-10-r3",
"ossrs/srs:v4.0.136",
"emqx/emqx:4.2.12",
"nacos/nacos-server:v2.1.2",
"mongo:5.0",
"bitnami/minio:2023.5.4",
"kubernetesui/metrics-scraper:v1.0.4",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"busybox:latest",
"busybox",
}

49
cmii_operator/image/image_sync.sh
View File

@@ -6,29 +6,29 @@ cmii_image_list=(
docker=ossrs=srs=v4.0.136.tar.gz
)
middle_image_list=(
cmlc=cmii=nfs-subdir-external-provisioner=v4.0.2.tar.gz
docker=busybox=latest.tar.gz
cmlc=cmii=srs=v4.0-r3.tar.gz
docker=emqx=emqx=4.2.12.tar.gz
docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz
docker=kubernetesui=dashboard=v2.0.1.tar.gz
docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz
docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz
docker=bitnami=minio=2022.5.4.tar.gz
docker=mongo=5.0.tar.gz
docker=library=busybox=latest.tar.gz
docker=bitnami=minio=2023.5.4.tar.gz
docker=nacos=nacos-server=v2.1.2-slim.tar.gz
docker=library=mongo=5.0.tar.gz
docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz
docker=nginx=1.21.3.tar.gz
docker=library=nginx=1.21.3.tar.gz
docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz
docker=ossrs=srs=v4.0.136.tar.gz
docker=library=rabbitmq=3.9-management.tar.gz
docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz
docker=ossrs=srs=v5.0.195.tar.gz
docker=library=redis=6.0.20-alpine.tar.gz
docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz
docker=rabbitmq=3.9-management.tar.gz
docker=nacos=nacos-server=v2.1.2-slim.tar.gz
docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz
docker=redis=6.0.20-alpine.tar.gz
docker=ossrs=srs=v4.0.136.tar.gz
docker=bitnami=redis=6.2.6-debian-10-r0.tar.gz
docker=ossrs=srs=v4.0-r3.tar.gz
docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz
docker=ossrs=srs=v5.0.195.tar.gz
docker=emqx=emqx=4.2.12.tar.gz
docker=simonrupf=chronyd=0.4.3.tar.gz
)
rke_image_list=(
@@ -161,19 +161,20 @@ image_load_to_harbor() {
echo
done
# for image in "${middle_image_list[@]}"; do
# echo "start to load => $image"
# docker load <"$image"
# echo ""
# local app_name_prefix=$(echo $image | cut -d "=" -f2)
# local app_name=$(echo $image | cut -d "=" -f3)
# local ccc=$(echo $image | cut -d "=" -f4)
# local app_tag="${ccc%.tar.gz}"
# echo "from $app_name_prefix/$app_name:$app_tag ==> $target_harbor_host/rancher/$app_name:$app_tag"
# echo "start to push => $target_harbor_host/rancher/$app_name:$app_tag"
# docker login -u admin -p V2ryStr@ngPss $target_harbor_host
# docker push "$target_harbor_host/rancher/$app_name:$app_tag"
# done
for image in "${middle_image_list[@]}"; do
echo "start to load => $image"
docker load <"$image"
echo ""
# docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz
local app_name_prefix=$(echo $image | cut -d "=" -f2)
local app_name=$(echo $image | cut -d "=" -f3)
local ccc=$(echo $image | cut -d "=" -f4)
local app_tag="${ccc%.tar.gz}"
echo "from $app_name_prefix/$app_name:$app_tag ==> $target_harbor_host/cmii/$app_name:$app_tag"
echo "start to push => $target_harbor_host/cmii/$app_name:$app_tag"
docker login -u admin -p V2ryStr@ngPss $target_harbor_host
docker push "$target_harbor_host/cmii/$app_name:$app_tag"
done
}

2
cmii_operator/log/cmii-update-log.txt
View File

@@ -109,3 +109,5 @@
2024-03-08-14-15-41 zyga cmii-uav-platform-share 8033/cmii/cmii-uav-platform-share5.4.0
2024-03-08-14-15-55 zyga cmii-uav-platform-multiterminal 8033/cmii/cmii-uav-platform-multiterminal5.4.0
2024-03-08-15-16-14 uavcloud-demo cmii-uav-platform 5.4.0-25263 5.4.0
2024-03-11-11-20-15 zyga cmii-uav-surveillance 5.4.0 5.4.0-leaflet
2024-03-11-15-42-15 uavcloud-demo cmii-uav-platform 5.4.0 5.4.0-25263-0311