zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Agent] [Octopus] - caqa fix bugs

Browse Source
This commit is contained in:
zeaslity
2024-01-30 15:54:33 +08:00
parent a471dabe46
commit 36c650301e
15 changed files with 327 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
cmii_operator/CmiiImageSync.go
View File

@@ -11,7 +11,9 @@ import (
"github.com/docker/docker/client"
"github.com/klauspost/pgzip"
"io"
"io/fs"
"os"
"strconv"
"strings"
)
@@ -137,6 +139,7 @@ func ImageTagFromSourceToTarget(sourceImageName, targetImageName string) bool {
log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error())
}
log.InfoF("[ImageTagFromSourceToTarget] - from %s to %s success!", sourceImageName, targetImageName)
return true
}
@@ -161,6 +164,45 @@ func ImagePushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadClo
return pushResult
}
func ImageTagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
for _, imageName := range referenceImageList {
// check image
cmiiImageFullName := imageName
if strings.HasPrefix(imageName, "cmii") {
cmiiImageFullName = CmiiHarborPrefix + imageName
}
targetProject := "cmii"
if strings.HasPrefix(imageName, "rancher") {
targetProject = "rancher"
}
if strings.Contains(imageName, "/") {
imageName = strings.Split(imageName, "/")[1]
}
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName
if ImageTagFromSourceToTarget(cmiiImageFullName, targetImageName) {
pushResult := ImagePushToOctopusKindHarbor(targetImageName)
if pushResult == nil {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
}
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
} else {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
}
}
return errorPushImageNameList
}
func ImagePullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
pullResult, err := apiClient.ImagePull(context.TODO(), imageName, types.ImagePullOptions{
All: false,
@@ -185,18 +227,18 @@ func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode
}
func ImagePullFromFileJson(filePathName string) {
func ImagePullCMiiFromFileJson(filePathName string) {
readFile, err := os.ReadFile(filePathName)
if err != nil {
log.ErrorF("[ImagePullFromFileJson] - file %s read error ! %s", filePathName, err.Error())
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s read error ! %s", filePathName, err.Error())
return
}
var resultMap map[string]string
err = json.Unmarshal(readFile, &readFile)
if err != nil {
log.ErrorF("[ImagePullFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
return
}
@@ -267,6 +309,53 @@ func ImagePullFromListAndCompressSplit(fullImageNameList []string, gzipFolder st
return errorPullImageList, errorGzipImageList
}
func ImageLoadFromFile(gzipFullPath string) bool {
openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error())
return false
}
loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFullPath, err.Error())
return false
}
log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFullPath, strconv.FormatBool(loadResponse.JSON))
scanner := bufio.NewScanner(loadResponse.Body)
for scanner.Scan() {
line := scanner.Text()
fmt.Println(line)
}
return true
}
func ImageLoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
if !strings.HasSuffix(folderPath, "/") {
folderPath += "/"
}
dirEntries, err := os.ReadDir(folderPath)
if err != nil {
log.ErrorF("[ImageLoadFromFolderPath] - error read folder %s error is %s", folderPath, err.Error())
return
}
// load gzip file
for _, dirEntry := range dirEntries {
if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
if !ImageLoadFromFile(folderPath + dirEntry.Name()) {
errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
}
}
}
return errorLoadImageNameList
}
func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
imageGetByName := ImageGetByName(targetImageName)

23
cmii_operator/CmiiImageSync_test.go
View File

@@ -97,6 +97,12 @@ func TestImagePushToOctopusKindHarbor(t *testing.T) {
fmt.Println("image push success!")
}
func TestImageLoadFromFile(t *testing.T) {
loadFromFile := ImageLoadFromFile("/root/octopus_image/cqga/cqga/cmlc=cmii=cmii-uav-user=5.2.0.tar.gz")
assert.Equal(t, loadFromFile, true, "image load error !")
}
func TestImageSaveToTarGZ(t *testing.T) {
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
@@ -147,3 +153,20 @@ func TestImagePruneAllCmiiImages(t *testing.T) {
utils.BeautifulPrintListWithTitle(errorRemoveImageNameList, "CMII Image Prune Error")
}
func TestImageTagFromSourceToTarget(t *testing.T) {
sourceImageName := "ossrs/srs:v5.0.195"
targetImageName := "harbor.wdd.io:8033/cmii/srs:v5.0.195"
if ImageTagFromSourceToTarget(sourceImageName, targetImageName) {
pushResult := ImagePushToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
}
}

32
cmii_operator/CmiiOperator.go
View File

@@ -8,6 +8,7 @@ import (
)
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
const OfflineDeployHarborHost = "harbor.wdd.io"
func FetchDemoImages(projectName string, gzipSplit bool) (errorPullImageList, errorGzipImageList []string) {
@@ -83,3 +84,34 @@ func FetchDependencyRepos(gzipSplit bool) (errorPullImageList, errorGzipImageLis
return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
}
func LoadSplitGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {
// list folder
projectGzipFolder := OfflineImageGzipFolderPrefix + projectName
errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(projectGzipFolder)...)
// read from json
errorPushImageNameList = append(errorPushImageNameList, ImageTagFromListAndPushToCHarbor(Cmii520DemoImageList, targetHarborHost)...)
// re-tag
// push
// todo clean host and harbor
// check harbor exits
return errorLoadImageNameList, errorPushImageNameList
}
func LoadSplitDepGzipImageToTargetHarbor(targetHarborHost string) (errorLoadImageNameList []string, errorPushImageNameList []string) {
middle := OfflineImageGzipFolderPrefix + "middle/"
rke := OfflineImageGzipFolderPrefix + "rke/"
errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(middle)...)
errorLoadImageNameList = append(errorLoadImageNameList, ImageLoadFromFolderPath(rke)...)
errorPushImageNameList = append(errorPushImageNameList, ImageTagFromListAndPushToCHarbor(MiddlewareAmd64, targetHarborHost)...)
errorPushImageNameList = append(errorPushImageNameList, ImageTagFromListAndPushToCHarbor(Rancher1204Amd64, targetHarborHost)...)
return errorLoadImageNameList, errorPushImageNameList
}

14
cmii_operator/CmiiOperator_test.go
View File

@@ -22,3 +22,17 @@ func TestFetchDependencyRepos(t *testing.T) {
utils.BeautifulPrintListWithTitle(errorGzipImageList, "dep errorGzipImageList")
}
func TestLoadSplitGzipImageToTargetHarbor(t *testing.T) {
errorLoadImageNameList, errorPushImageNameList := LoadSplitGzipImageToTargetHarbor("cqga", OfflineDeployHarborHost)
utils.BeautifulPrintListWithTitle(errorLoadImageNameList, "errorLoadImageNameList")
utils.BeautifulPrintListWithTitle(errorPushImageNameList, "errorPushImageNameList")
}
func TestLoadSplitDepGzipImageToTargetHarbor(t *testing.T) {
errorLoadImageNameList, errorPushImageNameList := LoadSplitDepGzipImageToTargetHarbor(OfflineDeployHarborHost)
utils.BeautifulPrintListWithTitle(errorLoadImageNameList, "errorLoadImageNameList")
utils.BeautifulPrintListWithTitle(errorPushImageNameList, "errorPushImageNameList")
}

70
cmii_operator/DependencyConfig.go
View File

@@ -21,10 +21,13 @@ var MiddlewareAmd64 = []string{
"kubernetesui/dashboard:v2.0.1",
"kubernetesui/metrics-scraper:v1.0.4",
"ossrs/srs:v4.0-r3",
"ossrs/srs:v5.0.195",
"nginx:1.21.3",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"busybox:latest",
"harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
"harbor.cdcyy.com.cn/cmii/cmii-srs-oss-adaptor:2023-SA",
}
var Rancher1204Amd64 = []string{
@@ -91,3 +94,70 @@ var Rancher1204Amd64 = []string{
"rancher/sonobuoy-sonobuoy:v0.16.3",
"rancher/system-upgrade-controller:v0.6.2",
}
var Cmii520DemoImageList = []string{
"cmii-admin-data:5.2.0",
"cmii-admin-gateway:5.2.0",
"cmii-admin-user:5.2.0",
"cmii-app-release:4.2.0-validation",
"cmii-open-gateway:5.2.0",
"cmii-suav-supervision:5.2.0",
"cmii-uav-airspace:5.2.0",
"cmii-uav-alarm:5.2.0",
"cmii-uav-autowaypoint:4.1.6-cm-0828",
"cmii-uav-brain:5.2.0",
"cmii-uav-cloud-live:5.2.0",
"cmii-uav-clusters:5.2.0",
"cmii-uav-cms:5.2.0",
"cmii-uav-data-post-process:5.2.0",
"cmii-uav-depotautoreturn:4.2.0",
"cmii-uav-developer:5.2.0-25858",
"cmii-uav-device:5.2.0",
"cmii-uav-emergency:5.2.0",
"cmii-uav-gateway:5.2.0",
"cmii-uav-gis-server:5.2.0",
"cmii-uav-grid-datasource:5.2.0-24810",
"cmii-uav-grid-engine:5.1.0",
"cmii-uav-grid-manage:5.1.0",
"cmii-uav-industrial-portfolio:5.2.0-25268-12",
"cmii-uav-integration:5.2.0-25447",
"cmii-uav-kpi-monitor:5.2.0",
"cmii-uav-logger:5.2.0",
"cmii-uav-material-warehouse:5.2.0",
"cmii-uav-mission:5.2.0-25840",
"cmii-uav-mqtthandler:5.2.0-25340-1",
"cmii-uav-notice:5.2.0",
"cmii-uav-oauth:5.2.0",
"cmii-uav-process:5.2.0",
"cmii-uav-surveillance:5.2.0-21794",
"cmii-uav-threedsimulation:5.2.0",
"cmii-uav-tower:5.3.0",
"cmii-uav-user:5.2.0",
"cmii-uav-waypoint:5.2.0-011102",
"cmii-suav-platform-supervision:5.2.0",
"cmii-suav-platform-supervisionh5:5.2.0",
"cmii-uav-platform:5.2.0",
"cmii-uav-platform-ai-brain:5.2.0",
"cmii-uav-platform-armypeople:5.2.0-24538",
"cmii-uav-platform-base:5.2.0",
"cmii-uav-platform-cms-portal:5.2.0",
"cmii-uav-platform-detection:5.2.0",
"cmii-uav-platform-emergency-rescue:5.2.0",
"cmii-uav-platform-hljtt:5.2.0",
"cmii-uav-platform-jiangsuwenlv:4.1.3-jiangsu-0427",
"cmii-uav-platform-logistics:5.2.0",
"cmii-uav-platform-media:5.2.0",
"cmii-uav-platform-multiterminal:5.2.0",
"cmii-uav-platform-mws:5.2.0",
"cmii-uav-platform-oms:5.2.0",
"cmii-uav-platform-open:5.2.0",
"cmii-uav-platform-qingdao:4.1.6-24238-qingdao",
"cmii-uav-platform-qinghaitourism:4.1.0-21377-0508",
"cmii-uav-platform-security:4.1.6",
"cmii-uav-platform-securityh5:5.2.0",
"cmii-uav-platform-seniclive:5.2.0",
"cmii-uav-platform-share:5.2.0",
"cmii-uav-platform-splice:5.2.0",
"cmii-uav-platform-threedsimulation:5.2.0-21392",
"cmii-uav-platform-visualization:5.2.0",
}