[agent-operator] - update dltu part

This commit is contained in:
zeaslity
2024-04-26 17:51:11 +08:00
parent 318a5a8026
commit 021690f5c4
12 changed files with 338 additions and 197 deletions

View File

@@ -120,6 +120,15 @@ func WordSpaceCompletion(source string, totalLength int) string {
return source
}
// IsFileOrDir 如果是目录则返回true是文件则返回false
func IsFileOrDir(path string) bool {
info, err := os.Stat(path)
if err != nil {
return false
}
return info.IsDir()
}
// FileExists 文件存在返回true不存在返回false如果文件是一个目录也返回false
func FileExists(fileFullPath string) bool {
_, err := os.Stat(fileFullPath)

View File

@@ -3,6 +3,8 @@ package a_status
import (
"fmt"
"github.com/shirou/gopsutil/v3/disk"
"os"
"path/filepath"
"regexp"
"time"
)
@@ -115,3 +117,45 @@ func MatchNeededDisk(deviceName string) bool {
return true
}
func dirSize(path string) (int64, error) {
var size int64
err := filepath.Walk(path, func(filePath string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
size += info.Size()
}
return nil
})
return size, err
}
//func DiskUsages() {
//
//
// root := "/"
// maxDepth := 3
//
// fmt.Println("Scanning directories...")
// err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
// if err != nil {
// return err
// }
// if info.IsDir() && filepath.DirCount(path) <= maxDepth {
// size, err := dirSize(path)
// if err != nil {
// fmt.Printf("Error: %v\n", err)
// return nil
// }
// fmt.Printf("%s: %d bytes\n", path, size)
// }
// return nil
// })
// if err != nil {
// fmt.Println(err)
// }
//
//
//}

View File

@@ -2,6 +2,7 @@ package main
import (
"fmt"
"regexp"
"strings"
)
@@ -25,6 +26,17 @@ func splitTest() {
func main() {
splitTest()
inputList := []string{
"4.1.6-xxx",
"5.1.0",
"3.2.0-0123-123",
}
r, _ := regexp.Compile(`\d+.+\d+.+\d+`)
for _, input := range inputList {
matches := r.FindAllString(input, -1)
for _, match := range matches {
fmt.Println(match) // Output: 4.1.6 5.1.0 3.2.0
}
}
}

View File

@@ -8,7 +8,7 @@ import (
"wdd.io/agent-common/utils"
)
var CmiiOperator = CmiiK8sOperator{}
var DefaultCmiiOperator = CmiiK8sOperator{}
// var updateLogPath = "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\cmii-update-log.txt"
var updateLogPath = "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log/cmii-update-log.txt"
@@ -19,7 +19,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in
//podInterface := CmiiPodInterface{}
// get all pods
podAll := CmiiOperator.PodAllInterface(cmiiEnv)
podAll := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
// restart map
restartMap := make(map[string]int32, len(podAll))
@@ -47,7 +47,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in
// find deployment convert to interface
for key, value := range restartMap {
// container Name must equals deployment name
deployment := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deployment := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deployment != nil {
// deployment exists
log.DebugF("[FindAppNotHealthyOrRestartCountGreaterThanN] - restart [%s] [%s] is [%d]", cmiiEnv, key, value)
@@ -63,7 +63,7 @@ func FindAppNotHealthyOrRestartCountGreaterThanN(cmiiEnv string, restartCount in
func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (deploymentList []CmiiDeploymentInterface) {
// get all deployments
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces = FilterAllCmiiAppSoft(cmiiDeploymentInterfaces)
// filter
@@ -80,13 +80,13 @@ func FindDeploymentReplicasSmallerThanN(cmiiEnv string, replicasMin int32) (depl
func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
if !podInterface.PodStatus {
// unhealthy pod
deploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
deploymentInterface := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, podInterface.ContainerName)
if deploymentInterface != nil {
deploymentList = append(deploymentList, *deploymentInterface)
}
@@ -98,10 +98,10 @@ func FindDeploymentNotHealthy(cmiiEnv string) (deploymentList []CmiiDeploymentIn
func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
// dev-cluster
devNodeList := CmiiOperator.NodeAllInterface("dev")
devNodeList := DefaultCmiiOperator.NodeAllInterface("dev")
// core-cluster
coreNodeList := CmiiOperator.NodeAllInterface("uat")
coreNodeList := DefaultCmiiOperator.NodeAllInterface("uat")
// append
coreNodeList = append(coreNodeList, devNodeList...)
@@ -129,7 +129,7 @@ func FindAllNodeNotHealthy() (nodeList []CmiiNodeInterface) {
func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
// all unhealthy pods
allInterface := CmiiOperator.PodAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
// find the deployments
for _, podInterface := range allInterface {
@@ -145,7 +145,7 @@ func FindPodNotHealthy(cmiiEnv string) (podList []CmiiPodInterface) {
func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCommit string) {
// get app
podList := CmiiOperator.PodByAppName(cmiiEnv, appName)
podList := DefaultCmiiOperator.PodByAppName(cmiiEnv, appName)
// get pod
if podList == nil || len(podList) == 0 {
@@ -154,7 +154,7 @@ func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCo
}
// exec env
stdout, stderr := CmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
stdout, stderr := DefaultCmiiOperator.PodExec(cmiiEnv, podList[0], []string{"env"})
errLog := stderr.String()
if errLog != "" {
@@ -182,7 +182,7 @@ func GetDeploymentGitInfoFromInnerEnv(cmiiEnv, appName string) (gitBranch, gitCo
func FindCmiiMiddlewarePodInterface(cmiiEnv string) (podList []CmiiPodInterface) {
cmiiPodInterfaces := CmiiOperator.PodAllInterface(cmiiEnv)
cmiiPodInterfaces := DefaultCmiiOperator.PodAllInterface(cmiiEnv)
for _, podInterface := range cmiiPodInterfaces {
for key := range CmiiMiddlewareNameMap {
@@ -219,9 +219,9 @@ func ScaleDeploymentToDesiredReplicasFromMap(cmiiEnv string, nameReplicasMap map
// 遍历
for appName, replica := range nameReplicasMap {
exists := CmiiOperator.DeploymentExist(cmiiEnv, appName)
exists := DefaultCmiiOperator.DeploymentExist(cmiiEnv, appName)
if exists != nil {
scale := CmiiOperator.DeploymentScale(cmiiEnv, appName, replica)
scale := DefaultCmiiOperator.DeploymentScale(cmiiEnv, appName, replica)
if !scale {
errorUpdateMap[appName] = replica
}
@@ -238,12 +238,12 @@ func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
result := true
for _, deployment := range deploymentList {
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
result = DefaultCmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, 0)
if !result {
return result
}
time.Sleep(time.Second)
result = CmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
result = DefaultCmiiOperator.DeploymentScale(deployment.Namespace, deployment.Name, deployment.Replicas)
if !result {
return result
}
@@ -254,11 +254,11 @@ func RestartDeploymentFromList(deploymentList []CmiiDeploymentInterface) bool {
func RestartCmiiBackendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
if AppNameBelongsToCmiiImage(deploymentInterface.Name) {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
if !DefaultCmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiBackendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
@@ -267,17 +267,17 @@ func RestartCmiiBackendDeployment(cmiiEnv string) {
}
}
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
log.InfoF("[RestartCmiiBackendDeployment] - restart of all backend app in [%s] success !", DefaultCmiiOperator.CurrentNamespace)
}
func RestartCmiiFrontendDeployment(cmiiEnv string) {
cmiiDeploymentInterfaces := CmiiOperator.DeploymentAllInterface(cmiiEnv)
cmiiDeploymentInterfaces := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
for _, deploymentInterface := range cmiiDeploymentInterfaces {
_, ok := CmiiFrontendAppMap[deploymentInterface.Name]
if ok {
if !CmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
if !DefaultCmiiOperator.DeploymentRestart(deploymentInterface.Namespace, deploymentInterface.Name) {
log.ErrorF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] failed !", deploymentInterface.Namespace, deploymentInterface.Name)
} else {
log.DebugF("[RestartCmiiFrontendDeployment] - restart of [%s] [%s] success !", deploymentInterface.Namespace, deploymentInterface.Name)
@@ -285,38 +285,39 @@ func RestartCmiiFrontendDeployment(cmiiEnv string) {
}
}
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", CmiiOperator.CurrentNamespace)
log.InfoF("[RestartCmiiFrontendDeployment] - restart of all backend app in [%s] success !", DefaultCmiiOperator.CurrentNamespace)
}
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) (updateOK bool, oldImageTag, newImageTag string) {
cmiiDeploymentInterface := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
cmiiDeploymentInterface := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if cmiiDeploymentInterface == nil {
return false
return updateOK, oldImageTag, newImageTag
}
// check if need to update
if cmiiDeploymentInterface.ImageTag == newTag {
oldImageTag = cmiiDeploymentInterface.ImageTag
if oldImageTag == newTag {
log.DebugF("[UpdateCmiiDeploymentImageTag] - [%s] [%s] image tag are the same ! no need to update !", cmiiEnv, appName)
// restart
if CmiiOperator.DeploymentRestart(cmiiEnv, appName) {
return true
if DefaultCmiiOperator.DeploymentRestart(cmiiEnv, appName) {
return true, oldImageTag, oldImageTag
} else {
return false
return false, oldImageTag, oldImageTag
}
}
content := utils.WordSpaceCompletion(utils.TimeSplitFormatString()+" "+cmiiDeploymentInterface.Namespace, 35)
content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.Name, 75)
content = utils.WordSpaceCompletion(content+cmiiDeploymentInterface.ImageTag, 105)
content = utils.WordSpaceCompletion(content+oldImageTag, 105)
content = content + newTag + "\n"
log.DebugF("[UpdateCmiiDeploymentImageTag] - prepare to update [%s]!", content)
// update
tag := CmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
tag := DefaultCmiiOperator.DeploymentUpdateTag(cmiiDeploymentInterface.Namespace, cmiiDeploymentInterface.Name, newTag)
if !tag {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - [%s] update failed !", content)
return false
return false, oldImageTag, newImageTag
}
// append log
@@ -324,15 +325,13 @@ func UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag string) bool {
// re-get from env
time.Sleep(time.Second)
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
if deploy == nil {
log.ErrorF("[UpdateCmiiDeploymentImageTag] - unknown error happened ! [%s] [%s] not exists !", cmiiEnv, appName)
return false
return false, oldImageTag, newImageTag
}
// log
//log.InfoF("[UpdateCmiiDeploymentImageTag] - real image tag are [%s] update tag [%s] success ! ", deploy.Image, content)
return true
return true, oldImageTag, deploy.ImageTag
}
func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]string) (result map[string]string) {
@@ -340,12 +339,12 @@ func UpdateCmiiImageTagFromNameTagMap(cmiiEnv string, nameTagMap map[string]stri
result = make(map[string]string, len(nameTagMap))
for appName, newTag := range nameTagMap {
if AppNameBelongsToCmiiImage(appName) {
if UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag) {
ok, oldImageTag, newImageTag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
if ok {
log.InfoF("[UpdateCmiiImageTagFromNameTagMap] - %s %s to %s", cmiiEnv, appName, newTag)
result[appName] = newTag
result[appName] = newImageTag
} else {
result[appName] = "false"
result[appName] = oldImageTag
}
}
}
@@ -396,23 +395,23 @@ func RollBackCmiiDeploymentFromUpdateLog(updateLog string) bool {
}
log.InfoF("[RollBackCmiiDeploymentFromUpdateLog] - rollback [%s] [%s] from [%s] to [%s]", cmiiEnv, appName, newTag, fromTag)
rollback := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
ok, _, _ := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, fromTag)
return rollback
return ok
}
// BackupAllDeploymentFromEnv 从DEMO提取全部的CMII的应用
func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
// must filter
allInterface = FilterAllCmiiAppSoft(allInterface)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\all-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/all-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", CmiiOperator.CurrentNamespace, filePath)
log.InfoF("[BackupAllDeploymentFromEnv] - backup all image from %s => %s", DefaultCmiiOperator.CurrentNamespace, filePath)
firstCol := 0
secondCol := 0
@@ -462,7 +461,7 @@ func BackupAllDeploymentFromEnv(cmiiEnv string) bool {
// BackupAllCmiiDeploymentToMap 备份DEMO环境全部的镜像名称为Map 包括SRS 前端 后端
func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsMap map[string]string) {
allInterface := CmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface := DefaultCmiiOperator.DeploymentAllInterface(cmiiEnv)
allInterface = FilterAllCmiiAppSoft(allInterface)
backendMap = make(map[string]string, len(allInterface))
@@ -481,7 +480,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM
for key, value := range CmiiSrsAppMap {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
split := strings.Split(imageName, ":")
@@ -492,7 +491,7 @@ func BackupAllCmiiDeploymentToMap(cmiiEnv string) (backendMap, frontendMap, srsM
}
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
split := strings.Split(imageName, ":")
@@ -531,9 +530,9 @@ func BackupAllCmiiDeploymentToList(cmiiEnv string, completePrefix bool) (allCmii
func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
CmiiOperator.changeOperatorEnv(cmiiEnv)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + CmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
DefaultCmiiOperator.changeOperatorEnv(cmiiEnv)
//filePath := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\log\\images-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
filePath := "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/log/images-" + DefaultCmiiOperator.CurrentNamespace + "-" + utils.TimeSplitFormatString() + ".txt"
only := make(map[string]string, 150)
// front
@@ -541,7 +540,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
for key, value := range CmiiFrontendAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
@@ -552,7 +551,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
for key, value := range CmiiBackendAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
@@ -565,7 +564,7 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
for key, value := range CmiiGISAppMap {
_, ok := only[key]
if !ok {
deploy := CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if deploy != nil {
only[key] = value
utils.AppendContentToFile(deploy.Image+"\n", filePath)
@@ -579,13 +578,13 @@ func BackUpAllCmiiAppImageNameFromEnv(cmiiEnv string) {
if !ok {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, key)
if app != nil {
only[key] = value
utils.AppendContentToFile(app.Image+"\n", filePath)
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
app = DefaultCmiiOperator.StatefulSetOneInterface(cmiiEnv, key)
if app != nil {
only[key] = value
for _, imageName := range app.ContainerImageMap {

View File

@@ -2,6 +2,7 @@ package main
import (
"fmt"
"strconv"
"testing"
"time"
"wdd.io/agent-common/assert"
@@ -110,7 +111,7 @@ func TestFindPodNotHealthy_And_Delete(t *testing.T) {
for _, podInterface := range podNotHealthy {
t.Logf("[%s] [%s]", podInterface.Name, podInterface.PodPhase)
podDelete := CmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name)
podDelete := DefaultCmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name)
assert.Equal(t, podDelete, true, "delete of ", podInterface.Namespace, podInterface.Name, " failed !")
}
@@ -118,12 +119,12 @@ func TestFindPodNotHealthy_And_Delete(t *testing.T) {
}
func TestFilterAllCmiiAppStrict(t *testing.T) {
allInterface := CmiiOperator.DeploymentAllInterface("devflight")
allInterface := DefaultCmiiOperator.DeploymentAllInterface("devflight")
FilterAllCmiiAppStrict(allInterface)
}
func TestRestartDeploymentFromList(t *testing.T) {
allInterface := CmiiOperator.DeploymentAllInterface("devflight")
allInterface := DefaultCmiiOperator.DeploymentAllInterface("devflight")
allInterface = FilterAllCmiiAppSoft(allInterface)
RestartDeploymentFromList(allInterface)
@@ -240,10 +241,10 @@ func TestRestartCmiiDeployment(t *testing.T) {
cmiiEnv := integration
appName := "cmii-uav-platform"
kill := CmiiOperator.DeploymentRestartByKill(cmiiEnv, appName)
kill := DefaultCmiiOperator.DeploymentRestartByKill(cmiiEnv, appName)
assert.Equal(t, kill, true, "have unhealthy pod !")
check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
assert.Equal(t, check, true, "DeploymentStatusCheck failed !")
}
@@ -278,20 +279,22 @@ func TestUpdateCmiiDeploymentImageTag(t *testing.T) {
}
for appName, newTag := range appNameTagMap {
tag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
assert.Equal(t, tag, true, "update image tag failed !")
ok, oldImageTag, newImageTag := UpdateCmiiDeploymentImageTag(cmiiEnv, appName, newTag)
assert.Equal(t, ok, true, "update image tag failed !")
utils.SplitLinePrint()
check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 300)
check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 300)
assert.Equal(t, check, true, "deployment run failed!")
deploy := DefaultCmiiOperator.DeploymentOneInterface(cmiiEnv, appName)
// push message
message := pusher.CmiiUpdateMessage{
Namespace: cmiiEnv,
AppName: appName,
FromTag: "",
ToTag: newTag,
Replicas: "",
FromTag: oldImageTag,
ToTag: newImageTag,
Replicas: strconv.FormatInt(int64(deploy.Replicas), 10),
DeployStatus: check,
}
message.SendMessage()

View File

@@ -1,12 +1,8 @@
package main
import (
"bufio"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"slices"
"strings"
@@ -59,13 +55,14 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
// compress
if sync.ProjectVersion != "" {
// get version images
// 获取特定版本的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromVersion(sync.ProjectVersion, sync.CompressImageToGzip, sync.UploadToDemoMinio)
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectVersion
} else {
// get demo images
// 获取DEMO的镜像
errorPullImageList, errorGzipImageList, allCmiiImageNameList, allGzipFileNameList = DownloadCompressUploadFromDemo(sync.ProjectName, sync.CompressImageToGzip, sync.UploadToDemoMinio)
gzipFolderFullPath = image.OfflineImageGzipFolderPrefix + sync.ProjectName
@@ -147,9 +144,13 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
log.Info("COMPRESS START")
for _, imageFullName := range fullNameList {
if !image.SaveToTarGZ(imageFullName, gzipFolderFullPath) {
ok, gzipImageFileFullPath := image.SaveToGzipFile(imageFullName, gzipFolderFullPath)
if !ok {
errorGzipImageList = append(errorGzipImageList, imageFullName)
continue
}
// 压缩成功
allGzipFileNameList = append(allGzipFileNameList, gzipImageFileFullPath)
}
// remove failed
fullNameList = slices.DeleteFunc(fullNameList, func(imageName string) bool {
@@ -162,28 +163,17 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
//uploadGzipFileToDemoMinio()
// get gzip file name list
log.Info("UPLOAD OSS START !")
err := filepath.WalkDir(gzipFolderFullPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
log.ErrorF("error getting gzip file name list 1! %s", err.Error())
}
if !d.IsDir() {
allGzipFileNameList = append(allGzipFileNameList, d.Name())
}
return nil
})
if err != nil {
log.ErrorF("error getting gzip file name list 2! %s", err.Error())
}
// start to upload
// extract demo oss location suffix from gzipFolderFullPath
trimPrefix := strings.TrimPrefix(gzipFolderFullPath, image.OfflineImageGzipFolderPrefix)
bucketName := "cmlc-installation/" + trimPrefix
log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketName)
bucketNameWithPrefix := "cmlc-installation/" + trimPrefix
log.InfoF("gzip file location in demo oss is %s", DefaultDemoEndpoint+"/"+bucketNameWithPrefix)
for _, gzipFileName := range allGzipFileNameList {
if !DefaultCmiiMinioOperator.UploadToDemo(bucketName, gzipFolderFullPath, gzipFileName) {
// SaveToGzipFile 返回的是全路径 归一化处理 gzip file name
gzipFileName = strings.TrimPrefix(gzipFileName, gzipFolderFullPath)
if !DefaultCmiiMinioOperator.UploadToDemo(bucketNameWithPrefix, gzipFolderFullPath, gzipFileName) {
log.ErrorF("upload of %s to demo oss error !", gzipFolderFullPath+gzipFileName)
}
}
@@ -193,59 +183,75 @@ func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFu
return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileNameList
}
// DownloadLoadTagPush DLTU procedure ImageSync的另外一般流程需要支持 堡垒机(纯离线)的模式
// DownloadLoadTagUpload DLTU procedure ImageSync的另外一般流程需要支持 堡垒机(纯离线)的模式
// 2. Gzip文件目录RKE MIDDLE CMII三个文件目录 - 约定目录
// 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/
// 3. 读取本机的IP地址 - 参数传递
// 4. OSS地址 - ossUrlPrefix传空 则使用默认值
// 5. ossFileName - 如果结尾为txt则为文件的形式如果为tar.gz则为gzip文件夹的形式
func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolder string, targetHarborFullName string) []string {
func DownloadLoadTagUpload(downloadFromOss bool, ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile string, targetHarborFullName string) (targetImageFullNameList []string) {
// 支持单文件的形式
if !utils.IsFileOrDir(localGzipFolderOrGzipFile) {
// 单个压缩文件
if !strings.HasSuffix(localGzipFolderOrGzipFile, ".tar.gz") {
log.ErrorF("local gzip file %s is not a .tar.gz file !", localGzipFolderOrGzipFile)
return nil
}
// load
image.LoadFromGzipFilePath(localGzipFolderOrGzipFile)
}
separator := os.PathSeparator
if !strings.HasSuffix(localGzipFolder, string(separator)) {
localGzipFolder += string(separator)
if !strings.HasSuffix(localGzipFolderOrGzipFile, string(separator)) {
localGzipFolderOrGzipFile += string(separator)
}
// download
if downloadFromOss {
if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolder) {
if !parseAndDownloadFromOss(ossUrlPrefix, ossFileName, localGzipFolderOrGzipFile) {
log.ErrorF("download from oss error !")
return nil
}
}
// load
loadAllGzipImageFromLocalFolder(localGzipFolder)
loadAllGzipImageFromLocalFolder(localGzipFolderOrGzipFile)
// tag
// push
allFileInFolder, err := utils.ListAllFileInFolder(localGzipFolder)
allFileInFolder, err := utils.ListAllFileInFolder(localGzipFolderOrGzipFile)
if err != nil {
return nil
}
for _, gzipFileName := range allFileInFolder {
// 过滤非.tar.gz结尾的文件
if !strings.HasSuffix(gzipFileName, ".tar.gz") {
continue
}
log.DebugF("gzip file name is %s", gzipFileName)
// gzip to image full name 拿到镜像的原始名称
imageFullName := image2.GzipFileNameToImageFullName(gzipFileName)
if imageFullName == "" {
log.ErrorF("gzip file %s to image full name error !", gzipFileName)
continue
}
// tag 拿到目标名称 然后重新Tag
targetImageFullName := image2.ImageNameToTargetImageFullName(imageFullName, targetHarborFullName)
// tag
image.TagFromSourceToTarget(imageFullName, targetImageFullName)
//push
pushResult := image.PushToOctopusKindHarbor(targetImageFullName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
// uploadToHarbor 上传到目标Harbor
if image.UploadToHarbor(targetImageFullName) {
targetImageFullNameList = append(targetImageFullNameList, targetHarborFullName)
} else {
log.ErrorF("upload to harbor error of %s", targetImageFullName)
}
fmt.Println()
fmt.Printf("%s to %s push success !", gzipFileName, targetImageFullName)
fmt.Println()
}
return nil
return targetImageFullNameList
}
func loadAllGzipImageFromLocalFolder(localGzipFolder string) {
@@ -381,12 +387,12 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
for key, value := range CmiiSrsAppMap {
var app *CmiiDeploymentInterface
if strings.Contains(value, "deployment") {
app = CmiiOperator.DeploymentOneInterface(demo, key)
app = DefaultCmiiOperator.DeploymentOneInterface(demo, key)
if app != nil {
realCmiiImageName = append(realCmiiImageName, app.Image)
}
} else if strings.Contains(value, "state") {
app = CmiiOperator.StatefulSetOneInterface(demo, key)
app = DefaultCmiiOperator.StatefulSetOneInterface(demo, key)
if app != nil {
for _, imageName := range app.ContainerImageMap {
realCmiiImageName = append(realCmiiImageName, imageName)

View File

@@ -386,6 +386,25 @@ func (op *CmiiK8sOperator) DeploymentScale(cmiiEnv, appName string, scaleCount i
return true
}
func (op *CmiiK8sOperator) DeploymentUpdateTagByImageFullName(cmiiEnv, imageFullName string) bool {
split := strings.Split(imageFullName, ":")
// harbor
// 192.168.6.6:8033/rancher/k8s-dns-sidecar:v1.0.2
newTag := split[1]
appName := strings.Split(split[0], "/")[len(strings.Split(split[0], "/"))-1]
if strings.Contains(imageFullName, "8033") {
newTag = split[2]
appName = strings.Split(split[1], "/")[len(strings.Split(split[1], "/"))-1]
}
// 拿到AppName
log.DebugF("DeploymentUpdateTagByImageFullName - appName => %s, newTag => %s", appName, newTag)
return op.DeploymentUpdateTag(cmiiEnv, appName, newTag)
}
func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string) bool {
@@ -400,37 +419,55 @@ func (op *CmiiK8sOperator) DeploymentUpdateTag(cmiiEnv, appName, newTag string)
}
containers := deployment.Spec.Template.Spec.Containers
if len(containers) == 1 {
// only update this kind
container := containers[0]
oldName := container.Image
split := strings.Split(container.Image, ":")
if strings.HasPrefix(container.Image, image2.CmiiHarborPrefix) {
// harbor
container.Image = split[0] + ":" + newTag
} else if strings.Contains(container.Image, "8033") {
// 192.168.6.6:8033/rancher/k8s-dns-sidecar:v1.0.2
container.Image = split[0] + ":" + split[1] + ":" + newTag
}
log.DebugF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s]", op.CurrentNamespace, appName, oldName, container.Image)
// re assign
deployment.Spec.Template.Spec.Containers[0] = container
// update
_, err := op.CurrentClient.AppsV1().Deployments(deployment.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
if err != nil {
log.ErrorF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s] error ! %s", op.CurrentNamespace, appName, split[1], container.Image, err.Error())
return false
}
} else if len(containers) == 2 {
if len(containers) >= 2 {
log.ErrorF("[DeploymentUpdateTag] - cant update app with 2 containers !")
return false
}
// 只支持container的数量为1的形式
container := containers[0]
oldName := container.Image
split := strings.Split(container.Image, ":")
if strings.HasPrefix(container.Image, image2.CmiiHarborPrefix) {
// harbor
container.Image = split[0] + ":" + newTag
} else if strings.Contains(container.Image, "8033") {
// 192.168.6.6:8033/rancher/k8s-dns-sidecar:v1.0.2
// 重新拼接
container.Image = split[0] + ":" + split[1] + ":" + newTag
}
log.DebugF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s]", op.CurrentNamespace, appName, oldName, container.Image)
// 更新Cmii BIZ_GROUP
tagVersion := newTag
if strings.Contains(newTag, "-") {
tagVersion = strings.Split(newTag, "-")[0]
}
envList := container.Env
for _, envVar := range envList {
if envVar.Name == "IMAGE_VERSION" {
envVar.Value = tagVersion
}
if envVar.Name == "BIZ_CONFIG_GROUP" {
envVar.Value = tagVersion
}
if envVar.Name == "SYS_CONFIG_GROUP" {
envVar.Value = tagVersion
}
}
log.DebugF("[DeploymentUpdateTag] - update env IMAGE_VERSION to [%s]", tagVersion)
// 赋值回去 很关键
deployment.Spec.Template.Spec.Containers[0] = container
// update
_, err := op.CurrentClient.AppsV1().Deployments(deployment.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
if err != nil {
log.ErrorF("[DeploymentUpdateTag] - update [%s] [%s] from [%s] to [%s] error ! %s", op.CurrentNamespace, appName, split[1], container.Image, err.Error())
return false
}
return true
}

View File

@@ -12,7 +12,7 @@ import (
func TestCmiiK8sOperator_DeploymentAll(t *testing.T) {
start := time.Now()
deploymentList := CmiiOperator.DeploymentAll("devflight")
deploymentList := DefaultCmiiOperator.DeploymentAll("devflight")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -39,7 +39,7 @@ func TestCmiiK8sOperator_DeploymentAll(t *testing.T) {
func TestCmiiK8sOperator_DeploymentAllInterface(t *testing.T) {
start := time.Now()
deploymentList := CmiiOperator.DeploymentAllInterface("devflight")
deploymentList := DefaultCmiiOperator.DeploymentAllInterface("devflight")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -54,7 +54,7 @@ func TestCmiiK8sOperator_DeploymentAllInterface(t *testing.T) {
func TestCmiiK8sOperator_DeploymentFizz(t *testing.T) {
start := time.Now()
deploymentFizz := CmiiOperator.DeploymentFizz("int", "cmii-suav-supervision")
deploymentFizz := DefaultCmiiOperator.DeploymentFizz("int", "cmii-suav-supervision")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -75,7 +75,7 @@ func TestCmiiK8sOperator_DeploymentFizz(t *testing.T) {
func TestCmiiK8sOperator_DeploymentScale(t *testing.T) {
start := time.Now()
CmiiOperator.DeploymentScale(demo, "cmii-uav-industrial-portfolio", 1)
DefaultCmiiOperator.DeploymentScale(demo, "cmii-uav-industrial-portfolio", 1)
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -84,7 +84,7 @@ func TestCmiiK8sOperator_DeploymentScale(t *testing.T) {
func TestCmiiK8sOperator_DeploymentUpdateTag(t *testing.T) {
start := time.Now()
CmiiOperator.DeploymentUpdateTag("demo", "cmii-uav-platform", "5.2.0-011001")
DefaultCmiiOperator.DeploymentUpdateTag("demo", "cmii-uav-platform", "5.2.0-011001")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
}
@@ -94,11 +94,11 @@ func TestCmiiK8sOperator_DeploymentRestart(t *testing.T) {
cmiiEnv := integration
appName := "cmii-uav-data-post-process"
CmiiOperator.DeploymentRestart(cmiiEnv, appName)
DefaultCmiiOperator.DeploymentRestart(cmiiEnv, appName)
utils.SplitLinePrint()
check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
assert.Equal(t, check, true, "deployment run failed!")
}
@@ -106,19 +106,19 @@ func TestCmiiK8sOperator_DeploymentRestartByKill(t *testing.T) {
cmiiEnv := "demo"
appName := "cmii-uav-platform"
kill := CmiiOperator.DeploymentRestartByKill(cmiiEnv, appName)
kill := DefaultCmiiOperator.DeploymentRestartByKill(cmiiEnv, appName)
assert.Equal(t, kill, true, "deployment restart by kill failed !")
utils.SplitLinePrint()
check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
assert.Equal(t, check, true, "deployment run failed!")
}
func TestCmiiK8sOperator_DeploymentOneInterface(t *testing.T) {
start := time.Now()
deploy := CmiiOperator.DeploymentOneInterface("devflight", "cmii-uav-depotautoreturn")
deploy := DefaultCmiiOperator.DeploymentOneInterface("devflight", "cmii-uav-depotautoreturn")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -130,7 +130,7 @@ func TestCmiiK8sOperator_ReplicaSetExists(t *testing.T) {
cmiiEnv := "uavcloud-devflight"
appName := "cmii-admin-data-bf8f87cb7"
exists := CmiiOperator.ReplicaSetExists(cmiiEnv, appName)
exists := DefaultCmiiOperator.ReplicaSetExists(cmiiEnv, appName)
utils.BeautifulPrint(*exists)
}
@@ -140,7 +140,7 @@ func TestCmiiK8sOperator_ReplicaSetByAppName(t *testing.T) {
cmiiEnv := "uavcloud-devflight"
appName := "cmii-admin-data"
exists := CmiiOperator.ReplicaSetByAppName(cmiiEnv, appName)
exists := DefaultCmiiOperator.ReplicaSetByAppName(cmiiEnv, appName)
for _, replicaSet := range exists {
utils.BeautifulPrint(replicaSet)
@@ -149,7 +149,7 @@ func TestCmiiK8sOperator_ReplicaSetByAppName(t *testing.T) {
func TestCmiiK8sOperator_PodAll(t *testing.T) {
start := time.Now()
podList := CmiiOperator.PodAll("devflight")
podList := DefaultCmiiOperator.PodAll("devflight")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -163,7 +163,7 @@ func TestCmiiK8sOperator_PodAll(t *testing.T) {
func TestCmiiK8sOperator_PodFizz(t *testing.T) {
start := time.Now()
podList := CmiiOperator.PodFizz("devflight", "cmii-uav-data-post-process")
podList := DefaultCmiiOperator.PodFizz("devflight", "cmii-uav-data-post-process")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
t.Logf("pod list lenght is => %d", len(podList))
@@ -183,7 +183,7 @@ func TestCmiiK8sOperator_PodByAppName(t *testing.T) {
cmiiEnv := "uat"
appName := "cmii-admin-data"
exists := CmiiOperator.PodByNodeName(cmiiEnv, appName)
exists := DefaultCmiiOperator.PodByNodeName(cmiiEnv, appName)
for _, podInterface := range exists {
utils.BeautifulPrint(podInterface)
@@ -193,7 +193,7 @@ func TestCmiiK8sOperator_PodByAppName(t *testing.T) {
func TestCmiiK8sOperator_PodFizz2(t *testing.T) {
start := time.Now()
podList := CmiiOperator.PodFizz("devflight", "notice")
podList := DefaultCmiiOperator.PodFizz("devflight", "notice")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
t.Logf("pod list lenght is => %d", len(podList))
@@ -215,13 +215,13 @@ func TestCmiiK8sOperator_PodByNodeName(t *testing.T) {
cmiiEnv := "uat"
nodeName := "test-03.ecs.io"
exists := CmiiOperator.PodByNodeName(cmiiEnv, nodeName)
exists := DefaultCmiiOperator.PodByNodeName(cmiiEnv, nodeName)
exists = FilterAllCmiiPodSoft(exists)
for _, podInterface := range exists {
utils.BeautifulPrint(podInterface)
if !podInterface.PodStatus {
podDelete := CmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name)
podDelete := DefaultCmiiOperator.PodDelete(podInterface.Namespace, podInterface.Name)
assert.Equal(t, podDelete, true, "delete pod failed !")
}
}
@@ -229,9 +229,9 @@ func TestCmiiK8sOperator_PodByNodeName(t *testing.T) {
func TestCmiiK8sOperator_PodExec(t *testing.T) {
podList := CmiiOperator.PodByAppName(devFlight, "cmii-uav-gateway")
podList := DefaultCmiiOperator.PodByAppName(devFlight, "cmii-uav-gateway")
stdout, stderr := CmiiOperator.PodExec(devFlight, podList[0], []string{
stdout, stderr := DefaultCmiiOperator.PodExec(devFlight, podList[0], []string{
"env",
})
@@ -262,14 +262,14 @@ func TestCmiiK8sOperator_DeploymentStatusCheck(t *testing.T) {
cmiiEnv := "devflight"
appName := "cmii-uav-gateway"
check := CmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
check := DefaultCmiiOperator.DeploymentStatusCheck(cmiiEnv, appName, 180)
assert.Equal(t, check, true, "deployment run failed!")
}
func TestCmiiK8sOperator_NodeAll(t *testing.T) {
start := time.Now()
nodeList := CmiiOperator.NodeAll("dev")
nodeList := DefaultCmiiOperator.NodeAll("dev")
elapsed := time.Since(start).Milliseconds()
fmt.Printf("执行耗时: %d ms\n", elapsed)
@@ -282,7 +282,7 @@ func TestCmiiK8sOperator_NodeAll(t *testing.T) {
}
func TestCmiiK8sOperator_NodeAllInterface(t *testing.T) {
cmiiNodeInterfaces := CmiiOperator.NodeAllInterface("uat")
cmiiNodeInterfaces := DefaultCmiiOperator.NodeAllInterface("uat")
for _, nodeInterface := range cmiiNodeInterfaces {
println()

View File

@@ -181,7 +181,7 @@ func TagFromSourceToTarget(sourceImageName, targetImageName string) bool {
return true
}
func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) {
func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) {
if GetByName(targetImageName) == nil {
log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName)
@@ -202,6 +202,17 @@ func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser)
return pushResult
}
func UploadToHarbor(targetImageName string) (uploadOK bool) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
fmt.Println()
log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
fmt.Println()
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
@@ -239,7 +250,7 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
fmt.Println(targetImageName)
//if TagFromSourceToTarget(cmiiImageFullName, targetImageName) {
// pushResult := PushToOctopusKindHarbor(targetImageName)
// pushResult := UploadToOctopusKindHarbor(targetImageName)
// if pushResult == nil {
// errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
// log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName)
@@ -360,11 +371,13 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
for _, image := range fullImageNameList {
if !SaveToTarGZ(image, gzipFolder) {
ok, path := SaveToGzipFile(image, gzipFolder)
if !ok {
errorGzipImageList = append(errorGzipImageList, image)
continue
}
tarGzipFileNameList = append(tarGzipFileNameList, image2.ImageFullNameToGzipFileName(image))
tarGzipFileNameList = append(tarGzipFileNameList, path)
}
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
@@ -372,20 +385,20 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
}
// LoadFromGzipFilePath 根据Gzip文件的全名称进行Load操作
func LoadFromGzipFilePath(gzipFullPath string) bool {
openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
func LoadFromGzipFilePath(gzipFileNameFullPath string) bool {
openFile, err := os.OpenFile(gzipFileNameFullPath, 0, fs.ModePerm)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error())
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFileNameFullPath, err.Error())
return false
}
loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFullPath, err.Error())
log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFileNameFullPath, err.Error())
return false
}
log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFullPath, strconv.FormatBool(loadResponse.JSON))
log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFileNameFullPath, strconv.FormatBool(loadResponse.JSON))
scanner := bufio.NewScanner(loadResponse.Body)
for scanner.Scan() {
@@ -420,18 +433,19 @@ func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
return errorLoadImageNameList
}
func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
// SaveToGzipFile 根据目标镜像的全名称 将镜像压缩为Gzip文件
func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipImageFileFullPath string) {
imageGetByName := GetByName(targetImageName)
imageGetByName := GetByName(imageFullName)
if imageGetByName == nil {
log.ErrorF("[ImageSaveToTarGZ] - %s not exits", targetImageName)
return false
log.ErrorF("[ImageSaveToTarGZ] - %s not exits", imageFullName)
return false, ""
}
imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error())
return false
return false, ""
}
var realImageTag string
@@ -442,19 +456,25 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
}
}
gzipImageFile := image2.ImageFullNameToGzipFileName(realImageTag)
gzipImageFileFullPath = image2.ImageFullNameToGzipFileName(realImageTag)
if !strings.HasSuffix(folderPathPrefix, "/") {
folderPathPrefix += "/"
}
_ = os.MkdirAll(folderPathPrefix, os.ModeDir)
gzipImageFile = folderPathPrefix + gzipImageFile
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFile)
_ = os.Remove(gzipImageFile)
tarFile, err := os.Create(gzipImageFile)
// 生成gzip压缩文件的全路径名称
gzipImageFileFullPath = folderPathPrefix + gzipImageFileFullPath
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFileFullPath)
// 删除掉旧的Gzip文件
_ = os.Remove(gzipImageFileFullPath)
// 创建
tarFile, err := os.Create(gzipImageFileFullPath)
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFile, err.Error())
return false
log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFileFullPath, err.Error())
return false, ""
}
defer tarFile.Close()
@@ -468,10 +488,11 @@ func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
// Copy the tar archive to the gzip writer.
if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error())
return false
return false, ""
}
return true
// 成功
return true, gzipImageFileFullPath
}
func CmiiImageMapToFullNameList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {

View File

@@ -88,7 +88,7 @@ func TestImagePushToOctopusKindHarbor(t *testing.T) {
assert.Equal(t, target, true, "image re-tag error !")
// push
pushResult := PushToOctopusKindHarbor(newTag)
pushResult := UploadToOctopusKindHarbor(newTag)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
@@ -109,7 +109,7 @@ func TestImageLoadFromFile(t *testing.T) {
func TestImageSaveToTarGZ(t *testing.T) {
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
imageSaveToTarGZ := SaveToTarGZ(image, "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log")
imageSaveToTarGZ, _ := SaveToGzipFile(image, "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log")
assert.Equal(t, imageSaveToTarGZ, true, "image save to tar gz file error !")
}
@@ -149,7 +149,7 @@ func TestImageTagFromSourceToTarget(t *testing.T) {
targetImageName := "harbor.wdd.io:8033/cmii/srs:v5.0.195"
if TagFromSourceToTarget(sourceImageName, targetImageName) {
pushResult := PushToOctopusKindHarbor(targetImageName)
pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
@@ -188,7 +188,7 @@ func TestSaveSpecificImageToGzipFile(t *testing.T) {
// image pull success
log.InfoF("image should have pulled successful ! => %s", imageFullName)
if !SaveToTarGZ(imageFullName, imageGzipFilePathPrefix) {
if !SaveToGzipFile(imageFullName, imageGzipFilePathPrefix) {
log.ErrorF("image save to gzip file error ! => %s", imageFullName)
return
}

View File

@@ -33,7 +33,7 @@ func RealProjectRunner() {
op := CmiiK8sOperator{}
op.BuildCurrentClientFromConfig(readFile)
CmiiOperator = op
DefaultCmiiOperator = op
// ops
@@ -77,7 +77,7 @@ func CmiiRunner() {
var DLTUHelp = `
DLTUHelp
dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName]
dltu [ossUrlPrefix] [ossFileName] [localGzipFolder] [harborHostFullName] [namespace]
`
func main() {
@@ -113,7 +113,7 @@ func main() {
result = append(result, text)
}
if len(result) != 5 {
if len(result) != 6 {
fmt.Println("input error!")
fmt.Printf(DLTUHelp)
return
@@ -123,11 +123,13 @@ func main() {
ossFileName := result[2]
localGzipFolder := result[3]
harborHostFullName := result[4]
namespace := result[5]
fmt.Println("ossUrlPrefix: ", ossUrlPrefix)
fmt.Println("ossFileName: ", ossFileName)
fmt.Println("localGzipFolder: ", localGzipFolder)
fmt.Println("harborHostFullName: ", harborHostFullName)
fmt.Println("namespace: ", namespace)
fmt.Println()
downloadFromOss := true
@@ -135,9 +137,17 @@ func main() {
downloadFromOss = false
}
DownloadLoadTagPush(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName)
// DLTU
targetImageFullNameList := DownloadLoadTagUpload(downloadFromOss, ossUrlPrefix, ossFileName, localGzipFolder, harborHostFullName)
// 下载
// 是否需要更新
if namespace != "" {
for _, targetImageFullName := range targetImageFullNameList {
if !DefaultCmiiOperator.DeploymentUpdateTagByImageFullName(namespace, targetImageFullName) {
fmt.Printf("[Update] update [%s] [%s] failed", namespace, targetImageFullName)
}
}
}
fmt.Println()
}

View File

@@ -38,13 +38,13 @@ public class TestImageSyncScheduler {
public void runImageSync() {
ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
// "cmii-uav-process:5.4.0-041901"
// "cmii-uav-process:5.4.0-041901"1
));
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
// "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
// "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195"
"harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0-cqly-042601"
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.3.0-cqly-042601"
));
Boolean downloadAndCompressOnly = false;