zeaslity/WddSuperAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初始化项目

Browse Source
This commit is contained in:
zeaslity
2025-03-27 16:09:20 +08:00
parent e09a32d1e8
commit fc2d585489
709 changed files with 516391 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
agent-operator/image/HarborOperator.go Normal file
View File

@@ -0,0 +1,165 @@
package image
import (
"context"
"fmt"
"github.com/mittwald/goharbor-client/v5/apiv2"
"github.com/mittwald/goharbor-client/v5/apiv2/model"
"strings"
)
type HarborOperator struct {
HarborHost string
HarborPort string
HarborUser string
HarborPass string
HarborClient *apiv2.RESTClient
}
var LocalHarborOperator = newDefaultInstance
func newDefaultInstance() {
}
const (
DefaultHarborPort = "8033"
DefaultHarborUser = "admin"
DefaultHarborPass = "V2ryStr@ngPss"
)
func (op *HarborOperator) BuildOperator() (bool, error) {
if op.HarborPort != "" {
op.HarborHost = op.HarborHost + ":" + op.HarborPort
}
if !strings.HasSuffix(op.HarborHost, "/api/") {
op.HarborHost = op.HarborHost + "/api/"
}
log.InfoF("[Harbor Client Create] - start to create harbor client %s", op.HarborHost)
if op.HarborUser == "" {
op.HarborUser = DefaultHarborUser
}
if op.HarborPass == "" {
op.HarborPass = DefaultHarborPass
}
// check connection
client, err := apiv2.NewRESTClientForHost(op.HarborHost, op.HarborUser, op.HarborPass, nil)
if err != nil {
errorLog := fmt.Sprintf("Error creating REST client: %s\n", err.Error())
log.Error(errorLog)
return false, err
}
op.HarborClient = client
return true, nil
}
func (op *HarborOperator) RepoListAll(projectName string) []*model.Repository {
listRepositories, err := op.HarborClient.ListRepositories(context.TODO(), projectName)
if err != nil {
log.ErrorF("RepoListAll error is %s", err.Error())
return nil
}
return listRepositories
}
// RepoAllCmiiImage 返回没有前缀的内容 cmii-uav-user
func (op *HarborOperator) RepoAllCmiiImage() (repoNameList []string) {
listAll := op.RepoListAll("cmii")
for _, repository := range listAll {
prefix := strings.TrimPrefix(repository.Name, "cmii/")
if strings.HasPrefix(prefix, "cmii") {
repoNameList = append(repoNameList, prefix)
}
}
// 返回没有
return repoNameList
}
func (op *HarborOperator) ArtifactListAll(projectName, repoName string) []*model.Artifact {
listArtifacts, err := op.HarborClient.ListArtifacts(context.TODO(), projectName, repoName)
if err != nil {
log.ErrorF("ArtifactListAll error is %s", err.Error())
return nil
}
return listArtifacts
}
func (op *HarborOperator) ArtifactListOne(projectName, repoName, artifactReference string) *model.Artifact {
artifact, err := op.HarborClient.GetArtifact(context.TODO(), projectName, repoName, artifactReference)
if err != nil {
log.ErrorF("ArtifactListOne error is %s", err.Error())
return nil
}
return artifact
}
func (op *HarborOperator) ArtifactDeleteOne(projectName, repoName, artifactReference string) bool {
err := op.HarborClient.DeleteArtifact(context.TODO(), projectName, repoName, artifactReference)
if err != nil {
log.ErrorF("ArtifactDeleteOne error is %s", err.Error())
return false
}
log.InfoF("ArtifactDeleteOne of %s %s %s is success", projectName, repoName, artifactReference)
return true
}
func (op *HarborOperator) RepoListOne(projectName, repoName string) *model.Repository {
repository, err := op.HarborClient.GetRepository(context.TODO(), projectName, repoName)
if err != nil {
log.ErrorF("RepoListOne error is %s", err.Error())
return nil
}
return repository
}
// CmiiTagFilter 根据tag前缀 找到所有的cmii tag
func (op *HarborOperator) CmiiTagFilter(tagPrefix string) (allCmiiImageList []string) {
repoNameList := op.RepoAllCmiiImage() // all cmii repo name
for _, cmiiRepo := range repoNameList {
// cmii-uav-user
artifactList := op.ArtifactListAll("cmii", cmiiRepo)
for _, artifact := range artifactList {
// list all artifact of cmii-uav-user
for _, tag := range artifact.Tags {
if strings.HasPrefix(tag.Name, tagPrefix) {
// tag has prefix 5.4.0-123 5.4.0-123-123
allCmiiImageList = append(allCmiiImageList, cmiiRepo+":"+tag.Name)
}
}
}
}
return allCmiiImageList
}
func (op *HarborOperator) ArtifactDeleteFromNameTagList(projectName string, nameTagList []string) (errorDeleteList []string) {
for _, nameTag := range nameTagList {
split := strings.Split(nameTag, ":")
if len(split) != 2 {
log.ErrorF("ArtifactDeleteFromNameTagList error is %s", nameTag)
errorDeleteList = append(errorDeleteList, nameTag)
continue
}
repoName := split[0]
tagName := split[1]
if !op.ArtifactDeleteOne(projectName, repoName, tagName) {
log.ErrorF("ArtifactDeleteFromNameTagList error is %s %s %s", projectName, repoName, tagName)
errorDeleteList = append(errorDeleteList, nameTag)
}
}
return errorDeleteList
}

149
agent-operator/image/HarborOperator_test.go Normal file
View File

@@ -0,0 +1,149 @@
package image
import (
"fmt"
"strings"
"testing"
"wdd.io/agent-common/utils"
)
var DefaultHarborOperator *HarborOperator
func TestHarborOperator_BuildOperator_CMII(t *testing.T) {
harborOperator := &HarborOperator{
HarborHost: "http://harbor.cdcyy.com.cn",
HarborPort: "",
HarborUser: "rad02_drone",
HarborPass: "Drone@1234",
HarborClient: nil,
}
_, err := harborOperator.BuildOperator()
if err != nil {
t.Logf("error is %s", err.Error())
return
}
DefaultHarborOperator = harborOperator
}
func TestHarborOperator_BuildOperator(t *testing.T) {
harborOperator := &HarborOperator{
HarborHost: "http://harbor.wdd.io",
HarborPort: "8033",
HarborUser: "admin",
HarborPass: "V2ryStr@ngPss",
HarborClient: nil,
}
_, err := harborOperator.BuildOperator()
if err != nil {
t.Logf("error is %s", err.Error())
return
}
DefaultHarborOperator = harborOperator
}
func TestHarborOperator_RepoListAll(t *testing.T) {
TestHarborOperator_BuildOperator(t)
//repoListAll := DefaultHarborOperator.RepoListAll("ran")
repoListAll := DefaultHarborOperator.RepoListAll("cmii")
utils.BeautifulPrint(repoListAll)
}
func TestHarborOperator_RepoAllCmiiImage(t *testing.T) {
TestHarborOperator_BuildOperator(t)
repoListAll := DefaultHarborOperator.RepoAllCmiiImage()
utils.BeautifulPrint(repoListAll)
}
func TestHarborOperator_ArtifactListAll(t *testing.T) {
TestHarborOperator_BuildOperator(t)
artifactListAll := DefaultHarborOperator.ArtifactListAll("cmii", "cmii-uav-user")
for _, artifact := range artifactListAll {
//if len(artifact.Tags) == 0 {
// fmt.Println(artifact.Digest)
//}
for _, tag := range artifact.Tags {
fmt.Println(tag.Name)
}
}
//utils.BeautifulPrint(artifactListAll)
}
func TestHarborOperator_CmiiHarborCleanUp(t *testing.T) {
TestHarborOperator_BuildOperator_CMII(t)
//repoListAll := DefaultHarborOperator.RepoListAll("ran")
repoListAll := DefaultHarborOperator.RepoListAll("cmii")
for _, repo := range repoListAll {
repoName := strings.TrimPrefix(repo.Name, "cmii/")
artifactListAll := DefaultHarborOperator.ArtifactListAll("cmii", repoName)
for _, artifact := range artifactListAll {
//if len(artifact.Tags) == 0 {
// fmt.Println(artifact.Digest)
//}
for _, tag := range artifact.Tags {
if strings.Contains(tag.Name, "snapshot") || strings.Contains(tag.Name, "validation") || strings.Contains(tag.Name, "beta") || strings.Contains(tag.Name, "staging") || strings.Contains(tag.Name, "test") {
fmt.Println(tag.Name)
fmt.Println(artifact.Digest)
fmt.Println()
DefaultHarborOperator.ArtifactDeleteOne("cmii", repoName, artifact.Digest)
}
}
}
}
//utils.BeautifulPrint(artifactListAll)
}
func TestHarborOperator_ArtifactListOne(t *testing.T) {
TestHarborOperator_BuildOperator(t)
//reference := "sha256:0048162a053eef4d4ce3fe7518615bef084403614f8bca43b40ae2e762e11e06" // not ok icon
reference := "sha256:27bd0055156abc20c29863750f13bbcc14019126da36d3941cfd82eb104ec31a" // ok digest
// reference := "5.2.0" // ok tag
artifactListOne := DefaultHarborOperator.ArtifactListOne("cmii", "cmii-uav-user", reference)
utils.BeautifulPrint(artifactListOne)
}
func TestHarborOperator_ArtifactDeleteOne(t *testing.T) {
TestHarborOperator_BuildOperator(t)
reference := "sha256:27bd0055156abc20c29863750f13bbcc14019126da36d3941cfd82eb104ec31a"
ccc := DefaultHarborOperator.ArtifactDeleteOne("cmii", "cmii-uav-user", reference)
if !ccc {
log.Error("delete failed")
}
}
func TestHarborOperator_CmiiTagFilter(t *testing.T) {
TestHarborOperator_BuildOperator_CMII(t)
imageMap := DefaultHarborOperator.CmiiTagFilter("5.7")
utils.BeautifulPrint(imageMap)
}
func TestHarborOperator_ArtifactDeleteFromNameTagList(t *testing.T) {
TestHarborOperator_BuildOperator(t)
allCmiiImageList := DefaultHarborOperator.CmiiTagFilter("5.5")
errorDeleteList := DefaultHarborOperator.ArtifactDeleteFromNameTagList("cmii", allCmiiImageList)
utils.BeautifulPrint(errorDeleteList)
}

659
agent-operator/image/ImageOperator.go Normal file
View File

@@ -0,0 +1,659 @@
package image
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/klauspost/pgzip"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
image2 "wdd.io/agent-common/image"
"wdd.io/agent-common/logger"
"wdd.io/agent-common/utils"
"wdd.io/agent-deploy/d_app"
)
var apiClient = newClient()
var log = logger.Log
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
func newClient() *client.Client {
apiClient, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
log.ErrorF("[newClient] - new client failed ! => %s", err.Error())
panic(err)
}
defer apiClient.Close()
return apiClient
}
func GetRunningContainer() []types.Container {
return getContainerList(false)
}
func GetAllContainer() []types.Container {
return getContainerList(true)
}
func getContainerList(all bool) []types.Container {
containers, err := apiClient.ContainerList(context.Background(), types.ContainerListOptions{
Quiet: false,
Size: false,
All: all,
Latest: false,
Limit: 0,
Filters: filters.Args{},
})
if err != nil {
log.ErrorF("[getContainerList] - error => %s", err.Error())
}
return containers
}
//func SetDockerProxy(httpProxy string) {
// proxy_all := "HTTP_PROXY=" + httpProxy + " HTTPS_PROXY=" + httpProxy
//
// log.InfoF("[SetDockerProxy] - set docker proxy to %s", proxy_all)
//
// // Set the HTTP proxy for the Docker daemon
// configFile, err := apiClient.ConfigCreate(context.Background(), swarm.ConfigSpec{
// Annotations: swarm.Annotations{
// Name: "proxy.conf",
// },
// Data: []byte(proxy_all),
// })
// if err != nil {
// // 处理错误
// log.ErrorF("[SetDockerProxy] - set docker proxy error %s", err.Error())
// }
//
//
//
//
//}
func GetAll() []types.ImageSummary {
list, err := apiClient.ImageList(context.TODO(), types.ImageListOptions{
All: true,
Filters: filters.Args{},
})
if err != nil {
log.ErrorF("[ImageGetAll] --error => %s", err.Error())
}
return list
}
func GetByName(imageName string) *types.ImageSummary {
imageGetAll := GetAll()
for _, imageSummary := range imageGetAll {
for _, tag := range imageSummary.RepoTags {
if strings.Contains(tag, imageName) {
return &imageSummary
}
}
}
return nil
}
func Delete(imageName string) []types.ImageDeleteResponseItem {
imageGetByName := GetByName(imageName)
if imageGetByName == nil {
log.ErrorF("[ImageDelete] -- image not exists ! %s", imageGetByName.RepoTags)
return nil
}
remove, err := apiClient.ImageRemove(context.TODO(), imageGetByName.ID, types.ImageRemoveOptions{
Force: true,
PruneChildren: false,
})
if err != nil {
log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error())
return nil
}
return remove
}
func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
imageGetAll := GetAll()
// ip:8033
//re := regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}:\d{1,4}`)
for _, imageSummary := range imageGetAll {
for _, repoTag := range imageSummary.RepoTags {
if strings.HasPrefix(repoTag, image2.CmiiHarborPrefix) || strings.HasPrefix(repoTag, "harbor.wdd.io") || strings.Contains(repoTag, ":8033") {
for _, tag := range imageSummary.RepoTags {
_, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{
Force: true,
PruneChildren: true,
})
if err != nil {
log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error())
errorRemoveImageNameList = append(errorRemoveImageNameList, tag)
}
log.InfoF("[ImageDelete] - image remove of [%s] success!", tag)
}
}
}
}
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
return errorRemoveImageNameList
}
func TagFromSourceToTarget(sourceImageName, targetImageName string) bool {
getByName := GetByName(sourceImageName)
if getByName == nil {
log.ErrorF("[ImageTagFromSourceToTarget] - %s not exits !", sourceImageName)
return false
}
sourceImageName = getByName.RepoTags[0]
err := apiClient.ImageTag(context.TODO(), sourceImageName, targetImageName)
if err != nil {
log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error())
}
log.InfoF("[ImageTagFromSourceToTarget] - from %s to %s success!", sourceImageName, targetImageName)
return true
}
func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) {
if GetByName(targetImageName) == nil {
log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName)
return pushResult
}
pushResult, err := apiClient.ImagePush(context.TODO(), targetImageName, types.ImagePushOptions{
All: false,
//RegistryAuth: "eyAidXNlcm5hbWUiOiAiYWRtaW4iLCAicGFzc3dvcmQiOiAiVjJyeVN0ckBuZ1BzcyIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
// 重庆移动二级平台
RegistryAuth: "eyAidXNlcm5hbWUiOiAiemd5ZHR4anRjcXl4Z3MxODg4MzI1NzMxMSIsICJwYXNzd29yZCI6ICJEaWN0QDIwMjQiLCAiZW1haWwiOiAiaWNlQHFxLmNvbSIgfQ==",
PrivilegeFunc: nil,
Platform: "amd64",
})
if err != nil {
log.ErrorF("[ImagePushToOctopusKindHarbor] - push %s error %s", targetImageName, err.Error())
return nil
}
return pushResult
}
func UploadToHarbor(targetImageName string) (uploadOK bool) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
fmt.Println()
log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
fmt.Println()
return true
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
targetHarborHost = strings.TrimPrefix(targetHarborHost, "http://")
targetHarborHost = strings.TrimPrefix(targetHarborHost, "https://")
for _, imageName := range referenceImageList {
// check image
// harbor.cdcyy.cn
cmiiImageFullName := imageName
// convert image name
targetImageName := image2.ImageNameToTargetImageFullName(imageName, targetHarborHost)
if TagFromSourceToTarget(cmiiImageFullName, targetImageName) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
if pushResult == nil {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName)
break
}
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
text := scanner.Text()
if strings.Contains(text, "digest: sha256:") {
fmt.Println(scanner.Text())
}
}
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
fmt.Println()
} else {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
}
}
return errorPushImageNameList
}
func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
pullResult, err := apiClient.ImagePull(context.TODO(), imageName, types.ImagePullOptions{
All: false,
RegistryAuth: "eyAidXNlcm5hbWUiOiAicmFkMDJfZHJvbmUiLCAicGFzc3dvcmQiOiAiRHJvbmVAMTIzNCIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
PrivilegeFunc: func() (string, error) {
return "authorization: Basic cmFkMDJfZHJvbmU6RHJvbmVAMTIzNA==", nil
},
Platform: "amd64",
})
if err != nil {
log.ErrorF("[ImagePullFromCmiiHarbor]- image pull of %s ,error => %s", imageName, err.Error())
return nil
}
return pullResult
}
func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
fullImageNameList = CmiiImageMapToFullNameList(imageVersionMap)
return fullImageNameList, PullFromFullNameList(fullImageNameList)
}
func PullCmiiFromFileJson(filePathName string) {
readFile, err := os.ReadFile(filePathName)
if err != nil {
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s read error ! %s", filePathName, err.Error())
return
}
var resultMap map[string]string
err = json.Unmarshal(readFile, &readFile)
if err != nil {
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
return
}
for image, tag := range resultMap {
pullResult := PullFromCmiiHarbor(image + ":" + tag)
if pullResult == nil {
continue
}
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
fmt.Println()
}
}
// PullFromFullNameList 根据镜像名列表拉取全部的镜像
func PullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
start := time.Now()
for _, fullImageName := range fullImageNameList {
if !strings.HasPrefix(fullImageName, "harbor.cdcyy.com.cn") {
since := time.Since(start)
if since < 60*time.Second {
duration := 60*time.Second - since
log.DebugF("PullFromFullNameList - wait for %s !", duration.String())
time.Sleep(duration)
start = time.Now()
}
}
log.DebugF("start to pull => [%s]", fullImageName)
pullResult := PullFromCmiiHarbor(fullImageName)
if pullResult == nil {
errorPullImageList = append(errorPullImageList, fullImageName)
continue
}
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
//if strings.Contains(line, "\"status\":\"Pulling from") {
// fmt.Println(line)
//}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
fmt.Println()
}
return errorPullImageList
}
func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
errorPullImageList = PullFromFullNameList(fullImageNameList)
// generate a project folder
err := os.MkdirAll(gzipFolder, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error())
}
}
var tarGzipFileNameList []string
for _, image := range fullImageNameList {
ok, path := SaveToGzipFile(image, gzipFolder)
if !ok {
errorGzipImageList = append(errorGzipImageList, image)
continue
}
tarGzipFileNameList = append(tarGzipFileNameList, path)
}
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
return errorPullImageList, errorGzipImageList
}
// LoadFromGzipFilePath 根据Gzip文件的全名称进行Load操作
func LoadFromGzipFilePath(gzipFileNameFullPath string) bool {
openFile, err := os.OpenFile(gzipFileNameFullPath, 0, fs.ModePerm)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFileNameFullPath, err.Error())
return false
}
loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFileNameFullPath, err.Error())
return false
}
log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFileNameFullPath, strconv.FormatBool(loadResponse.JSON))
scanner := bufio.NewScanner(loadResponse.Body)
for scanner.Scan() {
line := scanner.Text()
fmt.Println(line)
}
return true
}
func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
if !strings.HasSuffix(folderPath, "/") {
folderPath += "/"
}
dirEntries, err := os.ReadDir(folderPath)
if err != nil {
log.ErrorF("[ImageLoadFromFolderPath] - error read folder %s error is %s", folderPath, err.Error())
return
}
// load gzip file
for _, dirEntry := range dirEntries {
if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
if !LoadFromGzipFilePath(folderPath + dirEntry.Name()) {
errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
}
}
}
return errorLoadImageNameList
}
// SaveToGzipFile 根据目标镜像的全名称 将镜像压缩为Gzip文件
func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipImageFileFullPath string) {
imageGetByName := GetByName(imageFullName)
if imageGetByName == nil {
log.ErrorF("[ImageSaveToTarGZ] - %s not exits", imageFullName)
return false, ""
}
imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error())
return false, ""
}
var realImageTag string
for _, repoTag := range imageGetByName.RepoTags {
if !strings.Contains(repoTag, "8033") {
realImageTag = repoTag
break
}
}
gzipImageFileFullPath = image2.ImageFullNameToGzipFileName(realImageTag)
if err := os.MkdirAll(filepath.Dir(gzipImageFileFullPath), os.ModePerm); err != nil {
log.ErrorF("[ImageSaveToTarGZ] - failed to create directory: %s", err)
return false, ""
}
// 生成gzip压缩文件的全路径名称
gzipImageFileFullPath = filepath.Join(folderPathPrefix, gzipImageFileFullPath)
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFileFullPath)
// 删除掉旧的Gzip文件
if err := os.Remove(gzipImageFileFullPath); err != nil && !os.IsNotExist(err) {
log.ErrorF("[ImageSaveToTarGZ] - failed to remove old gzip file: %s", err)
return false, ""
}
// 创建
tarFile, err := os.Create(gzipImageFileFullPath)
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFileFullPath, err.Error())
return false, ""
}
defer tarFile.Close()
// 创建pgzip writer
gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression) // 你可以调整压缩级别
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - pgzip create writer error: %s", err.Error())
}
defer gw.Close()
// Copy the tar archive to the gzip writer.
if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error())
return false, ""
}
// 成功
return true, gzipImageFileFullPath
}
// SaveImageListToGzipFile 将一个列表内的镜像全部压缩为一个tar.gz文件
func SaveImageListToGzipFile(imageFullNames []string, folderPathPrefix string, outputFileName string) (gzipOK bool, gzipFileFullPath string, errorGzipImageList []string) {
//if len(imageFullNames) == 0 {
// log.Error("[SaveImagesToGzipFile] - no images provided")
// return false, "", errorGzipImageList
//}
//
//// 确保输出文件路径
//gzipFileFullPath = folderPathPrefix + outputFileName
//if err := os.MkdirAll(filepath.Dir(gzipFileFullPath), os.ModePerm); err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - failed to create directory: %s", err)
// return false, "", errorGzipImageList
//}
//
//log.InfoF("[SaveImagesToGzipFile] - start saving images to [%s]", gzipFileFullPath)
//
//// 删除旧的Gzip文件
//if err := os.removePrefix(gzipFileFullPath); err != nil && !os.IsNotExist(err) {
// log.ErrorF("[SaveImagesToGzipFile] - failed to remove old gzip file: %s", err)
// return false, "", errorGzipImageList
//}
//
//tarFile, err := os.Create(gzipFileFullPath)
//if err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - error creating gzip file: %s", err)
// return false, "", errorGzipImageList
//}
//defer tarFile.Close()
//
//gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression)
//if err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - pgzip writer creation error: %s", err)
// return false, "", errorGzipImageList
//}
//defer gw.Close()
//
//for _, imageFullName := range imageFullNames {
// imageGetByName := GetByName(imageFullName)
// if imageGetByName == nil {
// log.WarnF("[SaveImagesToGzipFile] - %s not exists, skipping", imageFullName)
// continue
// }
//
// imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
// if err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - image save error for %s: %s", imageFullName, err)
// continue
// }
//
// if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - failed to copy tar archive for %s to gzip writer: %s", imageFullName, err)
// continue
// }
//
//}
//
//if err := gw.Close(); err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - error closing gzip writer: %s", err)
// return false, "", errorGzipImageList
//}
//
//log.InfoF("[SaveImagesToGzipFile] - successfully saved images to [%s]", gzipFileFullPath)
return true, gzipFileFullPath, errorGzipImageList
}
func CmiiImageMapToFullNameList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
for image, tag := range cmiiImageVersionMap {
s := image2.CmiiHarborPrefix + image + ":" + tag
fullImageNameList = append(fullImageNameList, s)
}
return fullImageNameList
}
func CmiiImageMapFromGzipFolder(gzipFileFolder string) (cmiiImageVersionMap map[string]string) {
allFileInFolder, err := utils.ListAllFileInFolder(gzipFileFolder)
if err != nil {
return nil
}
cmiiImageVersionMap = make(map[string]string)
for _, gzipFileName := range allFileInFolder {
log.DebugF("gzip file name is %s", gzipFileName)
imageName, imageTag := image2.GzipFileNameToImageNameAndTag(gzipFileName)
cmiiImageVersionMap[imageName] = imageTag
}
return cmiiImageVersionMap
}
// GenerateCmiiTagVersionImageMap 生成特定版本的ImageTagMap
func GenerateCmiiTagVersionImageMap(specificTag string) (backendMap, frontendMap, srsMap map[string]string) {
matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+$`, specificTag)
if !matched {
sprintf := fmt.Sprintf("tag is not match ! [%s]", specificTag)
log.Error(sprintf)
panic(sprintf)
}
backendMap = make(map[string]string, len(d_app.CmiiBackendAppMap))
frontendMap = make(map[string]string, len(d_app.CmiiFrontendAppMap))
srsMap = make(map[string]string, len(d_app.CmiiSrsAppMap))
for imageName := range d_app.CmiiBackendAppMap {
backendMap[imageName] = specificTag
}
for imageName := range d_app.CmiiFrontendAppMap {
frontendMap[imageName] = specificTag
}
for imageName, imageTag := range d_app.CmiiSrsAppMap {
srsMap[imageName] = imageTag
}
return backendMap, frontendMap, srsMap
}
func loginToDockerHub(HarborFullHost string) {
if HarborFullHost == "" {
HarborFullHost = "https://registry-1.docker.io"
}
login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{
Username: "icederce",
Password: "loveff.cxc.23",
Auth: "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==",
ServerAddress: HarborFullHost,
})
if err != nil {
log.ErrorF("[loginToDockerHub] - login to %s failed !", HarborFullHost)
}
log.DebugF("[loginToDockerHub] - login is %s", login.Status)
}
func WriteDependencyImageToFile() {
imageFilePrefix := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\image\\"
middleFile := imageFilePrefix + "middle-image.txt"
_ = os.Remove(middleFile)
for _, image := range d_app.MiddlewareAmd64 {
utils.AppendContentToFile(image+"\n", middleFile)
}
rkeFile := imageFilePrefix + "rke-image.txt"
_ = os.Remove(rkeFile)
for _, image := range d_app.Rancher1204Amd64 {
utils.AppendContentToFile(image+"\n", rkeFile)
}
}

302
agent-operator/image/ImageOperator_test.go Normal file
View File

@@ -0,0 +1,302 @@
package image
import (
"bufio"
"fmt"
"strings"
"testing"
"wdd.io/agent-common/assert"
"wdd.io/agent-common/image"
"wdd.io/agent-common/utils"
"wdd.io/agent-deploy/d_app"
)
func TestGetRunningContainer(t *testing.T) {
allContainer := GetRunningContainer()
for _, container := range allContainer {
utils.BeautifulPrint(container)
}
}
func TestGetAllContainer(t *testing.T) {
allContainer := GetAllContainer()
for _, container := range allContainer {
utils.BeautifulPrint(container)
}
}
func TestImageGetAll(t *testing.T) {
imageGetAll := GetAll()
for _, summary := range imageGetAll {
utils.BeautifulPrint(summary)
}
}
func TestImageGetByName(t *testing.T) {
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
imageGetByName := GetByName(image)
utils.BeautifulPrint(imageGetByName)
}
func TestImageDelete(t *testing.T) {
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
imageDelete := Delete(image)
for _, item := range imageDelete {
utils.BeautifulPrint(item)
}
}
func TestImagePullFromCmiiHarbor(t *testing.T) {
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
pullFromCmiiHarbor := PullFromCmiiHarbor(image)
defer pullFromCmiiHarbor.Close()
scanner := bufio.NewScanner(pullFromCmiiHarbor)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
//var fs uintptr
//
//tools.DisplayJSONMessagesStream(pullFromCmiiHarbor, os.Stdout, fs, true, func(message tools.JSONMessage) {
//
//})
}
func TestImagePushToOctopusKindHarbor(t *testing.T) {
// re-tag
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
newTag := "10.250.0.100:8033/cmii/cmii-uav-gateway:4.1.6-beta"
target := TagFromSourceToTarget(image, newTag)
assert.Equal(t, target, true, "image re-tag error !")
// push
pushResult := UploadToOctopusKindHarbor(newTag)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
fmt.Println("image push success!")
}
func TestImageLoadFromFile(t *testing.T) {
loadFromFile := LoadFromGzipFilePath("/root/cmlc=cmii=srs=v5.0.195.tar.gz")
assert.Equal(t, loadFromFile, true, "image load error !")
}
func TestImageSaveToTarGZ(t *testing.T) {
image := "harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:4.1.6-beta"
imageSaveToTarGZ, _ := SaveToGzipFile(image, "/home/wdd/IdeaProjects/ProjectOctopus/cmii_operator/log")
assert.Equal(t, imageSaveToTarGZ, true, "image save to tar gz file error !")
}
func TestImageFullNameToGzipFileName(t *testing.T) {
test := []string{
"bitnami/redis:6.2.6-debian-10-r0",
"simonrupf/chronyd:0.4.3",
"harbor.cdcyy.com.cn/cmii/cmii-rtsp-operator:v4.1.0",
"harbor.cdcyy.com.cn/cmii/ossrs/srs:v4.0.136",
"ossrs/srs:v4.0.136",
"mongo:5.0",
"bitnami/minio:2023.5.4",
"busybox:latest",
"busybox",
}
test = append(test, d_app.MiddlewareAmd64...)
for _, s := range test {
gzipFileName := image.ImageFullNameToGzipFileName(s)
fmt.Println(gzipFileName)
}
}
func TestImageTagFromSourceToTarget(t *testing.T) {
sourceImageName := "ossrs/srs:v5.0.195"
targetImageName := "harbor.wdd.io:8033/cmii/srs:v5.0.195"
if TagFromSourceToTarget(sourceImageName, targetImageName) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
}
}
func TestSaveSpecificImageToGzipFile(t *testing.T) {
imageGzipFilePathPrefix := "/root/ai_image/"
imageFullNameList := []string{
"harbor.cdcyy.com.cn/cmii/cmlc-ai/cmlc-ai-operator:v5.2.0-t4-no-dino",
}
for _, imageFullName := range imageFullNameList {
result := PullFromCmiiHarbor(imageFullName)
if result == nil {
log.ErrorF("image pull error ! => %s", imageFullName)
return
}
scc := bufio.NewScanner(result)
for scc.Scan() {
line := scc.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
// image pull success
log.InfoF("image should have pulled successful ! => %s", imageFullName)
ok, _ := SaveToGzipFile(imageFullName, imageGzipFilePathPrefix)
if !ok {
log.ErrorF("image save to gzip file error ! => %s", imageFullName)
return
}
}
}
func TestConvertCmiiImageMapFromGzipFolder(t *testing.T) {
versionMap := CmiiImageMapFromGzipFolder(OfflineImageGzipFolderPrefix)
utils.BeautifulPrint(versionMap)
}
func TestImageNameToTargetImageFullName(t *testing.T) {
AllCmiiImageTagList := []string{
"harbor.cdcyy.com.cn/cmii/cmii-uav-cloud-live:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-data-post-process:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-depotautoreturn:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-logger:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-mqtthandler:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-notice:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-admin-gateway:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-cms:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-material-warehouse:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-process:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-autowaypoint:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-brain:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-mission:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-oauth:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-airspace:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-suav-supervision:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-user:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-admin-user:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-emergency:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-kpi-monitor:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-waypoint:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-device:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-alarm:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-clusters:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-surveillance:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-threedsimulation:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-open-gateway:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-tower:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-developer:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-app-release:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-integration:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-admin-data:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-armypeople:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-base:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-logistics:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-seniclive:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-visualization:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-cms-portal:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-multiterminal:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-security:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-media:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-oms:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-securityh5:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-share:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-suav-platform-supervision:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-ai-brain:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-emergency-rescue:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-splice:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-jiangsuwenlv:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-open:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-qinghaitourism:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-qingdao:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-mws:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-threedsimulation:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-suav-platform-supervisionh5:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-detection:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-platform-hljtt:5.4.0",
"harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
"ossrs/srs:v5.0.195",
"harbor.cdcyy.com.cn/cmii/cmii-srs-oss-adaptor:2023-SA",
}
AllCmiiImageTagList = append(AllCmiiImageTagList, d_app.Rancher1204Amd64...)
AllCmiiImageTagList = append(AllCmiiImageTagList, d_app.MiddlewareAmd64...)
var result []string
for _, imageName := range AllCmiiImageTagList {
targetImageFullName := image.ImageNameToTargetImageFullName(imageName, "cq.com")
result = append(result, targetImageFullName)
}
utils.BeautifulPrint(result)
}
func TestSaveImageListToGzipFile(t *testing.T) {
allImageList := []string{
"harbor.cdcyy.com.cn/cmii/cmii-uav-oauth:5.7.0",
"harbor.cdcyy.com.cn/cmii/cmii-iam-gateway:5.6.0",
"harbor.cdcyy.com.cn/cmii/cmii-uav-kpi-monitor:5.5.0",
}
gzipOK, gzipFileFullPath, errorGzipImageList := SaveImageListToGzipFile(allImageList, "/home/wdd/IdeaProjects/ProjectOctopus/agent-operator/image", "test.tar.gz")
assert.Equal(t, gzipOK, true, "gzip image list to single file failed !")
log.Info("gzip file is " + gzipFileFullPath)
utils.BeautifulPrint(errorGzipImageList)
}
func TestImagePruneAllCmiiImages(t *testing.T) {
errorRemoveImageNameList := PruneAllCmiiImages()
utils.BeautifulPrintListWithTitle(errorRemoveImageNameList, "CMII Image Prune Error")
}

166
agent-operator/image/ImageSyncDLTU.sh Normal file
View File

@@ -0,0 +1,166 @@
#!/bin/bash
all_image_list_txt="all-cmii-image-list.txt" # 需要修改版本
gzip_image_list_txt="all-gzip-image-list.txt" # 一般不需要修改
oss_prefix_url="https://oss.demo.uavcmlc.com/cmlc-installation"
local_gzip_path="/root/octopus_image"
DockerRegisterDomain="10.40.51.5:8033" # 需要根据实际修改
HarborAdminPass=V2ryStr@ngPss # 需要跟第一脚本中的密码保持一致
print_green() {
echo -e "\033[32m${1}\033[0m"
echo ""
}
print_red() {
echo -e "\033[31m${1}\033[0m"
echo ""
}
Download_Load_Tag_Upload() {
print_green "[DLTU] - start !"
while [[ $# -gt 0 ]]; do
case "$1" in
rke)
# print_green "download rke "
local_gzip_path="$local_gzip_path/rke"
mkdir -p ${local_gzip_path}
oss_prefix_url="$oss_prefix_url/rke/"
dltu
shift # past argument
;;
middle)
local_gzip_path="$local_gzip_path/middle"
mkdir -p $local_gzip_path
oss_prefix_url="$oss_prefix_url/middle/"
dltu
shift # past argument
;;
cmii)
local_gzip_path="$local_gzip_path/cmii"
mkdir -p $local_gzip_path
oss_prefix_url="$oss_prefix_url/cmii/"
dltu
shift # past argument
;;
*)
# unknown option
print_red "bad arguments"
;;
esac
done
}
dltu() {
print_green "download all image name list and gzip file list!"
cd $local_gzip_path || exit
rm $all_image_list_txt
rm $gzip_image_list_txt
wget "$oss_prefix_url$all_image_list_txt"
wget "$oss_prefix_url$gzip_image_list_txt"
docker login -u admin -p ${HarborAdminPass} ${DockerRegisterDomain}
echo ""
while IFS= read -r i; do
[ -z "${i}" ] && continue
echo "download gzip file =>: $oss_prefix_url${i}"
if wget "$oss_prefix_url${i}" >/dev/null 2>&1; then
echo "Gzip file download success : ${i}"
image_full_name=$(docker load -i ${i} | head -n1 |awk -F': ' '{print $2}')
app_name=$(echo "$image_full_name" | sed 's|.*/||g')
echo "extract short name is $app_name"
if echo $image_full_name | grep -q "rancher"
then
print_green "tag image to => $DockerRegisterDomain/rancher/$app_name"
docker tag ${image_full_name} $DockerRegisterDomain/rancher/$app_name
docker push $DockerRegisterDomain/rancher/$app_name
else
print_green "tag image to => $DockerRegisterDomain/cmii/$app_name"
docker tag ${image_full_name} $DockerRegisterDomain/cmii/$app_name
docker push $DockerRegisterDomain/cmii/$app_name
fi
else
print_red "Gzip file download FAILED : ${i}"
fi
echo "-------------------------------------------------"
done <"${gzip_image_list_txt}"
shift
}
Load_Tag_Upload(){
print_green "[LTU] - start to load image from offline !"
while [[ $# -gt 0 ]]; do
case "$1" in
rke)
# print_green "download rke "
local_gzip_path="$local_gzip_path/rke"
mkdir -p ${local_gzip_path}
oss_prefix_url="$oss_prefix_url/rke/"
ltu
shift # past argument
;;
middle)
local_gzip_path="$local_gzip_path/middle"
mkdir -p $local_gzip_path
oss_prefix_url="$oss_prefix_url/middle/"
ltu
shift # past argument
;;
cmii)
local_gzip_path="$local_gzip_path/cmii"
mkdir -p $local_gzip_path
oss_prefix_url="$oss_prefix_url/cmii/"
ltu
shift # past argument
;;
*)
# unknown option
print_red "bad arguments"
;;
esac
done
}
ltu(){
all_file_list=$(find $local_gzip_path -type f -name "*.tar.gz")
for file in $all_file_list; do
echo "offline gzip file is => : $file"
image_full_name=$(docker load -i ${file} | head -n1 |awk -F': ' '{print $2}')
app_name=$(echo "$image_full_name" | sed 's|.*/||g')
echo "extract short name is $app_name"
if echo $image_full_name | grep -q "rancher"
then
print_green "tag image to => $DockerRegisterDomain/rancher/$app_name"
docker tag ${image_full_name} $DockerRegisterDomain/rancher/$app_name
docker push $DockerRegisterDomain/rancher/$app_name
else
print_green "tag image to => $DockerRegisterDomain/cmii/$app_name"
docker tag ${image_full_name} $DockerRegisterDomain/cmii/$app_name
docker push $DockerRegisterDomain/cmii/$app_name
fi
done
}
test(){
app_name=$(echo "nginx:latest" | sed 's|.*/||g')
echo "extract short name is $app_name"
}
# test
#Download_Load_Tag_Upload "rke" "middle" "cmii"
Load_Tag_Upload "rke"

15
agent-operator/image/image_upload_oss.sh Normal file
View File

@@ -0,0 +1,15 @@
#!/bin/bash
#for image in $(ls /root/octopus_image/rke)
#do
# echo "start to upload => $image"
# mc cp /root/octopus_image/rke/$image demo/cmlc-installation/rke-image-amd64/
# echo ""
#done
export local_path=/root/octopus_image/5.4.0
for image in $(ls $local_path); do
echo "start to upload => $image"
mc cp $local_path/$image demo/cmlc-installation/5.4.0/
echo ""
done

17
agent-operator/image/middle-image-arm64-250218.txt Normal file
View File

@@ -0,0 +1,17 @@
bitnami/redis:6.2.14-debian-11-r1
bitnami/mysql:8.1.0-debian-11-r42
simonrupf/chronyd:0.4.3
bitnami/bitnami-shell:11-debian-11-r136
bitnami/rabbitmq:3.11.26-debian-11-r2
ossrs/srs:v5.0.195
emqx/emqx:5.5.1
nacos/nacos-server:v2.1.2-slim
mongo:5.0
bitnami/minio:2023.5.4
kubernetesui/dashboard:v2.0.1
kubernetesui/metrics-scraper:v1.0.4
nginx:1.24.0
dyrnq/nfs-subdir-external-provisioner:v4.0.2
jerrychina2020/rke-tools:v0.175-linux
jerrychina2020/rke-tools:v0.175
busybox:latest

18
agent-operator/image/middle-image.txt Normal file
View File

@@ -0,0 +1,18 @@
bitnami/redis:6.2.14-debian-11-r1
bitnami/mysql:8.1.0-debian-11-r42
simonrupf/chronyd:0.4.3
bitnami/bitnami-shell:11-debian-11-r136
bitnami/rabbitmq:3.11.26-debian-11-r2
ossrs/srs:v5.0.195
emqx/emqx:5.5.1
nacos/nacos-server:v2.1.2
nacos/nacos-server:v2.1.2-slim
mongo:5.0
bitnami/minio:2023.5.4
kubernetesui/dashboard:v2.0.1
kubernetesui/metrics-scraper:v1.0.4
nginx:1.24.0
dyrnq/nfs-subdir-external-provisioner:v4.0.2
jerrychina2020/rke-tools:v0.175-linux
jerrychina2020/rke-tools:v0.175
busybox:latest

185
agent-operator/image/octopus_image_sync.sh Normal file
View File

@@ -0,0 +1,185 @@
#!/bin/bash
cmii_image_list=(
cmlc=cmii=cmii-admin-data=5.4.0.tar.gz
cmlc=cmii=cmii-admin-gateway=5.4.0.tar.gz
docker=ossrs=srs=v4.0.136.tar.gz
)
middle_image_list=(
docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz
docker=kubernetesui=dashboard=v2.0.1.tar.gz
docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz
docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz
docker=bitnami=minio=2022.5.4.tar.gz
docker=library=busybox=latest.tar.gz
docker=bitnami=minio=2023.5.4.tar.gz
docker=library=mongo=5.0.tar.gz
docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz
docker=library=nginx=1.21.3.tar.gz
docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz
docker=library=rabbitmq=3.9-management.tar.gz
docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz
docker=library=redis=6.0.20-alpine.tar.gz
docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz
docker=nacos=nacos-server=v2.1.2-slim.tar.gz
docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz
docker=ossrs=srs=v4.0.136.tar.gz
docker=bitnami=redis=6.2.6-debian-10-r0.tar.gz
docker=ossrs=srs=v4.0-r3.tar.gz
docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz
docker=ossrs=srs=v5.0.195.tar.gz
docker=emqx=emqx=4.2.12.tar.gz
docker=simonrupf=chronyd=0.4.3.tar.gz
)
rke_image_list=(
docker=rancher=backup-restore-operator=v1.0.3.tar.gz
docker=rancher=calico-cni=v3.17.2.tar.gz
docker=rancher=calico-ctl=v3.17.2.tar.gz
docker=rancher=calico-kube-controllers=v3.17.2.tar.gz
docker=rancher=calico-node=v3.17.2.tar.gz
docker=rancher=calico-pod2daemon-flexvol=v3.17.2.tar.gz
docker=rancher=cis-operator=v1.0.3.tar.gz
docker=rancher=cluster-proportional-autoscaler=1.7.1.tar.gz
docker=rancher=cluster-proportional-autoscaler=1.8.1.tar.gz
docker=rancher=configmap-reload=v0.3.0-rancher4.tar.gz
docker=rancher=coredns-coredns=1.8.0.tar.gz
docker=rancher=coreos-etcd=v3.4.14-rancher1.tar.gz
docker=rancher=coreos-flannel=v0.13.0-rancher1.tar.gz
docker=rancher=coreos-kube-state-metrics=v1.9.7.tar.gz
docker=rancher=coreos-prometheus-config-reloader=v0.39.0.tar.gz
docker=rancher=coreos-prometheus-operator=v0.39.0.tar.gz
docker=rancher=externalip-webhook=v0.1.6.tar.gz
docker=rancher=flannel-cni=v0.3.0-rancher6.tar.gz
docker=rancher=fleet-agent=v0.3.4.tar.gz
docker=rancher=fleet=v0.3.4.tar.gz
docker=rancher=fluentd=v0.1.24.tar.gz
docker=rancher=grafana-grafana=7.1.5.tar.gz
docker=rancher=hyperkube=v1.20.4-rancher1.tar.gz
docker=rancher=istio-kubectl=1.5.10.tar.gz
docker=rancher=jimmidyson-configmap-reload=v0.3.0.tar.gz
docker=rancher=k8s-dns-dnsmasq-nanny=1.15.2.tar.gz
docker=rancher=k8s-dns-kube-dns=1.15.2.tar.gz
docker=rancher=k8s-dns-node-cache=1.15.13.tar.gz
docker=rancher=k8s-dns-sidecar=1.15.2.tar.gz
docker=rancher=klipper-lb=v0.1.2.tar.gz
docker=rancher=kube-api-auth=v0.1.4.tar.gz
docker=rancher=kubernetes-external-dns=v0.7.3.tar.gz
docker=rancher=library-busybox=1.31.1.tar.gz
docker=rancher=library-busybox=1.32.1.tar.gz
docker=rancher=library-nginx=1.19.2-alpine.tar.gz
docker=rancher=library-traefik=1.7.19.tar.gz
docker=rancher=local-path-provisioner=v0.0.11.tar.gz
docker=rancher=local-path-provisioner=v0.0.14.tar.gz
docker=rancher=local-path-provisioner=v0.0.19.tar.gz
docker=rancher=log-aggregator=v0.1.7.tar.gz
docker=rancher=metrics-server=v0.4.1.tar.gz
docker=rancher=nginx-ingress-controller-defaultbackend=1.5-rancher1.tar.gz
docker=rancher=nginx-ingress-controller=nginx-0.43.0-rancher1.tar.gz
docker=rancher=opa-gatekeeper=v3.1.0-beta.7.tar.gz
docker=rancher=openzipkin-zipkin=2.14.2.tar.gz
docker=rancher=pause=3.2.tar.gz
docker=rancher=plugins-docker=18.09.tar.gz
docker=rancher=prom-alertmanager=v0.21.0.tar.gz
docker=rancher=prometheus-auth=v0.2.1.tar.gz
docker=rancher=prom-node-exporter=v1.0.1.tar.gz
docker=rancher=prom-prometheus=v2.18.2.tar.gz
docker=rancher=rancher-agent=v2.5.7.tar.gz
docker=rancher=rancher=v2.5.7.tar.gz
docker=rancher=rancher-webhook=v0.1.0-beta9.tar.gz
docker=rancher=rke-tools=v0.1.72.tar.gz
docker=rancher=security-scan=v0.1.14.tar.gz
docker=rancher=security-scan=v0.2.2.tar.gz
docker=rancher=shell=v0.1.6.tar.gz
docker=rancher=sonobuoy-sonobuoy=v0.16.3.tar.gz
docker=rancher=system-upgrade-controller=v0.6.2.tar.gz
)
oss_prefix=https://oss.demo.uavcmlc.com/cmlc-installation/shls
oss_middle_prefix=https://oss.demo.uavcmlc.com/cmlc-installation/mid-image-amd64
oss_rke_prefix=https://oss.demo.uavcmlc.com/cmlc-installation/rke-image-amd64
target_harbor_host=103.0.180.181:8033
cmii_image_download_from_oss() {
for image in "${cmii_image_list[@]}"; do
echo "start to download => $image"
curl -x socks5h://103.0.180.82:9997 $oss_prefix/$image -o $image
echo ""
done
}
middle_image_download_from_oss() {
mkdir -p /wdd/image/middle/
for image in "${middle_image_list[@]}"; do
echo "start to download => $image"
curl -x socks5h://103.0.180.82:9997 $oss_middle_prefix/$image -o /wdd/image/middle/$image
echo ""
done
}
rke_image_download_from_oss() {
mkdir -p /wdd/image/rke/
for image in "${rke_image_list[@]}"; do
echo "start to download => $image"
curl -x socks5h://103.0.180.82:9997 $oss_rke_prefix/$image -o /wdd/image/rke/$image
echo ""
done
}
image_load_to_harbor() {
local cmii_harbor_prefix="harbor.cdcyy.com.cn/cmii/"
for image in "${cmii_image_list[@]}"; do
echo "start to load => $image"
docker load <"$image"
echo ""
if [[ $image == cmlc* ]]; then
local app_name=$(echo $image | cut -d "=" -f3)
local ccc=$(echo $image | cut -d "=" -f4)
local app_tag="${ccc%.tar.gz}"
echo "from $cmii_harbor_prefix$app_name:$app_tag ==> $target_harbor_host/cmii/$app_name:$app_tag"
docker tag "$cmii_harbor_prefix$app_name:$app_tag" "$target_harbor_host/cmii/$app_name:$app_tag"
echo "start to push => $target_harbor_host/cmii/$app_name:$app_tag"
docker login -u admin -p V2ryStr@ngPss $target_harbor_host
docker push "$target_harbor_host/cmii/$app_name:$app_tag"
echo ""
fi
echo ""
done
for image in "${rke_image_list[@]}"; do
echo "start to load => $image"
docker load <"$image"
echo ""
local app_name_prefix=$(echo $image | cut -d "=" -f2)
local app_name=$(echo $image | cut -d "=" -f3)
local ccc=$(echo $image | cut -d "=" -f4)
local app_tag="${ccc%.tar.gz}"
echo "from $app_name_prefix/$app_name:$app_tag ==> $target_harbor_host/rancher/$app_name:$app_tag"
docker tag "$app_name_prefix/$app_name:$app_tag" "$target_harbor_host/rancher/$app_name:$app_tag"
echo "start to push => $target_harbor_host/rancher/$app_name:$app_tag"
docker login -u admin -p V2ryStr@ngPss $target_harbor_host
docker push "$target_harbor_host/rancher/$app_name:$app_tag"
echo
done
for image in "${middle_image_list[@]}"; do
echo "start to load => $image"
docker load <"$image"
echo ""
# docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz
local app_name_prefix=$(echo $image | cut -d "=" -f2)
local app_name=$(echo $image | cut -d "=" -f3)
local ccc=$(echo $image | cut -d "=" -f4)
local app_tag="${ccc%.tar.gz}"
echo "from $app_name_prefix/$app_name:$app_tag ==> $target_harbor_host/cmii/$app_name:$app_tag"
echo "start to push => $target_harbor_host/cmii/$app_name:$app_tag"
docker login -u admin -p V2ryStr@ngPss $target_harbor_host
docker push "$target_harbor_host/cmii/$app_name:$app_tag"
done
}
create_harbor_project() {
curl -X POST -u "admin:V2ryStr@ngPss" -H "authorization: Basic YWRtaW46VjJyeVN0ckBuZ1Bzcw==" -H "Content-Type: application/json" -d '{"project_name":"cmii","registry_id":null,"metadata":{"public":"true"},"storage_limit":-1}' http://$target_harbor_host/api/v2.0/projects
curl -X POST -u "admin:V2ryStr@ngPss" -H "authorization: Basic YWRtaW46VjJyeVN0ckBuZ1Bzcw==" -H "Content-Type: application/json" -d '{"project_name":"rancher","registry_id":null,"metadata":{"public":"true"},"storage_limit":-1}' http://$target_harbor_host/api/v2.0/projects
}

70
agent-operator/image/octopus_image_update.sh Normal file
View File

@@ -0,0 +1,70 @@
#!/bin/bash
full_image_name_list=(
harbor.cdcyy.com.cn/cmii/cmii-uav-cloud-live:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-data-post-process:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-depotautoreturn:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-logger:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-mqtthandler:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-notice:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-admin-gateway:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-cms:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-material-warehouse:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-process:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-autowaypoint:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-brain:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-mission:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-oauth:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-airspace:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-suav-supervision:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-user:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-admin-user:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-emergency:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-kpi-monitor:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-waypoint:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-device:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-alarm:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-clusters:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-surveillance:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-threedsimulation:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-open-gateway:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-tower:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-developer:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-app-release:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-gateway:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-integration:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-admin-data:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-armypeople:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-base:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-logistics:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-seniclive:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-visualization:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-cms-portal:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-multiterminal:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-security:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-media:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-oms:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-securityh5:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-share:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-suav-platform-supervision:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-ai-brain:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-emergency-rescue:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-splice:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-jiangsuwenlv:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-open:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-qinghaitourism:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-qingdao:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-mws:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-threedsimulation:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-suav-platform-supervisionh5:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-detection:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-uav-platform-hljtt:5.4.0
harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0
harbor.cdcyy.com.cn/cmii/cmii-srs-oss-adaptor:2023-SA
)
gzip_image_name_list=(
ossrs/srs:v5.0.195
)

62
agent-operator/image/rke-image.txt Normal file
View File

@@ -0,0 +1,62 @@
rancher/backup-restore-operator:v1.0.3
rancher/calico-cni:v3.17.2
rancher/calico-ctl:v3.17.2
rancher/calico-kube-controllers:v3.17.2
rancher/calico-node:v3.17.2
rancher/calico-pod2daemon-flexvol:v3.17.2
rancher/cis-operator:v1.0.3
rancher/cluster-proportional-autoscaler:1.7.1
rancher/coredns-coredns:1.8.0
rancher/coreos-etcd:v3.4.14-rancher1
rancher/coreos-kube-state-metrics:v1.9.7
rancher/coreos-prometheus-config-reloader:v0.39.0
rancher/coreos-prometheus-operator:v0.39.0
rancher/externalip-webhook:v0.1.6
rancher/flannel-cni:v0.3.0-rancher6
rancher/coreos-flannel:v0.13.0-rancher1
rancher/fleet-agent:v0.3.4
rancher/fleet:v0.3.4
rancher/fluentd:v0.1.24
rancher/grafana-grafana:7.1.5
rancher/hyperkube:v1.20.4-rancher1
rancher/jimmidyson-configmap-reload:v0.3.0
rancher/k8s-dns-dnsmasq-nanny:1.15.2
rancher/k8s-dns-kube-dns:1.15.2
rancher/k8s-dns-node-cache:1.15.13
rancher/k8s-dns-sidecar:1.15.2
rancher/klipper-lb:v0.1.2
rancher/kube-api-auth:v0.1.4
rancher/kubectl:v1.20.4
rancher/kubernetes-external-dns:v0.7.3
rancher/cluster-proportional-autoscaler:1.8.1
rancher/library-busybox:1.31.1
rancher/library-busybox:1.32.1
rancher/library-nginx:1.19.2-alpine
rancher/library-traefik:1.7.19
rancher/local-path-provisioner:v0.0.11
rancher/local-path-provisioner:v0.0.14
rancher/local-path-provisioner:v0.0.19
rancher/log-aggregator:v0.1.7
rancher/istio-kubectl:1.5.10
rancher/metrics-server:v0.4.1
rancher/configmap-reload:v0.3.0-rancher4
rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1
rancher/nginx-ingress-controller:nginx-0.43.0-rancher1
rancher/opa-gatekeeper:v3.1.0-beta.7
rancher/openzipkin-zipkin:2.14.2
rancher/pause:3.2
rancher/plugins-docker:18.09
rancher/prom-alertmanager:v0.21.0
rancher/prom-node-exporter:v1.0.1
rancher/prom-prometheus:v2.18.2
rancher/prometheus-auth:v0.2.1
rancher/rancher-agent:v2.5.7
rancher/rancher-webhook:v0.1.0-beta9
rancher/rancher:v2.5.7
rancher/rke-tools:v0.1.72
jerrychina2020/rke-tools:v0.175-linux
rancher/security-scan:v0.1.14
rancher/security-scan:v0.2.2
rancher/shell:v0.1.6
rancher/sonobuoy-sonobuoy:v0.16.3
rancher/system-upgrade-controller:v0.6.2