zeaslity/WddSuperAgent
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增xa空能院项目，新增大量的更新内容

Browse Source
This commit is contained in:
zeaslity
2025-11-05 16:44:40 +08:00
parent 0a1b92acc2
commit baf9272e2d
88 changed files with 18872 additions and 305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
agent-operator/image/HarborOperator_test.go
View File

@@ -4,6 +4,7 @@ import (
"fmt"
"strings"
"testing"
"wdd.io/agent-common/utils"
)
@@ -49,7 +50,7 @@ func TestHarborOperator_BuildOperator(t *testing.T) {
func TestHarborOperator_RepoListAll(t *testing.T) {
TestHarborOperator_BuildOperator(t)
TestHarborOperator_BuildOperator_CMII(t)
//repoListAll := DefaultHarborOperator.RepoListAll("ran")
repoListAll := DefaultHarborOperator.RepoListAll("cmii")
@@ -69,20 +70,31 @@ func TestHarborOperator_RepoAllCmiiImage(t *testing.T) {
func TestHarborOperator_ArtifactListAll(t *testing.T) {
TestHarborOperator_BuildOperator(t)
TestHarborOperator_BuildOperator_CMII(t)
artifactListAll := DefaultHarborOperator.ArtifactListAll("cmii", "cmii-uav-user")
var tagNameList []string
for _, artifact := range artifactListAll {
//if len(artifact.Tags) == 0 {
// fmt.Println(artifact.Digest)
//}
for _, tag := range artifact.Tags {
fmt.Println(tag.Name)
tagNameList = append(tagNameList, tag.Name)
}
}
//utils.BeautifulPrint(artifactListAll)
toDelete, keep := FindTagsToDelete(tagNameList)
fmt.Printf("总共有 %d 个标签, 其中 %d 个可以被删除, 需要保留 %d 个。\n\n", len(tagNameList), len(toDelete), len(keep))
fmt.Println("建议删除的标签列表:")
for _, tag := range toDelete {
fmt.Println(tag)
}
}
func TestHarborOperator_CmiiHarborCleanUp(t *testing.T) {
@@ -93,21 +105,43 @@ func TestHarborOperator_CmiiHarborCleanUp(t *testing.T) {
for _, repo := range repoListAll {
repoName := strings.TrimPrefix(repo.Name, "cmii/")
// 获取所有的
artifactListAll := DefaultHarborOperator.ArtifactListAll("cmii", repoName)
for _, artifact := range artifactListAll {
//if len(artifact.Tags) == 0 {
// fmt.Println(artifact.Digest)
//}
var tagNameList []string
for _, tag := range artifact.Tags {
// 添加到Tag列表中
tagNameList = append(tagNameList, tag.Name)
if strings.Contains(tag.Name, "snapshot") || strings.Contains(tag.Name, "validation") || strings.Contains(tag.Name, "beta") || strings.Contains(tag.Name, "staging") || strings.Contains(tag.Name, "test") {
fmt.Println(tag.Name)
fmt.Println(artifact.Digest)
fmt.Println(fmt.Sprintf("prepare delete image is %s:%s", repoName, tag.Name))
//DefaultHarborOperator.ArtifactDeleteOne("cmii", repoName, artifact.Digest)
fmt.Println()
DefaultHarborOperator.ArtifactDeleteOne("cmii", repoName, artifact.Digest)
}
}
// 过滤所有可以删除的Tag
//deletableTags := FilterTags(tagNameList)
//
//fmt.Printf("总共有 %d 个标签, 其中 %d 个可以被删除。\n\n", len(tagNameList), len(deletableTags))
//fmt.Println("建议删除的标签列表:")
//for _, tag := range deletableTags {
// fmt.Println(fmt.Sprintf("开始删除镜像 => %s:%s", repoName, tag))
// DefaultHarborOperator.ArtifactDeleteOne("cmii", repoName, tag)
//}
}
}
//utils.BeautifulPrint(artifactListAll)

169
agent-operator/image/HarborTagParser.go Normal file
View File

@@ -0,0 +1,169 @@
package image
import (
"fmt"
"sort"
"strconv"
"strings"
)
// TagInfo 结构体保持不变
type TagInfo struct {
Original string
Normalized string
Prefix string
Suffix string
SuffixValue int64
}
// FindTagsToDelete 是核心函数，用于找出需要删除的标签
func FindTagsToDelete(tags []string) (toDelete, toKeep []string) {
groups := make(map[string][]*TagInfo)
// --- MODIFICATION: START ---
// 提前建立保留清单，并根据新规则，无'-'的tag直接无条件保留。
// This is the new pre-processing step.
keepSet := make(map[string]bool)
for _, tag := range tags {
// 如果tag不包含'-', 视其为基础版本，直接保留
if !strings.Contains(tag, "-") {
keepSet[tag] = true
}
}
// --- MODIFICATION: END ---
// 1. 预处理和分组（与之前相同）
for _, tag := range tags {
if tag == "null" || tag == "" {
continue
}
normalizedTag := strings.TrimSuffix(tag, "-arm")
prefix := normalizedTag
suffix := ""
var suffixValue int64 = 0
if lastDashIndex := strings.LastIndex(normalizedTag, "-"); lastDashIndex != -1 {
prefix = normalizedTag[:lastDashIndex]
suffix = normalizedTag[lastDashIndex+1:]
if val, err := strconv.ParseInt(suffix, 10, 64); err == nil {
suffixValue = val
}
}
info := &TagInfo{
Original: tag,
Normalized: normalizedTag,
Prefix: prefix,
Suffix: suffix,
SuffixValue: suffixValue,
}
groups[prefix] = append(groups[prefix], info)
}
// 2. 筛选出每个分组中需要保留的标签（与之前相同）
// 注意：这里的 keepSet 是我们上面已经初始化并填充过的
for _, tagInfos := range groups {
if len(tagInfos) <= 1 {
if len(tagInfos) == 1 {
// 这一步确保了那些有'-'但自成一组的tag也被保留
keepSet[tagInfos[0].Original] = true
}
continue
}
bestTag := tagInfos[0]
for i := 1; i < len(tagInfos); i++ {
if tagInfos[i].SuffixValue > bestTag.SuffixValue {
bestTag = tagInfos[i]
}
}
keepSet[bestTag.Original] = true
}
// 3. 生成最终的删除列表和保留列表（与之前相同）
for _, tag := range tags {
if tag == "null" || tag == "" {
continue
}
if _, shouldKeep := keepSet[tag]; !shouldKeep {
toDelete = append(toDelete, tag)
} else {
toKeep = append(toKeep, tag)
}
}
sort.Strings(toDelete)
sort.Strings(toKeep)
return toDelete, toKeep
}
func main() {
tags := []string{
"6.2.0-1011", "5.7.0-szga", "6.2.0-35537-0911", "6.2.0-gz-090901-arm", "pro-6.0.11",
"6.2.0-35537-0904", "6.0.0-bjyd", "6.2.0-1792-082001-arm", "6.2.0-1792-081901-arm",
"6.2.0-gz-081509-arm", "6.2.0-gz-081506-arm", "6.2.0-gz-081501-arm", "5.6.0-ccga-0811-arm",
"6.2.0-gz-073101-arm", "6.2.0-gz-072402-arm", "6.2.0-gz-072401-arm", "6.2.0-gz-072303-arm",
"6.2.0-gz-072302-arm", "6.2.0-gz-072301-arm", "6.2.0-gz-072205-arm", "6.2.0-gz-072204-arm",
"6.2.0-gz-072203-arm", "6.2.0-gz-072202-arm", "6.2.0-gz-072201-arm", "6.2.0-gz-072105-arm",
"6.2.0-072101", "6.2.0-gz-072104-arm", "6.2.0-35926-gz-072103-arm", "6.2.0-gz-072102-arm",
"6.2.0-35537-arm-072101-arm", "6.2.0-gz-072101-arm", "6.2.0-gz-071801-arm", "5.7.0-szga-0617",
"5.7.0-szga-061301-arm", "5.7.0-szga-061201-arm", "5.7.0-szga-061101-arm",
"5.7.1-szga-061101-arm", "5.7.0-szga-061001-arm", "szga-2025052603-arm", "szga-2025052601-arm",
"szga-2025052303-arm", "szga-2025052302-arm", "szga-2025052301-arm", "szga-2025052202-arm",
"szga-2025052201-arm", "szga-2025052102-arm", "szga-2025052101-arm", "pro-6.0.10",
"sdyj-6.2.0-arm", "5.7.0-sz-spw-0423", "6.2.0-uavms", "5.7.0-szga-0421-arm", "6.0.0-041501",
"5.6.0-szns-0409", "6.2.0-lnyj", "6.2.0-031203", "2025032601", "2025032501", "null",
"6.2.0-031202", "6.2.0-35725-0311", "6.1.0-yn", "5.7.0-szga-0220-arm", "6.2.0-35537-0217",
"6.2.0-szgz-arm", "6.2.0-demo", "6.0.0-35056-011001", "6.1.0", "5.6.0-szga-1230-arm",
"6.0.0-uavms-1225", "6.1.1", "6.0.0-main1129", "6.0.0-20241204", "6.0.0-20241202",
"6.1.0-shbj-arm", "6.1.0-shbj", "5.6.0-32124-szga-1107-arm", "6.0.0-110601",
"5.6.0-32124-szga-1101-arm", "6.0.0", "5.6.0-32124-szga-1030-arm", "6.0.0-main",
"5.6.0-32124-szga-1010-arm", "5.6.0-32124-szga-092902-arm", "5.6.0-32124-szga-092901-arm",
"5.6.0-32124-szga-0929-arm", "5.6.0-32124-szga-092701-arm", "5.6.0-32124-szga-092601-arm",
"5.7.0-32124-0925-arm", "5.6.0-32124-szga-092002-arm", "5.6.0-32124-szga-0920-arm",
"5.6.0-32124-szga-091902-arm", "5.6.0-32124-szga-091901-arm", "5.6.0-32124-szga-091801-arm",
"5.6.0-32124-szga-091403-arm", "5.6.0-32124-szga-091401-arm", "5.6.0-32124-szga-091303-arm",
"5.6.0-32124-szga-091302-arm", "5.6.0-32124-szga-091301-arm", "5.6.0-32124-szga-0913-arm",
"5.6.0-32124-szga-0913", "5.6.0-szga-0912", "5.6.0-32124-szga-0911-arm", "5.7.0-30633-0905",
"5.6.0-32124-0905-arm", "5.6.0-32124-szga-090505", "5.6.0-32124-szga-090503",
"5.6.0-32124-szga-090502", "5.6.0-32124-szga-090501", "5.6.0-32124-szga-090403", "5.9.0",
"5.6.0-32124-szga-090402", "5.6.0-32124-szga-090401", "5.7.0-30633-082301", "5.8.0",
"szzsj-5.7.0-arm", "5.0.0-0723", "5.7.0", "5.6.0-szns-0710-arm", "5.7.0-0709",
"5.6.0-szns-arm", "5.6.0-0706", "5.6.0-0704", "5.5.0-arm", "5.6.0-062101", "5.6.0-amd00",
"5.6.0-amd0", "5.6.0-amdarm02", "5.6.0-amdarm", "5.5.0-integration", "5.6.0", "5.5.0",
"5.1.0-29309", "5.5.0-042801", "5.0.0-0422", "5.0.0-0418", "5.4.0-0417", "5.5.0-uat",
"5.3.0-27571-cqly", "5.4.0-27571", "5.4.0", "5.3.0-jcwg-4", "5.3.0-jcwg-3", "5.3.0-jcwg-2",
"5.3.0-jcwg-1", "5.3.0-jcwg", "5.0.0-0206", "5.3.0-jcwg-arm-0201", "5.3.0-jcwg-arm", "5.3.0",
"5.0.0", "5.2.0", "5.2.0-szzsj", "4.1.5-xlbg-25003", "5.0.0-2023SA-1116", "4.1.6-24919",
"5.0.0-24919", "5.1.0", "4.1.6", "4.1.0", "4.1.6-21941", "4.1.0-0731", "4.1.0-22191",
"4.1.0-22191-encrypt", "4.1.0-0626", "3.2.2-22202-addflyer", "4.1.5", "3.2.2-22202",
"4.1.4", "4.1.3", "4.1.2", "4.1.1", "4.1.0-encry", "4.1.0-hkyd", "4.0.4-zgyj-18768",
"3.2.2-20341-addmapexp", "3.2.2-18768", "4.0.6", "4.0.0", "4.0.5", "4.0.4", "3.2.2-18290",
"4.0.3", "4.0.2-wxyd", "4.0.2-17862", "4.0.2", "4.0.1", "3.1.0-prod", "3.1.0-hk2.0-0928",
"4.0.1-17862", "3.3.3", "3.2.2-17530", "3.3.2", "3.2.2-17000", "3.3.1", "3.3.0", "3.2.5",
"3.1.0-hk2.0-20220622", "3.1.0-hk2.0-20220620", "3.2.4", "3.2.3", "3.2.3-integration",
"2.2.2-hkgw-2022042902", "2.2.2-hkgw-2022042901", "2.2.2-hkgw-20220429", "3.2.2", "3.2.1",
"3.1.2", "3.2.0", "3.1.1", "2.1.3", "2.1.4", "2.1.5", "2.1.6", "2.1.7", "2.1.8", "2.1.9",
"2.1.10", "2.1.11", "2.1.12", "2.1.13", "2.1.14", "2.1.15", "2.1.15-UAVCLOUD-10525",
"2.1.15-PR-395", "2.2.0", "2.2.0-integration", "2.2.2-integration", "2.2.0-hkgw",
"2.2.2-hkgw", "2.2.2", "2.2.3-integration", "2.2.4-integration", "2.1.15-11851",
"2.2.4-integration-1019", "2.2.4", "3.0.0-12464", "3.0.1-integration", "3.1.0-integration",
"3.0.0-13121", "3.0.0", "3.0.1", "3.0.1-13290", "3.1.0",
}
toDelete, toKeep := FindTagsToDelete(tags)
fmt.Println("--- 🚀 Tags to Keep ---")
for _, tag := range toKeep {
fmt.Println(tag)
}
fmt.Println("\n--- 🗑️ Tags to Delete ---")
for _, tag := range toDelete {
fmt.Println(tag)
}
fmt.Printf("\nSummary: %d tags to keep, %d tags to delete.\n", len(toKeep), len(toDelete))
}

262
agent-operator/image/HarborTagParser.md Normal file
View File

@@ -0,0 +1,262 @@
针对一个[]string列表，内容为下面示例
6.2.0-1011
5.7.0-szga
6.2.0-35537-0911
6.2.0-gz-090901-arm
pro-6.0.11
6.2.0-35537-0904
6.0.0-bjyd
6.2.0-1792-082001-arm
6.2.0-1792-081901-arm
6.2.0-gz-081509-arm
6.2.0-gz-081506-arm
6.2.0-gz-081501-arm
5.6.0-ccga-0811-arm
6.2.0-gz-073101-arm
6.2.0-gz-072402-arm
6.2.0-gz-072401-arm
6.2.0-gz-072303-arm
6.2.0-gz-072302-arm
6.2.0-gz-072301-arm
6.2.0-gz-072205-arm
6.2.0-gz-072204-arm
6.2.0-gz-072203-arm
6.2.0-gz-072202-arm
6.2.0-gz-072201-arm
6.2.0-gz-072105-arm
6.2.0-072101
6.2.0-gz-072104-arm
6.2.0-35926-gz-072103-arm
6.2.0-gz-072102-arm
6.2.0-35537-arm-072101-arm
6.2.0-gz-072101-arm
6.2.0-gz-071801-arm
5.7.0-szga-0617
5.7.0-szga-061301-arm
5.7.0-szga-061201-arm
5.7.0-szga-061101-arm
5.7.1-szga-061101-arm
5.7.0-szga-061001-arm
szga-2025052603-arm
szga-2025052601-arm
szga-2025052303-arm
szga-2025052302-arm
szga-2025052301-arm
szga-2025052202-arm
szga-2025052201-arm
szga-2025052102-arm
szga-2025052101-arm
pro-6.0.10
sdyj-6.2.0-arm
5.7.0-sz-spw-0423
6.2.0-uavms
5.7.0-szga-0421-arm
6.0.0-041501
5.6.0-szns-0409
6.2.0-lnyj
6.2.0-031203
2025032601
2025032501
null
6.2.0-031202
6.2.0-35725-0311
6.1.0-yn
5.7.0-szga-0220-arm
6.2.0-35537-0217
6.2.0-szgz-arm
6.2.0-demo
6.0.0-35056-011001
6.1.0
5.6.0-szga-1230-arm
6.0.0-uavms-1225
6.1.1
6.0.0-main1129
6.0.0-20241204
6.0.0-20241202
6.1.0-shbj-arm
6.1.0-shbj
5.6.0-32124-szga-1107-arm
6.0.0-110601
5.6.0-32124-szga-1101-arm
6.0.0
5.6.0-32124-szga-1030-arm
6.0.0-main
5.6.0-32124-szga-1010-arm
5.6.0-32124-szga-092902-arm
5.6.0-32124-szga-092901-arm
5.6.0-32124-szga-0929-arm
5.6.0-32124-szga-092701-arm
5.6.0-32124-szga-092601-arm
5.7.0-32124-0925-arm
5.6.0-32124-szga-092002-arm
5.6.0-32124-szga-0920-arm
5.6.0-32124-szga-091902-arm
5.6.0-32124-szga-091901-arm
5.6.0-32124-szga-091801-arm
5.6.0-32124-szga-091403-arm
5.6.0-32124-szga-091401-arm
5.6.0-32124-szga-091303-arm
5.6.0-32124-szga-091302-arm
5.6.0-32124-szga-091301-arm
5.6.0-32124-szga-0913-arm
5.6.0-32124-szga-0913
5.6.0-szga-0912
5.6.0-32124-szga-0911-arm
5.7.0-30633-0905
5.6.0-32124-0905-arm
5.6.0-32124-szga-090505
5.6.0-32124-szga-090503
5.6.0-32124-szga-090502
5.6.0-32124-szga-090501
5.6.0-32124-szga-090403
5.9.0
5.6.0-32124-szga-090402
5.6.0-32124-szga-090401
5.7.0-30633-082301
5.8.0
szzsj-5.7.0-arm
5.0.0-0723
5.7.0
5.6.0-szns-0710-arm
5.7.0-0709
5.6.0-szns-arm
5.6.0-0706
5.6.0-0704
5.5.0-arm
5.6.0-062101
5.6.0-amd00
5.6.0-amd0
5.6.0-amdarm02
5.6.0-amdarm
5.5.0-integration
5.6.0
5.5.0
5.1.0-29309
5.5.0-042801
5.0.0-0422
5.0.0-0418
5.4.0-0417
5.5.0-uat
5.3.0-27571-cqly
5.4.0-27571
5.4.0
5.3.0-jcwg-4
5.3.0-jcwg-3
5.3.0-jcwg-2
5.3.0-jcwg-1
5.3.0-jcwg
5.0.0-0206
5.3.0-jcwg-arm-0201
5.3.0-jcwg-arm
5.3.0
5.0.0
5.2.0
5.2.0-szzsj
4.1.5-xlbg-25003
5.0.0-2023SA-1116
4.1.6-24919
5.0.0-24919
5.1.0
4.1.6
4.1.0
4.1.6-21941
4.1.0-0731
4.1.0-22191
4.1.0-22191-encrypt
4.1.0-0626
3.2.2-22202-addflyer
4.1.5
3.2.2-22202
4.1.4
4.1.3
4.1.2
4.1.1
4.1.0-encry
4.1.0-hkyd
4.0.4-zgyj-18768
3.2.2-20341-addmapexp
3.2.2-18768
4.0.6
4.0.0
4.0.5
4.0.4
3.2.2-18290
4.0.3
4.0.2-wxyd
4.0.2-17862
4.0.2
4.0.1
3.1.0-prod
3.1.0-hk2.0-0928
4.0.1-17862
3.3.3
3.2.2-17530
3.3.2
3.2.2-17000
3.3.1
3.3.0
3.2.5
3.1.0-hk2.0-20220622
3.1.0-hk2.0-20220620
3.2.4
3.2.3
3.2.3-integration
2.2.2-hkgw-2022042902
2.2.2-hkgw-2022042901
2.2.2-hkgw-20220429
3.2.2
3.2.1
3.1.2
3.2.0
3.1.1
2.1.3
2.1.4
2.1.5
2.1.6
2.1.7
2.1.8
2.1.9
2.1.10
2.1.11
2.1.12
2.1.13
2.1.14
2.1.15
2.1.15-UAVCLOUD-10525
2.1.15-PR-395
2.2.0
2.2.0-integration
2.2.2-integration
2.2.0-hkgw
2.2.2-hkgw
2.2.2
2.2.3-integration
2.2.4-integration
2.1.15-11851
2.2.4-integration-1019
2.2.4
3.0.0-12464
3.0.1-integration
3.1.0-integration
3.0.0-13121
3.0.0
3.0.1
3.0.1-13290
3.1.0
请设计一个方法，能够筛选出其中需要删除的tag，原则如下
1. 动态前缀匹配，使用-作为分隔符，组合成任意长度的前缀
1. 如3.0.1-integration 前缀为 3.0.1
2. 如3.1.0-hk2.0-20220622，前缀为3.1.0 3.1.0-hk2.0
3. 如3.0.0无-的，默认全部保存
2. 当进行过滤时候，忽略掉-arm的后缀
3. 当前缀均相同时，保留日期最新的一个
1. 如5.6.0-32124-szga-0920-arm
5.6.0-32124-szga-091902-arm
5.6.0-32124-szga-091901-arm
5.6.0-32124-szga-091801-arm 他们的共同前缀为 5.6.0-32124-szga 保留5.6.0-32124-szga-0920-arm 因为0920代表9月20日，091902代表9月19日02号
4. 当前缀相同，保留最新的一个，如5.3.0-jcwg-4
5.3.0-jcwg-3
5.3.0-jcwg-2
5.3.0-jcwg-1 他们的共同前缀为5.3.0-jcwg 保留5.3.0-jcwg-4

11
agent-operator/image/ImageOperator.go
View File

@@ -6,10 +6,6 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/klauspost/pgzip"
"io"
"io/fs"
"os"
@@ -18,6 +14,11 @@ import (
"strconv"
"strings"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/klauspost/pgzip"
image2 "wdd.io/agent-common/image"
"wdd.io/agent-common/logger"
"wdd.io/agent-common/utils"
@@ -27,7 +28,7 @@ import (
var apiClient = newClient()
var log = logger.Log
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
const OfflineImageGzipFolderPrefix = "/var/lib/docker/wdd/octopus_image/"
func newClient() *client.Client {
apiClient, err := client.NewClientWithOpts(client.FromEnv)