[agent-operator] - image bastion mode

This commit is contained in:
zeaslity
2024-04-19 16:50:56 +08:00
parent 6f9cba1270
commit c6ee9ed859
18 changed files with 609 additions and 83 deletions

View File

@@ -4,6 +4,7 @@ go 1.22.1
require (
go.uber.org/zap v1.27.0
golang.org/x/net v0.24.0
gopkg.in/yaml.v3 v3.0.1
)

View File

@@ -10,6 +10,8 @@ go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ=
go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

View File

@@ -2,8 +2,11 @@ package image
import (
"strings"
"wdd.io/agent-common/logger"
)
var log = logger.Log
var unSupported = "UN-SUPPORT-APP-NAME"
const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"
@@ -99,7 +102,8 @@ func ImageFullNameToGzipFileName(imageFullName string) (gzipFileName string) {
return gzipFileName
}
func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string {
// ImageNameToTargetImageFullName 将ImageName转换为目标TargetHosts的全名称ImageName的格式为 短名称或者长名称 均可
func ImageNameToTargetImageFullName(imageFullName, targetHostFullName string) string {
if strings.HasPrefix(imageFullName, CmiiHarborPrefix) {
imageFullName = strings.TrimPrefix(imageFullName, CmiiHarborPrefix)
@@ -108,7 +112,7 @@ func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string
}
// rancher/123:v123
if strings.HasPrefix(imageFullName, "rancher") {
return targetHost + "/" + imageFullName
return targetHostFullName + "/" + imageFullName
}
// ossr/srs:v4.0.5
if strings.Contains(imageFullName, "/") {
@@ -117,10 +121,25 @@ func ImageFullNameToTargetImageFullName(imageFullName, targetHost string) string
// srs:v4.0.5
// cmii-uav-platform:5.4.0
return targetHost + "/cmii/" + imageFullName
s := targetHostFullName + "/cmii/" + imageFullName
log.InfoF("ImageFullName: [%s] to TargetImageFullName: [%s]", imageFullName, s)
return s
}
func GzipFileNameToImageFullName(gzipFileName string) (imageFullName string) {
if !strings.HasSuffix(gzipFileName, ".tar.gz") {
log.ErrorF(" %s is not end with .tar.gz", gzipFileName)
return ""
}
gzipFileName = strings.TrimSuffix(gzipFileName, ".tar.gz")
if strings.HasPrefix(gzipFileName, "docker") {
return strings.Split(gzipFileName, "=")[1] + "/" + strings.Split(gzipFileName, "=")[2] + ":" + strings.Split(gzipFileName, "=")[3]
}
if strings.HasPrefix(gzipFileName, "cmlc=cmii=") {
return strings.Split(gzipFileName, "=")[2] + ":" + strings.Split(gzipFileName, "=")[3]
}
return gzipFileName
}

View File

@@ -0,0 +1,193 @@
package image
import (
"fmt"
"testing"
)
func TestImageFullNameToGzipFileName(t *testing.T) {
test := []string{
"bitnami/redis:6.2.6-debian-10-r0",
"simonrupf/chronyd:0.4.3",
"harbor.cdcyy.com.cn/cmii/cmii-rtsp-operator:v4.1.0",
"harbor.cdcyy.com.cn/cmii/ossrs/srs:v4.0.136",
"ossrs/srs:v4.0.136",
"mongo:5.0",
"bitnami/minio:2023.5.4",
"busybox:latest",
"busybox",
"rancher/rancher:v2.7.0",
}
for _, s := range test {
gzipFileName := ImageFullNameToGzipFileName(s)
fmt.Println(gzipFileName)
}
}
func TestGzipFileNameToImageFullName(t *testing.T) {
test := []string{
"cmlc=cmii=cmii-admin-data=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-mqtthandler=5.5.0.tar.gz",
"cmlc=cmii=cmii-admin-gateway=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-multilink=5.5.0.tar.gz",
"cmlc=cmii=cmii-admin-user=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-notice=5.5.0.tar.gz",
"cmlc=cmii=cmii-app-release=4.2.0-validation.tar.gz",
"cmlc=cmii=cmii-uav-oauth=5.5.0.tar.gz",
"cmlc=cmii=cmii-live-operator=5.2.0.tar.gz",
"cmlc=cmii=cmii-uav-platform=5.5.0.tar.gz",
"cmlc=cmii=cmii-open-gateway=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-ai-brain=5.5.0.tar.gz",
"cmlc=cmii=cmii-srs-oss-adaptor=2023-SA.tar.gz",
"cmlc=cmii=cmii-uav-platform-armypeople=5.5.0.tar.gz",
"cmlc=cmii=cmii-suav-platform-supervision=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-base=5.4.0.tar.gz",
"cmlc=cmii=cmii-suav-platform-supervisionh5=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-cms-portal=5.5.0.tar.gz",
"cmlc=cmii=cmii-suav-supervision=5.4.0-032501.tar.gz",
"cmlc=cmii=cmii-uav-platform-detection=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-airspace=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-jiangsuwenlv=4.1.3-jiangsu-0427.tar.gz",
"cmlc=cmii=cmii-uav-alarm=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-logistics=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-autowaypoint=4.2.0-beta.tar.gz",
"cmlc=cmii=cmii-uav-platform-media=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-brain=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-multiterminal=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-cloud-live=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-mws=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-cms=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-oms=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-data-post-process=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-open=5.5.0-0419.tar.gz",
"cmlc=cmii=cmii-uav-developer=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-qinghaitourism=4.1.0-21377-0508.tar.gz",
"cmlc=cmii=cmii-uav-device=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-security=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-emergency=5.3.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-securityh5=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-gateway=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-share=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-gis-server=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-platform-splice=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-grid-datasource=5.2.0-24810.tar.gz",
"cmlc=cmii=cmii-uav-platform-threedsimulation=5.2.0-21392.tar.gz",
"cmlc=cmii=cmii-uav-grid-engine=5.1.0.tar.gz",
"cmlc=cmii=cmii-uav-process=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-grid-manage=5.1.0.tar.gz",
"cmlc=cmii=cmii-uav-surveillance=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-industrial-portfolio=5.5.0-041801.tar.gz",
"cmlc=cmii=cmii-uav-threedsimulation=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-integration=5.5.0-0419.tar.gz",
"cmlc=cmii=cmii-uav-tower=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-kpi-monitor=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-user=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-logger=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-waypoint=5.5.0.tar.gz",
"cmlc=cmii=cmii-uav-material-warehouse=5.5.0.tar.gz",
"cmlc=cmii=srs=v5.0.195.tar.gz",
"cmlc=cmii=cmii-uav-mission=5.5.0.tar.gz",
}
test2 := []string{
"docker=rancher=backup-restore-operator=v1.0.3.tar.gz",
"docker=rancher=kube-api-auth = v0.1.4.tar.gz",
"docker=rancher=calico-cni=v3.17.2.tar.gz",
"docker=rancher=kubernetes-external-dns=v0.7.3.tar.gz",
"docker=rancher=calico-ctl=v3.17.2.tar.gz",
"docker=rancher=library-busybox=1.31.1.tar.gz",
"docker=rancher=calico-kube-controllers=v3.17.2.tar.gz",
"docker=rancher=library-busybox=1.32.1.tar.gz",
"docker=rancher=calico-node=v3.17.2.tar.gz",
"docker=rancher=library-nginx=1.19.2-alpine.tar.gz",
"docker=rancher=calico-pod2daemon-flexvol=v3.17.2.tar.gz",
"docker=rancher=library-traefik=1.7.19.tar.gz",
"docker=rancher=cis-operator=v1.0.3.tar.gz",
"docker=rancher=local-path-provisioner=v0.0.11.tar.gz",
"docker=rancher=cluster-proportional-autoscaler=1.7.1.tar.gz",
"docker=rancher=local-path-provisioner=v0.0.14.tar.gz",
"docker=rancher=cluster-proportional-autoscaler=1.8.1.tar.gz",
"docker=rancher=local-path-provisioner=v0.0.19.tar.gz",
"docker=rancher=configmap-reload=v0.3.0-rancher4.tar.gz",
"docker=rancher=log-aggregator=v0.1.7.tar.gz",
"docker=rancher=coredns-coredns=1.8.0.tar.gz",
"docker=rancher=metrics-server=v0.4.1.tar.gz",
"docker=rancher=coreos-etcd=v3.4.14-rancher1.tar.gz",
"docker=rancher=nginx-ingress-controller-defaultbackend=1.5-rancher1.tar.gz",
"docker=rancher=coreos-flannel=v0.13.0-rancher1.tar.gz",
"docker=rancher=nginx-ingress-controller=nginx-0.43.0-rancher1.tar.gz",
"docker=rancher=coreos-kube-state-metrics=v1.9.7.tar.gz",
"docker=rancher=opa-gatekeeper=v3.1.0-beta.7.tar.gz",
"docker=rancher=coreos-prometheus-config-reloader=v0.39.0.tar.gz",
"docker=rancher=openzipkin-zipkin=2.14.2.tar.gz",
"docker=rancher=coreos-prometheus-operator=v0.39.0.tar.gz",
"docker=rancher=pause=3.2.tar.gz",
"docker=rancher=externalip-webhook=v0.1.6.tar.gz",
"docker=rancher=plugins-docker=18.09.tar.gz",
"docker=rancher=flannel-cni=v0.3.0-rancher6.tar.gz",
"docker=rancher=prom-alertmanager=v0.21.0.tar.gz",
"docker=rancher=fleet-agent=v0.3.4.tar.gz",
"docker=rancher=prometheus-auth=v0.2.1.tar.gz",
"docker=rancher=fleet=v0.3.4.tar.gz",
"docker=rancher=prom-node-exporter=v1.0.1.tar.gz",
"docker=rancher=fluentd=v0.1.24.tar.gz",
"docker=rancher=prom-prometheus=v2.18.2.tar.gz",
"docker=rancher=grafana-grafana=7.1.5.tar.gz",
"docker=rancher=rancher-agent=v2.5.7.tar.gz",
"docker=rancher=hyperkube=v1.20.4-rancher1.tar.gz",
"docker=rancher=rancher=v2.5.7.tar.gz",
"docker=rancher=istio-kubectl=1.5.10.tar.gz",
"docker=rancher=rancher-webhook=v0.1.0-beta9.tar.gz",
"docker=rancher=jimmidyson-configmap-reload=v0.3.0.tar.gz",
"docker=rancher=rke-tools=v0.1.72.tar.gz",
"docker=rancher=k8s-dns-dnsmasq-nanny=1.15.2.tar.gz",
"docker=rancher=security-scan=v0.1.14.tar.gz",
"docker=rancher=k8s-dns-kube-dns=1.15.2.tar.gz",
"docker=rancher=security-scan=v0.2.2.tar.gz",
"docker=rancher=k8s-dns-node-cache=1.15.13.tar.gz",
"docker=rancher=shell=v0.1.6.tar.gz",
"docker=rancher=k8s-dns-sidecar=1.15.2.tar.gz",
"docker=rancher=sonobuoy-sonobuoy=v0.16.3.tar.gz",
"docker=rancher=klipper-lb=v0.1.2.tar.gz",
"docker=rancher=system-upgrade-controller=v0.6.2.tar.gz",
}
test3 := []string{
"docker=bitnami=bitnami-shell=10-debian-10-r140.tar.gz",
"docker=kubernetesui=dashboard=v2.0.1.tar.gz",
"docker=bitnami=bitnami-shell=11-debian-11-r136.tar.gz",
"docker=kubernetesui=metrics-scraper=v1.0.4.tar.gz",
"docker=bitnami=minio=2022.5.4.tar.gz",
"docker=mongo=5.0.tar.gz",
"docker=bitnami=minio=2023.5.4.tar.gz",
"docker=nacos=nacos-server=v2.1.2-slim.tar.gz",
"docker=bitnami=mysql=8.0.35-debian-11-r1.tar.gz",
"docker=nginx=1.21.3.tar.gz",
"docker=bitnami=mysql=8.1.0-debian-11-r42.tar.gz",
"docker=ossrs=srs=v4.0.136.tar.gz",
"docker=bitnami=rabbitmq=3.11.26-debian-11-r2.tar.gz",
"docker=ossrs=srs=v4.0-r3.tar.gz",
"docker=bitnami=rabbitmq=3.9.12-debian-10-r3.tar.gz",
"docker=rabbitmq=3.9-management.tar.gz",
"docker=bitnami=redis=6.2.14-debian-11-r1.tar.gz",
"docker=redis=6.0.20-alpine.tar.gz",
"docker=bitnami=redis=6.2.6-debian-10-r0.tar.gz",
"docker=simonrupf=chronyd=0.4.3.tar.gz",
"docker=busybox=latest.tar.gz",
"docker=dyrnq=nfs-subdir-external-provisioner=v4.0.2.tar.gz",
"docker=emqx=emqx=4.2.12.tar.gz",
}
test = append(test, test2...)
test = append(test, test3...)
for _, s := range test {
gzipFileName := GzipFileNameToImageFullName(s)
fmt.Println(gzipFileName)
fullName := ImageNameToTargetImageFullName(gzipFileName, "10.250.0.100:8033")
fmt.Println(fullName)
fmt.Println("--------------------")
}
}

View File

@@ -0,0 +1,112 @@
package utils
import (
"fmt"
"golang.org/x/net/proxy"
"io"
"net/http"
"net/url"
"os"
)
func DownloadFile(downloadUrl, desFile string) (downloadOk bool, resultLog []string) {
log.InfoF("Downloading file: %s", downloadUrl)
resp, err := http.Get(downloadUrl)
if err != nil {
return false, []string{fmt.Sprintf("Error downloading file: %v", err)}
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, []string{fmt.Sprintf("Unexpected status code: %d", resp.StatusCode)}
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return false, []string{fmt.Sprintf("Error reading response body: %v", err)}
}
err = os.WriteFile(desFile, body, 0644)
if err != nil {
return false, []string{fmt.Sprintf("Error writing file to disk: %v", err)}
}
log.InfoF("Downloading complete: %s", downloadUrl)
return true, []string{}
}
func DownloadFileBySocks5(downloadUrl string, socksProxyUrl string, proxyUser string, proxyPass string, desFile string) (downloadOk bool, resultLog []string) {
// 解析下载URL
_, err := url.Parse(downloadUrl)
if err != nil {
resultLog = append(resultLog, "Error parsing download URL: "+err.Error())
return false, resultLog
}
// 创建文件
out, err := os.Create(desFile)
if err != nil {
resultLog = append(resultLog, "Error creating file: "+err.Error())
return false, resultLog
}
defer out.Close()
// 创建HTTP客户端
client := &http.Client{}
// 如果提供了代理URL
if socksProxyUrl != "" {
// 解析代理URL
parsedProxyUrl, err := url.Parse(socksProxyUrl)
if err != nil {
resultLog = append(resultLog, "Error parsing proxy URL: "+err.Error())
return false, resultLog
}
// 使用SOCKS5代理
auth := &proxy.Auth{User: proxyUser, Password: proxyPass}
dialer, err := proxy.SOCKS5("tcp", parsedProxyUrl.Host, auth, proxy.Direct)
if err != nil {
resultLog = append(resultLog, "Error creating SOCKS5 dialer: "+err.Error())
return false, resultLog
}
// 设置HTTP和HTTPS代理
httpTransport := &http.Transport{
Dial: dialer.Dial,
}
client.Transport = httpTransport
}
// 发送HTTP GET请求
resp, err := client.Get(downloadUrl)
if err != nil {
resultLog = append(resultLog, "Error making GET request: "+err.Error())
return false, resultLog
}
defer resp.Body.Close()
// 检查HTTP响应状态码
if resp.StatusCode != http.StatusOK {
resultLog = append(resultLog, "Server returned HTTP status "+resp.Status)
return false, resultLog
}
// 将HTTP响应内容写入文件
_, err = io.Copy(out, resp.Body)
if err != nil {
resultLog = append(resultLog, "Error writing to file: "+err.Error())
return false, resultLog
}
// check file exists
existAndNotNull := FileExistAndNotNull(desFile)
if !existAndNotNull {
return false, []string{
"[DownloadFileBySocks5] - file not exist download error!",
}
}
return true, resultLog
}

View File

@@ -3,6 +3,7 @@ package utils
import (
"io"
"os"
"path/filepath"
)
// AppendSourceToFile 将源文件的内容添加到目标文件使用golang标准库完成跨平台、安全性更强
@@ -34,6 +35,17 @@ func AppendSourceToFile(sourceFile, targetFile string) bool {
return true
}
// AppendOverwriteContentToFile 向目标文件中写入一些内容,覆盖源文件
func AppendOverwriteContentToFile(content string, targetFile string) bool {
err := os.Remove(targetFile)
if err != nil {
log.WarnF("[BasicAppendOverwriteContentToFile] - Error removing file: %s , error is %s", targetFile, err.Error())
}
return AppendContentToFile(content, targetFile)
}
// AppendContentToFile 向目标文件中追加写入一些内容
func AppendContentToFile(content string, targetFile string) bool {
@@ -87,3 +99,64 @@ func FileExists(fileFullPath string) bool {
_, err := os.Stat(fileFullPath)
return err == nil
}
// FileExistAndNotNull 文件不为空返回true 文件为空返回false
func FileExistAndNotNull(filename string) bool {
// Check if the file exists
if _, err := os.Stat(filename); os.IsNotExist(err) {
log.DebugF("文件 %s 不存在!", filename)
return false
}
// Open the file for reading
file, err := os.Open(filename)
defer file.Close()
if err != nil {
log.DebugF("文件 %s 打开有误!", filename)
return false // Handle error according to your needs
}
// Get the file size
info, _ := file.Stat()
size := info.Size()
// Check if the file is not empty
return size > 0
}
func ListAllFileInFolder(folderName string) ([]string, error) {
files := make([]string, 0)
err := filepath.Walk(folderName, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
files = append(files, info.Name())
}
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func RemoveFolderComplete(folderName string) bool {
err := filepath.Walk(folderName,
func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
return os.Remove(path)
}
return nil
})
if err != nil {
return false
}
err = os.RemoveAll(folderName)
if err != nil {
return false
}
return true
}

View File

@@ -240,7 +240,7 @@ func (op *AgentOsOperator) pushImageToTargetHarbor(funcArgs []string) (bool, []s
targetHarborHost += ":8033"
}
targetImageFullName := image.ImageFullNameToTargetImageFullName(imageFullName, targetHarborHost)
targetImageFullName := image.ImageNameToTargetImageFullName(imageFullName, targetHarborHost)
if !PureResultSingleExecute([]string{
"docker",

View File

@@ -24,7 +24,6 @@ func main() {
// 堡垒机模式
// 初始化堡垒机模式
a_init.BastionModeInit()
flag.StringVar(&agentServerInfoConfFile, "agentServerInfoConfFile", "", "agent server info conf file")
return
}

View File

@@ -86,6 +86,7 @@ var CmiiGISAppMap = map[string]string{
"cmii-uav-grid-engine": "5.4.0",
"cmii-uav-grid-manage": "5.4.0",
}
var CmiiDevK8sConfig = `apiVersion: v1
clusters:
- cluster:

View File

@@ -36,6 +36,7 @@ const (
DefaultAccessKeyID = "cmii"
DefaultSecretAccessKey = "B#923fC7mk"
DefaultOctopusBucketName = "octopus"
DefaultOssUrlPrefix = "https://oss.demo.uavcmlc.com/cmlc-installation/tmp/"
)
func (op *CmiiMinioOperator) buildCmiiMinioOperator() {

View File

@@ -1,7 +1,9 @@
package main
import (
"bufio"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
@@ -18,15 +20,16 @@ const PublicDeployHarborHost = "42.192.52.227"
const DirectPushDeployHarborHost = "36.134.71.138"
type ImageSyncEntity struct {
ProjectName string // 优先级3
ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式
FullNameImageList []string // 优先级1
DownloadImage bool // 下载镜像
CompressImageToGzip bool // 压缩镜像
UploadToDemoMinio bool // 上传镜像
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 使用此部门
DirectHarborHost string // 目标Harbor仓库的Host全名称带端口 不带http前缀
ProjectName string // 优先级3
ProjectFromDemoNameNotNull bool
ProjectVersion string // 优先级2
CmiiNameTagList []string // 优先级1 appName:tag的形式
FullNameImageList []string // 优先级1
DownloadFromOss bool // 下载镜像
CompressImageToGzip bool // 压缩镜像
UploadToDemoMinio bool // 上传镜像
ShouldDirectPushToHarbor bool // 直接推送到对方的主机 || 离线部署机 使用此部门
DirectHarborHost string // 目标Harbor仓库的Host全名称带端口 不带http前缀
}
type ImageSyncResult struct {
@@ -76,7 +79,7 @@ func (sync ImageSyncEntity) PullFromEntityAndSyncConditionally() (imageSyncResul
allCmiiImageNameList = concatAndUniformCmiiImage(sync.FullNameImageList, sync.CmiiNameTagList)
// DCU
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUploadFromFullNameList(allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
errorPullImageList, errorGzipImageList, realCmiiImageList, allGzipFileNameList = DownloadCompressUpload(allCmiiImageNameList, sync.CompressImageToGzip, gzipFolderFullPath, sync.UploadToDemoMinio)
}
// 直接传输到目标Harbor仓库
@@ -118,7 +121,8 @@ func concatAndUniformCmiiImage(fullImageList []string, cmiiImageList []string) [
return fullImageList
}
func DownloadCompressUploadFromFullNameList(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// DownloadCompressUpload DCU 镜像同步的前半部分通常在35.71 LapPro执行无需Bastion Mode
func DownloadCompressUpload(fullNameList []string, shouldGzip bool, gzipFolderFullPath string, shouldOss bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
// Download
log.Info("DOWNLOAD START !")
@@ -190,8 +194,62 @@ func DownloadCompressUploadFromFullNameList(fullNameList []string, shouldGzip bo
return errorPullImageList, errorGzipImageList, fullNameList, allGzipFileNameList
}
func uploadGzipFileToDemoMinio() {
// DownloadLoadTagPush DLTU procedure ImageSync的另外一般流程需要支持 堡垒机(纯离线)的模式
// 1. 文件模式,从文件中读取全部的镜像全名称-写死到文件中
// 2. Gzip文件目录RKE MIDDLE CMII三个文件目录 - 是否需要约定目录
// 约定目录 /root/wdd/image/rke/ /root/wdd/image/middle/ /root/wdd/image/cmii/
// 3. 读取本机的IP地址 - 或者进行参数传递
// 4. OSS地址 - 参数传递gizpFileName
func DownloadLoadTagPush(downloadFromOss bool, ossUrlPrefix, localGzipFolder string, targetHarborFullName string) []string {
// download
if downloadFromOss {
parseAndDownloadFromOssUrlPrefix(ossUrlPrefix, localGzipFolder)
}
// load
loadAllGzipImageFromLocalFolder(localGzipFolder)
image.LoadFromFolderPath(localGzipFolder)
// tag
// push
allFileInFolder, err := utils.ListAllFileInFolder(localGzipFolder)
if err != nil {
return nil
}
for _, gzipFileName := range allFileInFolder {
log.DebugF("gzip file name is %s", gzipFileName)
imageFullName := image2.GzipFileNameToImageFullName(gzipFileName)
if imageFullName == "" {
log.ErrorF("gzip file %s to image full name error !", gzipFileName)
continue
}
targetImageFullName := image2.ImageNameToTargetImageFullName(imageFullName, targetHarborFullName)
// tag
image.TagFromSourceToTarget(imageFullName, targetImageFullName)
//push
pushResult := image.PushToOctopusKindHarbor(targetImageFullName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
fmt.Println()
fmt.Printf("%s to %s push success !", gzipFileName, targetImageFullName)
fmt.Println()
}
return nil
}
func loadAllGzipImageFromLocalFolder(localGzipFolder string) {
}
// todo
func parseAndDownloadFromOssUrlPrefix(ossUrlPrefix string, localGzipFolder string) {
// 解析
}
// DownloadCompressUploadFromDemo 获取DEMO环境的全部镜像
@@ -211,7 +269,7 @@ func DownloadCompressUploadFromDemo(projectName string, shouldGzip bool, shouldO
// do work
// DCU
return DownloadCompressUploadFromFullNameList(allCmiiImageNameListFromDemo, shouldGzip, OfflineImageGzipFolderPrefix+projectName, shouldOss)
return DownloadCompressUpload(allCmiiImageNameListFromDemo, shouldGzip, OfflineImageGzipFolderPrefix+projectName, shouldOss)
}
func buildAllCmiiImageNameListFromDemo(projectName string) []string {
@@ -267,7 +325,7 @@ func DownloadCompressUploadFromVersion(cmiiVersion string, shouldGzip bool, shou
// do work
// DCU procedure
return DownloadCompressUploadFromFullNameList(realCmiiImageName, shouldGzip, OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
return DownloadCompressUpload(realCmiiImageName, shouldGzip, OfflineImageGzipFolderPrefix+cmiiVersion, shouldOss)
}
@@ -308,7 +366,7 @@ func buildAllCmiiImageNameListFromVersion(cmiiVersion string) []string {
return realCmiiImageName
}
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool) (errorPullImageList, errorGzipImageList []string) {
func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool, downloadMiddle bool, downloadRke bool) (errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList []string) {
err := os.MkdirAll(OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
@@ -316,14 +374,22 @@ func DownloadCompressUploadDependency(shouldGzip bool, shouldOss bool) (errorPul
}
}
// pull middleware images
errorPullImageList, errorGzipImageList = image.PullFromListAndCompressSplit(image.MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
if downloadMiddle {
// pull rke images
pull, gzipImageList := image.PullFromListAndCompressSplit(image.Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
gzipFolderPrefix := OfflineImageGzipFolderPrefix + "middle/"
// result
return append(errorPullImageList, pull...), append(errorGzipImageList, gzipImageList...)
// remove folder first
utils.RemoveFolderComplete(gzipFolderPrefix)
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
}
if downloadRke {
gzipFolderPrefix := OfflineImageGzipFolderPrefix + "rke/"
return DownloadCompressUpload(image.MiddlewareAmd64, shouldGzip, gzipFolderPrefix, shouldOss)
}
return errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList
}
func LoadSplitCmiiGzipImageToTargetHarbor(projectName, targetHarborHost string) (errorLoadImageNameList, errorPushImageNameList []string) {

View File

@@ -9,29 +9,15 @@ import (
/* 拉取
*/
func TestFetchDemoImages(t *testing.T) {
//errorPullImageList, errorGzipImageList, allCmiiImageName := DownloadCompressUploadFromDemo("shls", true)
//
//utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList")
//utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList")
//utils.BeautifulPrintListWithTitle(allCmiiImageName, "cmii allCmiiImageName")
}
func TestFetchVersionImages(t *testing.T) {
//errorPullImageList, errorGzipImageList, allCmiiImageName := DownloadCompressUploadFromVersion("5.4.0", true)
//
//utils.BeautifulPrintListWithTitle(errorPullImageList, "cmii errorPullImageList")
//utils.BeautifulPrintListWithTitle(errorGzipImageList, "cmii errorGzipImageList")
//utils.BeautifulPrintListWithTitle(allCmiiImageName, "cmii allCmiiImageName")
}
func TestFetchDependencyRepos(t *testing.T) {
//errorPullImageList, errorGzipImageList := FetchDependencyRepos(true)
//
//utils.BeautifulPrintListWithTitle(errorPullImageList, "dep errorPullImageList")
//utils.BeautifulPrintListWithTitle(errorGzipImageList, "dep errorGzipImageList")
errorPullImageList, errorGzipImageList, realCmiiImageName, allGzipFileNameList := DownloadCompressUploadDependency(true, false, true, false)
utils.BeautifulPrintListWithTitle(errorPullImageList, "errorPullImageList")
utils.BeautifulPrintListWithTitle(errorGzipImageList, "errorGzipImageList")
utils.BeautifulPrintListWithTitle(realCmiiImageName, "realCmiiImageName")
utils.BeautifulPrintListWithTitle(allGzipFileNameList, "allGzipFileNameList")
}
func TestLoadSplitGzipImageToTargetHarbor(t *testing.T) {

View File

@@ -71,12 +71,12 @@ require (
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.uber.org/multierr v1.10.0 // indirect
go.uber.org/zap v1.27.0 // indirect
golang.org/x/crypto v0.16.0 // indirect
golang.org/x/crypto v0.22.0 // indirect
golang.org/x/mod v0.14.0 // indirect
golang.org/x/net v0.19.0 // indirect
golang.org/x/net v0.24.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sys v0.15.0 // indirect
golang.org/x/term v0.15.0 // indirect
golang.org/x/sys v0.19.0 // indirect
golang.org/x/term v0.19.0 // indirect
golang.org/x/text v0.14.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.16.1 // indirect

View File

@@ -269,8 +269,8 @@ golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -286,8 +286,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8=
golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -315,12 +315,12 @@ golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

View File

@@ -168,6 +168,7 @@ func TagFromSourceToTarget(sourceImageName, targetImageName string) bool {
return false
}
sourceImageName = getByName.RepoTags[0]
err := apiClient.ImageTag(context.TODO(), sourceImageName, targetImageName)
if err != nil {
log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error())
@@ -208,41 +209,47 @@ func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost s
cmiiImageFullName := imageName
if strings.HasPrefix(imageName, "cmii") {
// cmii-uav-gateway:v1.0.0 短名称的形式
cmiiImageFullName = image2.CmiiHarborPrefix + imageName
}
targetProject := "cmii"
if strings.HasPrefix(imageName, "rancher") {
// rancher/rancher:v2.5.7 rancher名称的形式
targetProject = "rancher"
}
// referenceImageList 存储的是全名称的形式
if strings.HasPrefix(imageName, image2.CmiiHarborPrefix) {
//
// harbor.cdcyy.cn/cmii/cmii-uav-gateway:v1.0.0 全名称的形式 需要拿到短名称
imageName = strings.TrimPrefix(imageName, image2.CmiiHarborPrefix)
} else {
// todo
if strings.Contains(imageName, "/") {
imageName = strings.Split(imageName, "/")[1]
// 10.250.0.10:8033/cmii/cmii-uav-gateway:v1.0.0
if strings.Contains(imageName, ":8033") {
imageName = strings.TrimLeft(imageName, ":8033")
}
}
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName
fmt.Println(cmiiImageFullName)
fmt.Println(targetImageName)
if TagFromSourceToTarget(cmiiImageFullName, targetImageName) {
pushResult := PushToOctopusKindHarbor(targetImageName)
if pushResult == nil {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName)
break
}
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
} else {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
}
//if TagFromSourceToTarget(cmiiImageFullName, targetImageName) {
// pushResult := PushToOctopusKindHarbor(targetImageName)
// if pushResult == nil {
// errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
// log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName)
// break
// }
// scanner := bufio.NewScanner(pushResult)
// for scanner.Scan() {
//
// }
// log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
//} else {
// errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
//}
}
@@ -361,6 +368,7 @@ func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string)
return errorPullImageList, errorGzipImageList
}
// LoadFromGzipFilePath 根据Gzip文件的全名称进行Load操作
func LoadFromGzipFilePath(gzipFullPath string) bool {
openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
if err != nil {
@@ -386,6 +394,7 @@ func LoadFromGzipFilePath(gzipFullPath string) bool {
}
func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
if !strings.HasSuffix(folderPath, "/") {
folderPath += "/"
}

View File

@@ -99,7 +99,8 @@ func TestImagePushToOctopusKindHarbor(t *testing.T) {
}
func TestImageLoadFromFile(t *testing.T) {
loadFromFile := LoadFromGzipFilePath("/root/octopus_image/cqga/cqga/cmlc=cmii=cmii-uav-user=5.2.0.tar.gz")
loadFromFile := LoadFromGzipFilePath("/root/cmlc=cmii=srs=v5.0.195.tar.gz")
assert.Equal(t, loadFromFile, true, "image load error !")
}
@@ -126,9 +127,11 @@ func TestImageFullNameToGzipFileName(t *testing.T) {
"busybox",
}
test = append(test, MiddlewareAmd64...)
for _, s := range test {
gzipFileName := image.ImageFullNameToGzipFileName(s)
fmt.Printf(" %s to %s \n", s, gzipFileName)
fmt.Println(gzipFileName)
}
}

View File

@@ -1,7 +1,11 @@
package main
import (
"bufio"
"flag"
"fmt"
"os"
"strings"
"wdd.io/agent-common/utils"
"wdd.io/agent-operator/real_project/bgtg"
)
@@ -71,7 +75,64 @@ func CmiiRunner() {
}
var DLTUHelp = `
DLTUHelp
dltu [ossUrl] [localGzipFolder] [harborHostFullName]
`
func main() {
RealProjectRunner()
//RealProjectRunner()
// 解析命令行参数
var mode string
flag.StringVar(&mode, "mode", "mode", "agent run mode")
flag.Parse()
if mode == "image" {
// 堡垒机模式 的 镜像美容
// 初始化堡垒机模式
for {
reader := bufio.NewReader(os.Stdin)
for {
fmt.Printf(DLTUHelp)
fmt.Print("enter ==> ")
text, _ := reader.ReadString('\n')
split := strings.Split(text, " ")
var result []string
for _, s := range split {
text = strings.TrimSpace(s)
if text == "" {
continue
}
result = append(result, text)
}
if len(result) != 4 {
fmt.Println("input error!")
fmt.Printf(DLTUHelp)
return
}
ossUrl := result[1]
localGzipFolder := result[2]
harborHostFullName := result[3]
fmt.Println("ossUrl: ", ossUrl)
fmt.Println("localGzipFolder: ", localGzipFolder)
fmt.Println("harborHostFullName: ", harborHostFullName)
fmt.Println()
DownloadLoadTagPush(false, ossUrl, localGzipFolder, harborHostFullName)
// 下载
fmt.Println()
}
}
}
}

View File

@@ -38,13 +38,13 @@ public class TestImageSyncScheduler {
public void runImageSync() {
ArrayList<String> CmiiAppNameList = new ArrayList<>(List.of(
// "cmii-uav-platform-armypeople:5.4.0-041201"
"cmii-uav-process:5.4.0-041901"
));
ArrayList<String> ImageFullNameList = new ArrayList<>(List.of(
// "harbor.cdcyy.com.cn/cmii/cmii-live-operator:5.2.0",
// "harbor.cdcyy.com.cn/cmii/cmii/srs:v5.0.195"
"harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.54.0-cqly-041601"
// "harbor.cdcyy.com.cn/cmii/cmii-uav-industrial-portfolio:5.54.0-cqly-041601"
));
Boolean downloadAndCompressOnly = false;