Files
ProjectOctopus/agent-operator/image/ImageOperator.go
2024-12-02 18:04:13 +08:00

660 lines
19 KiB
Go

package image
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/klauspost/pgzip"
"io"
"io/fs"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
image2 "wdd.io/agent-common/image"
"wdd.io/agent-common/logger"
"wdd.io/agent-common/utils"
"wdd.io/agent-deploy/d_app"
)
var apiClient = newClient()
var log = logger.Log
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
func newClient() *client.Client {
apiClient, err := client.NewClientWithOpts(client.FromEnv)
if err != nil {
log.ErrorF("[newClient] - new client failed ! => %s", err.Error())
panic(err)
}
defer apiClient.Close()
return apiClient
}
func GetRunningContainer() []types.Container {
return getContainerList(false)
}
func GetAllContainer() []types.Container {
return getContainerList(true)
}
func getContainerList(all bool) []types.Container {
containers, err := apiClient.ContainerList(context.Background(), types.ContainerListOptions{
Quiet: false,
Size: false,
All: all,
Latest: false,
Limit: 0,
Filters: filters.Args{},
})
if err != nil {
log.ErrorF("[getContainerList] - error => %s", err.Error())
}
return containers
}
//func SetDockerProxy(httpProxy string) {
// proxy_all := "HTTP_PROXY=" + httpProxy + " HTTPS_PROXY=" + httpProxy
//
// log.InfoF("[SetDockerProxy] - set docker proxy to %s", proxy_all)
//
// // Set the HTTP proxy for the Docker daemon
// configFile, err := apiClient.ConfigCreate(context.Background(), swarm.ConfigSpec{
// Annotations: swarm.Annotations{
// Name: "proxy.conf",
// },
// Data: []byte(proxy_all),
// })
// if err != nil {
// // 处理错误
// log.ErrorF("[SetDockerProxy] - set docker proxy error %s", err.Error())
// }
//
//
//
//
//}
func GetAll() []types.ImageSummary {
list, err := apiClient.ImageList(context.TODO(), types.ImageListOptions{
All: true,
Filters: filters.Args{},
})
if err != nil {
log.ErrorF("[ImageGetAll] --error => %s", err.Error())
}
return list
}
func GetByName(imageName string) *types.ImageSummary {
imageGetAll := GetAll()
for _, imageSummary := range imageGetAll {
for _, tag := range imageSummary.RepoTags {
if strings.Contains(tag, imageName) {
return &imageSummary
}
}
}
return nil
}
func Delete(imageName string) []types.ImageDeleteResponseItem {
imageGetByName := GetByName(imageName)
if imageGetByName == nil {
log.ErrorF("[ImageDelete] -- image not exists ! %s", imageGetByName.RepoTags)
return nil
}
remove, err := apiClient.ImageRemove(context.TODO(), imageGetByName.ID, types.ImageRemoveOptions{
Force: true,
PruneChildren: false,
})
if err != nil {
log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error())
return nil
}
return remove
}
func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
imageGetAll := GetAll()
// ip:8033
//re := regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}:\d{1,4}`)
for _, imageSummary := range imageGetAll {
for _, repoTag := range imageSummary.RepoTags {
if strings.HasPrefix(repoTag, image2.CmiiHarborPrefix) || strings.HasPrefix(repoTag, "harbor.wdd.io") || strings.Contains(repoTag, ":8033") {
for _, tag := range imageSummary.RepoTags {
_, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{
Force: true,
PruneChildren: true,
})
if err != nil {
log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error())
errorRemoveImageNameList = append(errorRemoveImageNameList, tag)
}
log.InfoF("[ImageDelete] - image remove of [%s] success!", tag)
}
}
}
}
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
return errorRemoveImageNameList
}
func TagFromSourceToTarget(sourceImageName, targetImageName string) bool {
getByName := GetByName(sourceImageName)
if getByName == nil {
log.ErrorF("[ImageTagFromSourceToTarget] - %s not exits !", sourceImageName)
return false
}
sourceImageName = getByName.RepoTags[0]
err := apiClient.ImageTag(context.TODO(), sourceImageName, targetImageName)
if err != nil {
log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error())
}
log.InfoF("[ImageTagFromSourceToTarget] - from %s to %s success!", sourceImageName, targetImageName)
return true
}
func UploadToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) {
if GetByName(targetImageName) == nil {
log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName)
return pushResult
}
pushResult, err := apiClient.ImagePush(context.TODO(), targetImageName, types.ImagePushOptions{
All: false,
//RegistryAuth: "eyAidXNlcm5hbWUiOiAiYWRtaW4iLCAicGFzc3dvcmQiOiAiVjJyeVN0ckBuZ1BzcyIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
// 重庆移动二级平台
RegistryAuth: "eyAidXNlcm5hbWUiOiAiemd5ZHR4anRjcXl4Z3MxODg4MzI1NzMxMSIsICJwYXNzd29yZCI6ICJEaWN0QDIwMjQiLCAiZW1haWwiOiAiaWNlQHFxLmNvbSIgfQ==",
PrivilegeFunc: nil,
Platform: "amd64",
})
if err != nil {
log.ErrorF("[ImagePushToOctopusKindHarbor] - push %s error %s", targetImageName, err.Error())
return nil
}
return pushResult
}
func UploadToHarbor(targetImageName string) (uploadOK bool) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
defer pushResult.Close()
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
}
fmt.Println()
log.InfoF("[UploadToHarbor] - upload %s success!", targetImageName)
fmt.Println()
return true
}
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
targetHarborHost = strings.TrimPrefix(targetHarborHost, "http://")
targetHarborHost = strings.TrimPrefix(targetHarborHost, "https://")
for _, imageName := range referenceImageList {
// check image
// harbor.cdcyy.cn
cmiiImageFullName := imageName
// convert image name
targetImageName := image2.ImageNameToTargetImageFullName(imageName, targetHarborHost)
if TagFromSourceToTarget(cmiiImageFullName, targetImageName) {
pushResult := UploadToOctopusKindHarbor(targetImageName)
if pushResult == nil {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName)
break
}
scanner := bufio.NewScanner(pushResult)
for scanner.Scan() {
text := scanner.Text()
if strings.Contains(text, "digest: sha256:") {
fmt.Println(scanner.Text())
}
}
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
fmt.Println()
} else {
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
}
}
return errorPushImageNameList
}
func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
pullResult, err := apiClient.ImagePull(context.TODO(), imageName, types.ImagePullOptions{
All: false,
RegistryAuth: "eyAidXNlcm5hbWUiOiAicmFkMDJfZHJvbmUiLCAicGFzc3dvcmQiOiAiRHJvbmVAMTIzNCIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
PrivilegeFunc: func() (string, error) {
return "authorization: Basic cmFkMDJfZHJvbmU6RHJvbmVAMTIzNA==", nil
},
Platform: "amd64",
})
if err != nil {
log.ErrorF("[ImagePullFromCmiiHarbor]- image pull of %s ,error => %s", imageName, err.Error())
return nil
}
return pullResult
}
func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
fullImageNameList = CmiiImageMapToFullNameList(imageVersionMap)
return fullImageNameList, PullFromFullNameList(fullImageNameList)
}
func PullCmiiFromFileJson(filePathName string) {
readFile, err := os.ReadFile(filePathName)
if err != nil {
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s read error ! %s", filePathName, err.Error())
return
}
var resultMap map[string]string
err = json.Unmarshal(readFile, &readFile)
if err != nil {
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
return
}
for image, tag := range resultMap {
pullResult := PullFromCmiiHarbor(image + ":" + tag)
if pullResult == nil {
continue
}
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
fmt.Println()
}
}
// PullFromFullNameList 根据镜像名列表拉取全部的镜像
func PullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
start := time.Now()
for _, fullImageName := range fullImageNameList {
if !strings.HasPrefix(fullImageName, "harbor.cdcyy.com.cn") {
since := time.Since(start)
if since < 60*time.Second {
duration := 60*time.Second - since
log.DebugF("PullFromFullNameList - wait for %s !", duration.String())
time.Sleep(duration)
start = time.Now()
}
}
log.DebugF("start to pull => [%s]", fullImageName)
pullResult := PullFromCmiiHarbor(fullImageName)
if pullResult == nil {
errorPullImageList = append(errorPullImageList, fullImageName)
continue
}
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
//if strings.Contains(line, "\"status\":\"Pulling from") {
// fmt.Println(line)
//}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
fmt.Println()
}
return errorPullImageList
}
func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
errorPullImageList = PullFromFullNameList(fullImageNameList)
// generate a project folder
err := os.MkdirAll(gzipFolder, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error())
}
}
var tarGzipFileNameList []string
for _, image := range fullImageNameList {
ok, path := SaveToGzipFile(image, gzipFolder)
if !ok {
errorGzipImageList = append(errorGzipImageList, image)
continue
}
tarGzipFileNameList = append(tarGzipFileNameList, path)
}
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
return errorPullImageList, errorGzipImageList
}
// LoadFromGzipFilePath 根据Gzip文件的全名称进行Load操作
func LoadFromGzipFilePath(gzipFileNameFullPath string) bool {
openFile, err := os.OpenFile(gzipFileNameFullPath, 0, fs.ModePerm)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFileNameFullPath, err.Error())
return false
}
loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true)
if err != nil {
log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFileNameFullPath, err.Error())
return false
}
log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFileNameFullPath, strconv.FormatBool(loadResponse.JSON))
scanner := bufio.NewScanner(loadResponse.Body)
for scanner.Scan() {
line := scanner.Text()
fmt.Println(line)
}
return true
}
func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
if !strings.HasSuffix(folderPath, "/") {
folderPath += "/"
}
dirEntries, err := os.ReadDir(folderPath)
if err != nil {
log.ErrorF("[ImageLoadFromFolderPath] - error read folder %s error is %s", folderPath, err.Error())
return
}
// load gzip file
for _, dirEntry := range dirEntries {
if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
if !LoadFromGzipFilePath(folderPath + dirEntry.Name()) {
errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
}
}
}
return errorLoadImageNameList
}
// SaveToGzipFile 根据目标镜像的全名称 将镜像压缩为Gzip文件
func SaveToGzipFile(imageFullName, folderPathPrefix string) (gzipOK bool, gzipImageFileFullPath string) {
imageGetByName := GetByName(imageFullName)
if imageGetByName == nil {
log.ErrorF("[ImageSaveToTarGZ] - %s not exits", imageFullName)
return false, ""
}
imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error())
return false, ""
}
var realImageTag string
for _, repoTag := range imageGetByName.RepoTags {
if !strings.Contains(repoTag, "8033") {
realImageTag = repoTag
break
}
}
gzipImageFileFullPath = image2.ImageFullNameToGzipFileName(realImageTag)
if err := os.MkdirAll(filepath.Dir(gzipImageFileFullPath), os.ModePerm); err != nil {
log.ErrorF("[ImageSaveToTarGZ] - failed to create directory: %s", err)
return false, ""
}
// 生成gzip压缩文件的全路径名称
gzipImageFileFullPath = filepath.Join(folderPathPrefix, gzipImageFileFullPath)
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFileFullPath)
// 删除掉旧的Gzip文件
if err := os.Remove(gzipImageFileFullPath); err != nil && !os.IsNotExist(err) {
log.ErrorF("[ImageSaveToTarGZ] - failed to remove old gzip file: %s", err)
return false, ""
}
// 创建
tarFile, err := os.Create(gzipImageFileFullPath)
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFileFullPath, err.Error())
return false, ""
}
defer tarFile.Close()
// 创建pgzip writer
gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression) // 你可以调整压缩级别
if err != nil {
log.ErrorF("[ImageSaveToTarGZ] - pgzip create writer error: %s", err.Error())
}
defer gw.Close()
// Copy the tar archive to the gzip writer.
if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error())
return false, ""
}
// 成功
return true, gzipImageFileFullPath
}
// SaveImageListToGzipFile 将一个列表内的镜像全部压缩为一个tar.gz文件
func SaveImageListToGzipFile(imageFullNames []string, folderPathPrefix string, outputFileName string) (gzipOK bool, gzipFileFullPath string, errorGzipImageList []string) {
//if len(imageFullNames) == 0 {
// log.Error("[SaveImagesToGzipFile] - no images provided")
// return false, "", errorGzipImageList
//}
//
//// 确保输出文件路径
//gzipFileFullPath = folderPathPrefix + outputFileName
//if err := os.MkdirAll(filepath.Dir(gzipFileFullPath), os.ModePerm); err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - failed to create directory: %s", err)
// return false, "", errorGzipImageList
//}
//
//log.InfoF("[SaveImagesToGzipFile] - start saving images to [%s]", gzipFileFullPath)
//
//// 删除旧的Gzip文件
//if err := os.Remove(gzipFileFullPath); err != nil && !os.IsNotExist(err) {
// log.ErrorF("[SaveImagesToGzipFile] - failed to remove old gzip file: %s", err)
// return false, "", errorGzipImageList
//}
//
//tarFile, err := os.Create(gzipFileFullPath)
//if err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - error creating gzip file: %s", err)
// return false, "", errorGzipImageList
//}
//defer tarFile.Close()
//
//gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression)
//if err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - pgzip writer creation error: %s", err)
// return false, "", errorGzipImageList
//}
//defer gw.Close()
//
//for _, imageFullName := range imageFullNames {
// imageGetByName := GetByName(imageFullName)
// if imageGetByName == nil {
// log.WarnF("[SaveImagesToGzipFile] - %s not exists, skipping", imageFullName)
// continue
// }
//
// imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
// if err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - image save error for %s: %s", imageFullName, err)
// continue
// }
//
// if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - failed to copy tar archive for %s to gzip writer: %s", imageFullName, err)
// continue
// }
//
//}
//
//if err := gw.Close(); err != nil {
// log.ErrorF("[SaveImagesToGzipFile] - error closing gzip writer: %s", err)
// return false, "", errorGzipImageList
//}
//
//log.InfoF("[SaveImagesToGzipFile] - successfully saved images to [%s]", gzipFileFullPath)
return true, gzipFileFullPath, errorGzipImageList
}
func CmiiImageMapToFullNameList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
for image, tag := range cmiiImageVersionMap {
s := image2.CmiiHarborPrefix + image + ":" + tag
fullImageNameList = append(fullImageNameList, s)
}
return fullImageNameList
}
func CmiiImageMapFromGzipFolder(gzipFileFolder string) (cmiiImageVersionMap map[string]string) {
allFileInFolder, err := utils.ListAllFileInFolder(gzipFileFolder)
if err != nil {
return nil
}
cmiiImageVersionMap = make(map[string]string)
for _, gzipFileName := range allFileInFolder {
log.DebugF("gzip file name is %s", gzipFileName)
imageName, imageTag := image2.GzipFileNameToImageNameAndTag(gzipFileName)
cmiiImageVersionMap[imageName] = imageTag
}
return cmiiImageVersionMap
}
// GenerateCmiiTagVersionImageMap 生成特定版本的ImageTagMap
func GenerateCmiiTagVersionImageMap(specificTag string) (backendMap, frontendMap, srsMap map[string]string) {
matched, _ := regexp.MatchString(`^\d+\.\d+\.\d+$`, specificTag)
if !matched {
sprintf := fmt.Sprintf("tag is not match ! [%s]", specificTag)
log.Error(sprintf)
panic(sprintf)
}
backendMap = make(map[string]string, len(d_app.CmiiBackendAppMap))
frontendMap = make(map[string]string, len(d_app.CmiiFrontendAppMap))
srsMap = make(map[string]string, len(d_app.CmiiSrsAppMap))
for imageName := range d_app.CmiiBackendAppMap {
backendMap[imageName] = specificTag
}
for imageName := range d_app.CmiiFrontendAppMap {
frontendMap[imageName] = specificTag
}
for imageName, imageTag := range d_app.CmiiSrsAppMap {
srsMap[imageName] = imageTag
}
return backendMap, frontendMap, srsMap
}
func loginToDockerHub(HarborFullHost string) {
if HarborFullHost == "" {
HarborFullHost = "https://registry-1.docker.io"
}
login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{
Username: "icederce",
Password: "loveff.cxc.23",
Auth: "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==",
ServerAddress: HarborFullHost,
})
if err != nil {
log.ErrorF("[loginToDockerHub] - login to %s failed !", HarborFullHost)
}
log.DebugF("[loginToDockerHub] - login is %s", login.Status)
}
func WriteDependencyImageToFile() {
imageFilePrefix := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\image\\"
middleFile := imageFilePrefix + "middle-image.txt"
_ = os.Remove(middleFile)
for _, image := range d_app.MiddlewareAmd64 {
utils.AppendContentToFile(image+"\n", middleFile)
}
rkeFile := imageFilePrefix + "rke-image.txt"
_ = os.Remove(rkeFile)
for _, image := range d_app.Rancher1204Amd64 {
utils.AppendContentToFile(image+"\n", rkeFile)
}
}