545 lines
14 KiB
Go
545 lines
14 KiB
Go
package image
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"github.com/docker/docker/api/types"
|
|
"github.com/docker/docker/api/types/filters"
|
|
"github.com/docker/docker/client"
|
|
"github.com/klauspost/pgzip"
|
|
"io"
|
|
"io/fs"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"wdd.io/agent-go/executor"
|
|
"wdd.io/agent-go/logger"
|
|
"wdd.io/agent-go/utils"
|
|
)
|
|
|
|
var apiClient = newClient()
|
|
var log = logger.Log
|
|
|
|
const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"
|
|
|
|
func newClient() *client.Client {
|
|
apiClient, err := client.NewClientWithOpts(client.FromEnv)
|
|
|
|
if err != nil {
|
|
log.ErrorF("[newClient] - new client failed ! => %s", err.Error())
|
|
panic(err)
|
|
}
|
|
defer apiClient.Close()
|
|
|
|
return apiClient
|
|
}
|
|
|
|
func GetRunningContainer() []types.Container {
|
|
return getContainerList(false)
|
|
}
|
|
|
|
func GetAllContainer() []types.Container {
|
|
return getContainerList(true)
|
|
}
|
|
|
|
func getContainerList(all bool) []types.Container {
|
|
|
|
containers, err := apiClient.ContainerList(context.Background(), types.ContainerListOptions{
|
|
Quiet: false,
|
|
Size: false,
|
|
All: all,
|
|
Latest: false,
|
|
Limit: 0,
|
|
Filters: filters.Args{},
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[getContainerList] - error => %s", err.Error())
|
|
}
|
|
|
|
return containers
|
|
}
|
|
|
|
func GetAll() []types.ImageSummary {
|
|
|
|
list, err := apiClient.ImageList(context.TODO(), types.ImageListOptions{
|
|
All: true,
|
|
Filters: filters.Args{},
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[ImageGetAll] --error => %s", err.Error())
|
|
}
|
|
|
|
return list
|
|
}
|
|
|
|
func GetByName(imageName string) *types.ImageSummary {
|
|
|
|
imageGetAll := GetAll()
|
|
|
|
for _, imageSummary := range imageGetAll {
|
|
for _, tag := range imageSummary.RepoTags {
|
|
if strings.Contains(tag, imageName) {
|
|
return &imageSummary
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func Delete(imageName string) []types.ImageDeleteResponseItem {
|
|
imageGetByName := GetByName(imageName)
|
|
if imageGetByName == nil {
|
|
log.ErrorF("[ImageDelete] -- image not exists ! %s", imageGetByName.RepoTags)
|
|
return nil
|
|
}
|
|
|
|
remove, err := apiClient.ImageRemove(context.TODO(), imageGetByName.ID, types.ImageRemoveOptions{
|
|
Force: true,
|
|
PruneChildren: false,
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error())
|
|
return nil
|
|
}
|
|
|
|
return remove
|
|
}
|
|
|
|
func PruneAllCmiiImages() (errorRemoveImageNameList []string) {
|
|
|
|
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
|
|
|
|
imageGetAll := GetAll()
|
|
|
|
// ip:8033
|
|
//re := regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}:\d{1,4}`)
|
|
|
|
for _, imageSummary := range imageGetAll {
|
|
for _, repoTag := range imageSummary.RepoTags {
|
|
if strings.HasPrefix(repoTag, CmiiHarborPrefix) {
|
|
for _, tag := range imageSummary.RepoTags {
|
|
_, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{
|
|
Force: true,
|
|
PruneChildren: true,
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error())
|
|
errorRemoveImageNameList = append(errorRemoveImageNameList, tag)
|
|
}
|
|
log.InfoF("[ImageDelete] - image remove of [%s] success!", tag)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
_, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{})
|
|
|
|
return errorRemoveImageNameList
|
|
}
|
|
|
|
func TagFromSourceToTarget(sourceImageName, targetImageName string) bool {
|
|
|
|
getByName := GetByName(sourceImageName)
|
|
if getByName == nil {
|
|
log.ErrorF("[ImageTagFromSourceToTarget] - %s not exits !", sourceImageName)
|
|
return false
|
|
}
|
|
|
|
err := apiClient.ImageTag(context.TODO(), sourceImageName, targetImageName)
|
|
if err != nil {
|
|
log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error())
|
|
}
|
|
|
|
log.InfoF("[ImageTagFromSourceToTarget] - from %s to %s success!", sourceImageName, targetImageName)
|
|
return true
|
|
}
|
|
|
|
func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) {
|
|
|
|
if GetByName(targetImageName) == nil {
|
|
log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName)
|
|
return pushResult
|
|
}
|
|
|
|
pushResult, err := apiClient.ImagePush(context.TODO(), targetImageName, types.ImagePushOptions{
|
|
All: false,
|
|
RegistryAuth: "eyAidXNlcm5hbWUiOiAiYWRtaW4iLCAicGFzc3dvcmQiOiAiVjJyeVN0ckBuZ1BzcyIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
|
|
PrivilegeFunc: nil,
|
|
Platform: "amd64",
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[ImagePushToOctopusKindHarbor] - push %s error %s", targetImageName, err.Error())
|
|
return nil
|
|
}
|
|
|
|
return pushResult
|
|
}
|
|
|
|
// TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest
|
|
func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {
|
|
|
|
for _, imageName := range referenceImageList {
|
|
|
|
// check image
|
|
// harbor.cdcyy.cn
|
|
cmiiImageFullName := imageName
|
|
|
|
if strings.HasPrefix(imageName, "cmii") {
|
|
cmiiImageFullName = CmiiHarborPrefix + imageName
|
|
}
|
|
|
|
targetProject := "cmii"
|
|
if strings.HasPrefix(imageName, "rancher") {
|
|
targetProject = "rancher"
|
|
}
|
|
|
|
if strings.Contains(imageName, "/") {
|
|
//
|
|
imageName = strings.Split(imageName, "/")[1]
|
|
} else {
|
|
|
|
}
|
|
|
|
targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName
|
|
|
|
if TagFromSourceToTarget(cmiiImageFullName, targetImageName) {
|
|
pushResult := PushToOctopusKindHarbor(targetImageName)
|
|
if pushResult == nil {
|
|
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
|
|
continue
|
|
}
|
|
scanner := bufio.NewScanner(pushResult)
|
|
for scanner.Scan() {
|
|
|
|
}
|
|
log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
|
|
} else {
|
|
errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
|
|
}
|
|
|
|
}
|
|
|
|
return errorPushImageNameList
|
|
}
|
|
|
|
func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
|
|
pullResult, err := apiClient.ImagePull(context.TODO(), imageName, types.ImagePullOptions{
|
|
All: false,
|
|
RegistryAuth: "eyAidXNlcm5hbWUiOiAicmFkMDJfZHJvbmUiLCAicGFzc3dvcmQiOiAiRHJvbmVAMTIzNCIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
|
|
PrivilegeFunc: func() (string, error) {
|
|
return "authorization: Basic cmFkMDJfZHJvbmU6RHJvbmVAMTIzNA==", nil
|
|
},
|
|
Platform: "amd64",
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[ImagePullFromCmiiHarbor]- image pull of %s ,error => %s", imageName, err.Error())
|
|
return nil
|
|
}
|
|
|
|
return pullResult
|
|
}
|
|
|
|
func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) {
|
|
|
|
fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap)
|
|
return fullImageNameList, PullFromFullNameList(fullImageNameList)
|
|
|
|
}
|
|
|
|
func PullCmiiFromFileJson(filePathName string) {
|
|
|
|
readFile, err := os.ReadFile(filePathName)
|
|
if err != nil {
|
|
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s read error ! %s", filePathName, err.Error())
|
|
return
|
|
}
|
|
|
|
var resultMap map[string]string
|
|
err = json.Unmarshal(readFile, &readFile)
|
|
if err != nil {
|
|
log.ErrorF("[ImagePullCMiiFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
|
|
return
|
|
}
|
|
|
|
for image, tag := range resultMap {
|
|
pullResult := PullFromCmiiHarbor(image + ":" + tag)
|
|
if pullResult == nil {
|
|
continue
|
|
}
|
|
scanner := bufio.NewScanner(pullResult)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if strings.Contains(line, "\"status\":\"Pulling from") {
|
|
fmt.Println(line)
|
|
}
|
|
if strings.Contains(line, "Status: Image is up to date for") {
|
|
fmt.Println(line)
|
|
}
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
}
|
|
|
|
// PullFromFullNameList 根据镜像名列表拉取全部的镜像
|
|
func PullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {
|
|
|
|
for _, dep := range fullImageNameList {
|
|
|
|
pullResult := PullFromCmiiHarbor(dep)
|
|
if pullResult == nil {
|
|
errorPullImageList = append(errorPullImageList, dep)
|
|
continue
|
|
}
|
|
scanner := bufio.NewScanner(pullResult)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
if strings.Contains(line, "\"status\":\"Pulling from") {
|
|
fmt.Println(line)
|
|
}
|
|
if strings.Contains(line, "Status: Image is up to date for") {
|
|
fmt.Println(line)
|
|
}
|
|
}
|
|
fmt.Println()
|
|
}
|
|
|
|
return errorPullImageList
|
|
}
|
|
|
|
func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {
|
|
|
|
errorPullImageList = PullFromFullNameList(fullImageNameList)
|
|
|
|
// generate a project folder
|
|
err := os.MkdirAll(gzipFolder, os.ModeDir)
|
|
if err != nil {
|
|
if !errors.Is(err, os.ErrExist) {
|
|
log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error())
|
|
}
|
|
}
|
|
|
|
var tarGzipFileNameList []string
|
|
|
|
for _, image := range fullImageNameList {
|
|
|
|
if !SaveToTarGZ(image, gzipFolder) {
|
|
errorGzipImageList = append(errorGzipImageList, image)
|
|
continue
|
|
}
|
|
tarGzipFileNameList = append(tarGzipFileNameList, convertImageGzipFileName(image))
|
|
}
|
|
|
|
utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list")
|
|
return errorPullImageList, errorGzipImageList
|
|
}
|
|
|
|
func LoadFromGzipFilePath(gzipFullPath string) bool {
|
|
openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
|
|
if err != nil {
|
|
log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error())
|
|
return false
|
|
}
|
|
|
|
loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true)
|
|
if err != nil {
|
|
log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFullPath, err.Error())
|
|
return false
|
|
}
|
|
|
|
log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFullPath, strconv.FormatBool(loadResponse.JSON))
|
|
|
|
scanner := bufio.NewScanner(loadResponse.Body)
|
|
for scanner.Scan() {
|
|
line := scanner.Text()
|
|
fmt.Println(line)
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
|
|
if !strings.HasSuffix(folderPath, "/") {
|
|
folderPath += "/"
|
|
}
|
|
|
|
dirEntries, err := os.ReadDir(folderPath)
|
|
if err != nil {
|
|
log.ErrorF("[ImageLoadFromFolderPath] - error read folder %s error is %s", folderPath, err.Error())
|
|
return
|
|
}
|
|
|
|
// load gzip file
|
|
for _, dirEntry := range dirEntries {
|
|
if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
|
|
if !LoadFromGzipFilePath(folderPath + dirEntry.Name()) {
|
|
errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
|
|
}
|
|
}
|
|
}
|
|
|
|
return errorLoadImageNameList
|
|
}
|
|
|
|
func SaveToTarGZ(targetImageName, folderPathPrefix string) bool {
|
|
|
|
imageGetByName := GetByName(targetImageName)
|
|
if imageGetByName == nil {
|
|
log.ErrorF("[ImageSaveToTarGZ] - %s not exits", targetImageName)
|
|
return false
|
|
}
|
|
|
|
imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
|
|
if err != nil {
|
|
log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error())
|
|
return false
|
|
}
|
|
|
|
var realImageTag string
|
|
for _, repoTag := range imageGetByName.RepoTags {
|
|
if !strings.Contains(repoTag, "8033") {
|
|
realImageTag = repoTag
|
|
break
|
|
}
|
|
}
|
|
|
|
gzipImageFile := convertImageGzipFileName(realImageTag)
|
|
if !strings.HasSuffix(folderPathPrefix, "/") {
|
|
folderPathPrefix += "/"
|
|
}
|
|
_ = os.MkdirAll(folderPathPrefix, os.ModeDir)
|
|
|
|
gzipImageFile = folderPathPrefix + gzipImageFile
|
|
log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFile)
|
|
_ = os.Remove(gzipImageFile)
|
|
tarFile, err := os.Create(gzipImageFile)
|
|
if err != nil {
|
|
log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFile, err.Error())
|
|
return false
|
|
}
|
|
defer tarFile.Close()
|
|
|
|
// 创建pgzip writer
|
|
gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression) // 你可以调整压缩级别
|
|
if err != nil {
|
|
log.ErrorF("[ImageSaveToTarGZ] - pgzip create writer error: %s", err.Error())
|
|
}
|
|
defer gw.Close()
|
|
|
|
// Copy the tar archive to the gzip writer.
|
|
if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
|
|
log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error())
|
|
return false
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// convertImageGzipFileName 必须输出长度为4的内容 =出现得次数为3
|
|
func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
|
|
|
|
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmlc=cmii=cmii-uav-platform=5.4.0.tar.gz
|
|
// rancher/fleet:v0.3.4
|
|
|
|
// ossr/srs:v5.0.1 ==> docker=cmii=srs=v5.0.1.tar.gz
|
|
// nginx:latest
|
|
// bitnami/minio:2022.5.4
|
|
// simonrupf/chronyd:0.4.3
|
|
|
|
// 10.1.1.1:8033/cmii/ok:1.2 不支持 不允许存在
|
|
|
|
split := strings.Split(imageRepoTag, ":")
|
|
//log.DebugF(" %s to %s", imageRepoTag, split)
|
|
if len(split) == 1 {
|
|
// nginx
|
|
return "docker=library=" + imageRepoTag + "=latest.tar.gz"
|
|
}
|
|
|
|
first := strings.Split(split[0], "/")
|
|
//log.DebugF(" split[0] %s to %s", split[0], first)
|
|
if len(first) == 3 {
|
|
// harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0
|
|
// docker.io/ossr/srs:v5.0.1
|
|
if strings.HasPrefix(split[0], CmiiHarborPrefix) {
|
|
gzipFileName += "cmlc=cmii="
|
|
} else {
|
|
gzipFileName += "docker=cmii="
|
|
}
|
|
gzipFileName += first[2]
|
|
gzipFileName += "="
|
|
|
|
} else if len(first) == 4 {
|
|
// harbor.cdcyy.cn/cmii/ossr/srs:v5.0.1
|
|
if !strings.HasPrefix(split[0], CmiiHarborPrefix) {
|
|
return imageRepoTag
|
|
}
|
|
gzipFileName += "cmlc=cmii="
|
|
gzipFileName += first[3]
|
|
gzipFileName += "="
|
|
} else if len(first) == 2 {
|
|
// bitnami/redis
|
|
// ossrs/srs
|
|
gzipFileName += "docker="
|
|
gzipFileName += first[0]
|
|
gzipFileName += "="
|
|
gzipFileName += first[1]
|
|
gzipFileName += "="
|
|
} else if len(first) == 1 {
|
|
// nginx:latest
|
|
return "docker=library=" + split[0] + "=" + split[1] + ".tar.gz"
|
|
}
|
|
|
|
gzipFileName += split[1]
|
|
gzipFileName += ".tar.gz"
|
|
|
|
return gzipFileName
|
|
}
|
|
|
|
func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {
|
|
|
|
for image, tag := range cmiiImageVersionMap {
|
|
s := CmiiHarborPrefix + image + ":" + tag
|
|
fullImageNameList = append(fullImageNameList, s)
|
|
}
|
|
|
|
return fullImageNameList
|
|
}
|
|
|
|
func loginToDockerHub(HarborFullHost string) {
|
|
|
|
if HarborFullHost == "" {
|
|
HarborFullHost = "https://registry-1.docker.io"
|
|
}
|
|
|
|
login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{
|
|
Username: "icederce",
|
|
Password: "loveff.cxc.23",
|
|
Auth: "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==",
|
|
ServerAddress: HarborFullHost,
|
|
})
|
|
if err != nil {
|
|
log.ErrorF("[loginToDockerHub] - login to %s failed !", HarborFullHost)
|
|
}
|
|
|
|
log.DebugF("[loginToDockerHub] - login is %s", login.Status)
|
|
}
|
|
|
|
func WriteDependencyImageToFile() {
|
|
imageFilePrefix := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\image\\"
|
|
|
|
middleFile := imageFilePrefix + "middle-image.txt"
|
|
_ = os.Remove(middleFile)
|
|
for _, image := range MiddlewareAmd64 {
|
|
executor.BasicAppendContentToFile(image+"\n", middleFile)
|
|
}
|
|
|
|
rkeFile := imageFilePrefix + "rke-image.txt"
|
|
_ = os.Remove(rkeFile)
|
|
for _, image := range Rancher1204Amd64 {
|
|
executor.BasicAppendContentToFile(image+"\n", rkeFile)
|
|
}
|
|
}
|