ProjectOctopus/cmii_operator/CmiiImageSync.go

package cmii_operator

import (
	"bufio"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/client"
	"github.com/klauspost/pgzip"
	"io"
	"io/fs"
	"os"
	"strconv"
	"strings"
)

var apiClient = newClient()

const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/"

func newClient() *client.Client {
	apiClient, err := client.NewClientWithOpts(client.FromEnv)

	if err != nil {
		log.ErrorF("[newClient] - new client failed ! => %s", err.Error())
		panic(err)
	}
	defer apiClient.Close()

	return apiClient
}

func GetRunningContainer() []types.Container {
	return getContainerList(false)
}

func GetAllContainer() []types.Container {
	return getContainerList(true)
}

func getContainerList(all bool) []types.Container {

	containers, err := apiClient.ContainerList(context.Background(), types.ContainerListOptions{
		Quiet:   false,
		Size:    false,
		All:     all,
		Latest:  false,
		Limit:   0,
		Filters: filters.Args{},
	})
	if err != nil {
		log.ErrorF("[getContainerList] - error => %s", err.Error())
	}

	return containers
}

func ImageGetAll() []types.ImageSummary {

	list, err := apiClient.ImageList(context.TODO(), types.ImageListOptions{
		All:     true,
		Filters: filters.Args{},
	})
	if err != nil {
		log.ErrorF("[ImageGetAll] --error => %s", err.Error())
	}

	return list
}

func ImageGetByName(imageName string) *types.ImageSummary {

	imageGetAll := ImageGetAll()

	for _, imageSummary := range imageGetAll {
		for _, tag := range imageSummary.RepoTags {
			if strings.Contains(tag, imageName) {
				return &imageSummary
			}
		}
	}
	return nil
}

func ImageDelete(imageName string) []types.ImageDeleteResponseItem {
	imageGetByName := ImageGetByName(imageName)
	if imageGetByName == nil {
		log.ErrorF("[ImageDelete] -- image not exists ! %s", imageGetByName.RepoTags)
		return nil
	}

	remove, err := apiClient.ImageRemove(context.TODO(), imageGetByName.ID, types.ImageRemoveOptions{
		Force:         true,
		PruneChildren: false,
	})
	if err != nil {
		log.ErrorF("[ImageDelete] -- ImageRemove error  ! %s", err.Error())
		return nil
	}

	return remove
}

func ImagePruneAllCmiiImages() (errorRemoveImageNameList []string) {

	imageGetAll := ImageGetAll()

	for _, imageSummary := range imageGetAll {
		if strings.Contains(imageSummary.RepoTags[0], CmiiHarborPrefix) {
			_, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{
				Force:         true,
				PruneChildren: false,
			})
			if err != nil {
				log.ErrorF("[ImageDelete] -- ImageRemove error  ! %s", err.Error())
				errorRemoveImageNameList = append(errorRemoveImageNameList, imageSummary.RepoTags[0])
			}

			log.InfoF("[ImageDelete] - image remove of [%s] success!", imageSummary.RepoTags[0])
		}
	}

	return errorRemoveImageNameList
}

func ImageTagFromSourceToTarget(sourceImageName, targetImageName string) bool {

	getByName := ImageGetByName(sourceImageName)
	if getByName == nil {
		log.ErrorF("[ImageTagFromSourceToTarget] -  %s not exits !", sourceImageName)
		return false
	}

	err := apiClient.ImageTag(context.TODO(), sourceImageName, targetImageName)
	if err != nil {
		log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error())
	}

	log.InfoF("[ImageTagFromSourceToTarget] - from %s to %s success!", sourceImageName, targetImageName)
	return true
}

func ImagePushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) {

	if ImageGetByName(targetImageName) == nil {
		log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName)
		return pushResult
	}

	pushResult, err := apiClient.ImagePush(context.TODO(), targetImageName, types.ImagePushOptions{
		All:           false,
		RegistryAuth:  "eyAidXNlcm5hbWUiOiAiYWRtaW4iLCAicGFzc3dvcmQiOiAiVjJyeVN0ckBuZ1BzcyIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
		PrivilegeFunc: nil,
		Platform:      "amd64",
	})
	if err != nil {
		log.ErrorF("[ImagePushToOctopusKindHarbor] - push %s error %s", targetImageName, err.Error())
		return nil
	}

	return pushResult
}

func ImageTagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) {

	for _, imageName := range referenceImageList {
		// check image
		cmiiImageFullName := imageName
		if strings.HasPrefix(imageName, "cmii") {
			cmiiImageFullName = CmiiHarborPrefix + imageName
		}

		targetProject := "cmii"
		if strings.HasPrefix(imageName, "rancher") {
			targetProject = "rancher"
		}

		if strings.Contains(imageName, "/") {
			imageName = strings.Split(imageName, "/")[1]
		}

		targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName

		if ImageTagFromSourceToTarget(cmiiImageFullName, targetImageName) {
			pushResult := ImagePushToOctopusKindHarbor(targetImageName)
			if pushResult == nil {
				errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
			}
			scanner := bufio.NewScanner(pushResult)
			for scanner.Scan() {

			}
			log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName)
		} else {
			errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName)
		}

	}

	return errorPushImageNameList
}

func ImagePullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) {
	pullResult, err := apiClient.ImagePull(context.TODO(), imageName, types.ImagePullOptions{
		All:          false,
		RegistryAuth: "eyAidXNlcm5hbWUiOiAicmFkMDJfZHJvbmUiLCAicGFzc3dvcmQiOiAiRHJvbmVAMTIzNCIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==",
		PrivilegeFunc: func() (string, error) {
			return "authorization: Basic cmFkMDJfZHJvbmU6RHJvbmVAMTIzNA==", nil
		},
		Platform: "amd64",
	})
	if err != nil {
		log.ErrorF("[ImagePullFromCmiiHarbor]- image pull of %s ,error => %s", imageName, err.Error())
		return nil
	}

	return pullResult
}

func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (errorPullImageList []string) {

	fullImageNameList := convertCMiiImageMapToList(imageVersionMap)
	return ImagePullFromFullNameList(fullImageNameList)

}

func ImagePullCMiiFromFileJson(filePathName string) {

	readFile, err := os.ReadFile(filePathName)
	if err != nil {
		log.ErrorF("[ImagePullCMiiFromFileJson] - file %s read error ! %s", filePathName, err.Error())
		return
	}

	var resultMap map[string]string
	err = json.Unmarshal(readFile, &readFile)
	if err != nil {
		log.ErrorF("[ImagePullCMiiFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
		return
	}

	for image, tag := range resultMap {
		pullResult := ImagePullFromCmiiHarbor(image + ":" + tag)
		if pullResult == nil {
			continue
		}
		scanner := bufio.NewScanner(pullResult)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.Contains(line, "\"status\":\"Pulling from") {
				fmt.Println(line)
			}
			if strings.Contains(line, "Status: Image is up to date for") {
				fmt.Println(line)
			}
		}
		fmt.Println()
	}

}

func ImagePullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) {

	for _, dep := range fullImageNameList {
		loginToDockerHub()

		pullResult := ImagePullFromCmiiHarbor(dep)
		if pullResult == nil {
			errorPullImageList = append(errorPullImageList, dep)
			continue
		}
		scanner := bufio.NewScanner(pullResult)
		for scanner.Scan() {
			line := scanner.Text()
			if strings.Contains(line, "\"status\":\"Pulling from") {
				fmt.Println(line)
			}
			if strings.Contains(line, "Status: Image is up to date for") {
				fmt.Println(line)
			}
		}
		fmt.Println()
	}

	return errorPullImageList
}

func ImagePullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) {

	errorPullImageList = ImagePullFromFullNameList(fullImageNameList)

	// generate a project folder
	err := os.MkdirAll(gzipFolder, os.ModeDir)
	if err != nil {
		if !errors.Is(err, os.ErrExist) {
			log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error())
		}
	}

	for _, image := range fullImageNameList {
		if !ImageSaveToTarGZ(image, gzipFolder) {
			errorGzipImageList = append(errorGzipImageList, image)
		}
	}

	return errorPullImageList, errorGzipImageList
}

func ImageLoadFromFile(gzipFullPath string) bool {
	openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm)
	if err != nil {
		log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error())
		return false
	}

	loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true)
	if err != nil {
		log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFullPath, err.Error())
		return false
	}

	log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFullPath, strconv.FormatBool(loadResponse.JSON))

	scanner := bufio.NewScanner(loadResponse.Body)
	for scanner.Scan() {
		line := scanner.Text()
		fmt.Println(line)
	}

	return true
}

func ImageLoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) {
	if !strings.HasSuffix(folderPath, "/") {
		folderPath += "/"
	}

	dirEntries, err := os.ReadDir(folderPath)
	if err != nil {
		log.ErrorF("[ImageLoadFromFolderPath] - error read folder %s error is %s", folderPath, err.Error())
		return
	}

	// load gzip file
	for _, dirEntry := range dirEntries {
		if strings.HasSuffix(dirEntry.Name(), ".tar.gz") {
			if !ImageLoadFromFile(folderPath + dirEntry.Name()) {
				errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name())
			}
		}
	}

	return errorLoadImageNameList
}

func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {

	imageGetByName := ImageGetByName(targetImageName)
	if imageGetByName == nil {
		log.ErrorF("[ImageSaveToTarGZ] - %s not exits", targetImageName)
		return false
	}

	imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags)
	if err != nil {
		log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error())
		return false
	}

	gzipImageFile := convertImageGzipFileName(imageGetByName.RepoTags[0])
	if !strings.HasSuffix(folderPathPrefix, "/") {
		folderPathPrefix += "/"
	}
	gzipImageFile = folderPathPrefix + gzipImageFile
	log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", imageGetByName.RepoTags[0], gzipImageFile)

	_ = os.Remove(gzipImageFile)
	tarFile, err := os.Create(gzipImageFile)
	if err != nil {
		log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFile, err.Error())
		return false
	}
	defer tarFile.Close()

	// 创建pgzip writer
	gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression) // 你可以调整压缩级别
	if err != nil {
		log.ErrorF("[ImageSaveToTarGZ] - pgzip create writer error: %s", err.Error())
	}
	defer gw.Close()

	// Copy the tar archive to the gzip writer.
	if _, err := io.Copy(gw, imageSaveTarStream); err != nil {
		log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error())
		return false
	}

	return true
}

func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {

	split := strings.Split(imageRepoTag, ":")
	//log.DebugF(" %s to %s", imageRepoTag, split)
	if len(split) == 1 {
		return "docker=" + imageRepoTag + "=latest.tar.gz"
	}

	first := strings.Split(split[0], "/")
	//log.DebugF(" split[0] %s to %s", split[0], first)
	if len(first) == 3 {
		if strings.Contains(first[0], "cdcyy") {
			gzipFileName += "cmlc="
		} else {
			gzipFileName += "docker="
		}

		gzipFileName += first[1]
		gzipFileName += "="
		gzipFileName += first[2]
		gzipFileName += "="

	} else if len(first) == 2 {
		//
		gzipFileName += "docker="
		gzipFileName += first[0]
		gzipFileName += "="
		gzipFileName += first[1]
		gzipFileName += "="
	} else if len(first) == 1 {
		//
		return "docker=" + split[0] + "=" + split[1] + ".tar.gz"
	}

	gzipFileName += split[1]
	gzipFileName += ".tar.gz"

	return gzipFileName
}

func convertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) {

	for image, tag := range cmiiImageVersionMap {
		s := CmiiHarborPrefix + image + ":" + tag
		fullImageNameList = append(fullImageNameList, s)
	}

	return fullImageNameList
}

func loginToDockerHub() {

	login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{
		Username:      "icederce",
		Password:      "loveff.cxc.23",
		Auth:          "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==",
		ServerAddress: "https://registry-1.docker.io",
	})
	if err != nil {
		log.ErrorF("[loginToDockerHub] - login failed !")
	}

	log.DebugF("[loginToDockerHub] - login is %s", login.Status)
}