package image import ( "bufio" "context" "encoding/json" "errors" "fmt" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/klauspost/pgzip" "io" "io/fs" "os" "strconv" "strings" "wdd.io/agent-go/executor" "wdd.io/agent-go/logger" "wdd.io/agent-go/utils" ) var apiClient = newClient() var log = logger.Log const CmiiHarborPrefix = "harbor.cdcyy.com.cn/cmii/" func newClient() *client.Client { apiClient, err := client.NewClientWithOpts(client.FromEnv) if err != nil { log.ErrorF("[newClient] - new client failed ! => %s", err.Error()) panic(err) } defer apiClient.Close() return apiClient } func GetRunningContainer() []types.Container { return getContainerList(false) } func GetAllContainer() []types.Container { return getContainerList(true) } func getContainerList(all bool) []types.Container { containers, err := apiClient.ContainerList(context.Background(), types.ContainerListOptions{ Quiet: false, Size: false, All: all, Latest: false, Limit: 0, Filters: filters.Args{}, }) if err != nil { log.ErrorF("[getContainerList] - error => %s", err.Error()) } return containers } func GetAll() []types.ImageSummary { list, err := apiClient.ImageList(context.TODO(), types.ImageListOptions{ All: true, Filters: filters.Args{}, }) if err != nil { log.ErrorF("[ImageGetAll] --error => %s", err.Error()) } return list } func GetByName(imageName string) *types.ImageSummary { imageGetAll := GetAll() for _, imageSummary := range imageGetAll { for _, tag := range imageSummary.RepoTags { if strings.Contains(tag, imageName) { return &imageSummary } } } return nil } func Delete(imageName string) []types.ImageDeleteResponseItem { imageGetByName := GetByName(imageName) if imageGetByName == nil { log.ErrorF("[ImageDelete] -- image not exists ! %s", imageGetByName.RepoTags) return nil } remove, err := apiClient.ImageRemove(context.TODO(), imageGetByName.ID, types.ImageRemoveOptions{ Force: true, PruneChildren: false, }) if err != nil { log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error()) return nil } return remove } func PruneAllCmiiImages() (errorRemoveImageNameList []string) { _, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{}) imageGetAll := GetAll() // ip:8033 //re := regexp.MustCompile(`\b(?:\d{1,3}\.){3}\d{1,3}:\d{1,4}`) for _, imageSummary := range imageGetAll { for _, repoTag := range imageSummary.RepoTags { if strings.HasPrefix(repoTag, CmiiHarborPrefix) { for _, tag := range imageSummary.RepoTags { _, err := apiClient.ImageRemove(context.TODO(), imageSummary.ID, types.ImageRemoveOptions{ Force: true, PruneChildren: true, }) if err != nil { log.ErrorF("[ImageDelete] -- ImageRemove error ! %s", err.Error()) errorRemoveImageNameList = append(errorRemoveImageNameList, tag) } log.InfoF("[ImageDelete] - image remove of [%s] success!", tag) } } } } _, _ = apiClient.ImagesPrune(context.TODO(), filters.Args{}) return errorRemoveImageNameList } func TagFromSourceToTarget(sourceImageName, targetImageName string) bool { getByName := GetByName(sourceImageName) if getByName == nil { log.ErrorF("[ImageTagFromSourceToTarget] - %s not exits !", sourceImageName) return false } err := apiClient.ImageTag(context.TODO(), sourceImageName, targetImageName) if err != nil { log.ErrorF("[ImageTagFromSourceToTarget] - from %s to %s error %s", sourceImageName, targetImageName, err.Error()) } log.InfoF("[ImageTagFromSourceToTarget] - from %s to %s success!", sourceImageName, targetImageName) return true } func PushToOctopusKindHarbor(targetImageName string) (pushResult io.ReadCloser) { if GetByName(targetImageName) == nil { log.ErrorF("[ImagePushToOctopusKindHarbor] - %s not exits !", targetImageName) return pushResult } pushResult, err := apiClient.ImagePush(context.TODO(), targetImageName, types.ImagePushOptions{ All: false, RegistryAuth: "eyAidXNlcm5hbWUiOiAiYWRtaW4iLCAicGFzc3dvcmQiOiAiVjJyeVN0ckBuZ1BzcyIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==", PrivilegeFunc: nil, Platform: "amd64", }) if err != nil { log.ErrorF("[ImagePushToOctopusKindHarbor] - push %s error %s", targetImageName, err.Error()) return nil } return pushResult } // TagFromListAndPushToCHarbor 需要支持 harbor.cdcyy.cn ip:8033 rancher/rancher:v2.5.7 nginx:latest func TagFromListAndPushToCHarbor(referenceImageList []string, targetHarborHost string) (errorPushImageNameList []string) { for _, imageName := range referenceImageList { // check image // harbor.cdcyy.cn cmiiImageFullName := imageName if strings.HasPrefix(imageName, "cmii") { cmiiImageFullName = CmiiHarborPrefix + imageName } targetProject := "cmii" if strings.HasPrefix(imageName, "rancher") { targetProject = "rancher" } if strings.HasPrefix(imageName, CmiiHarborPrefix) { // imageName = strings.TrimPrefix(imageName, CmiiHarborPrefix) } else { // todo if strings.Contains(imageName, "/") { imageName = strings.Split(imageName, "/")[1] } } targetImageName := targetHarborHost + ":8033/" + targetProject + "/" + imageName if TagFromSourceToTarget(cmiiImageFullName, targetImageName) { pushResult := PushToOctopusKindHarbor(targetImageName) if pushResult == nil { errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName) log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s error error !", targetImageName) break } scanner := bufio.NewScanner(pushResult) for scanner.Scan() { } log.InfoF("[ImageTagFromListAndPushToCHarbor] - push of %s success!", targetImageName) } else { errorPushImageNameList = append(errorPushImageNameList, cmiiImageFullName) } } return errorPushImageNameList } func PullFromCmiiHarbor(imageName string) (pullResult io.ReadCloser) { pullResult, err := apiClient.ImagePull(context.TODO(), imageName, types.ImagePullOptions{ All: false, RegistryAuth: "eyAidXNlcm5hbWUiOiAicmFkMDJfZHJvbmUiLCAicGFzc3dvcmQiOiAiRHJvbmVAMTIzNCIsICJlbWFpbCI6ICJpY2VAcXEuY29tIiB9Cg==", PrivilegeFunc: func() (string, error) { return "authorization: Basic cmFkMDJfZHJvbmU6RHJvbmVAMTIzNA==", nil }, Platform: "amd64", }) if err != nil { log.ErrorF("[ImagePullFromCmiiHarbor]- image pull of %s ,error => %s", imageName, err.Error()) return nil } return pullResult } func PullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode bool) (fullImageNameList, errorPullImageList []string) { fullImageNameList = ConvertCMiiImageMapToList(imageVersionMap) return fullImageNameList, PullFromFullNameList(fullImageNameList) } func PullCmiiFromFileJson(filePathName string) { readFile, err := os.ReadFile(filePathName) if err != nil { log.ErrorF("[ImagePullCMiiFromFileJson] - file %s read error ! %s", filePathName, err.Error()) return } var resultMap map[string]string err = json.Unmarshal(readFile, &readFile) if err != nil { log.ErrorF("[ImagePullCMiiFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error()) return } for image, tag := range resultMap { pullResult := PullFromCmiiHarbor(image + ":" + tag) if pullResult == nil { continue } scanner := bufio.NewScanner(pullResult) for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "\"status\":\"Pulling from") { fmt.Println(line) } if strings.Contains(line, "Status: Image is up to date for") { fmt.Println(line) } } fmt.Println() } } // PullFromFullNameList 根据镜像名列表拉取全部的镜像 func PullFromFullNameList(fullImageNameList []string) (errorPullImageList []string) { for _, dep := range fullImageNameList { pullResult := PullFromCmiiHarbor(dep) if pullResult == nil { errorPullImageList = append(errorPullImageList, dep) continue } scanner := bufio.NewScanner(pullResult) for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "\"status\":\"Pulling from") { fmt.Println(line) } if strings.Contains(line, "Status: Image is up to date for") { fmt.Println(line) } } fmt.Println() } return errorPullImageList } func PullFromListAndCompressSplit(fullImageNameList []string, gzipFolder string) (errorPullImageList, errorGzipImageList []string) { errorPullImageList = PullFromFullNameList(fullImageNameList) // generate a project folder err := os.MkdirAll(gzipFolder, os.ModeDir) if err != nil { if !errors.Is(err, os.ErrExist) { log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error()) } } var tarGzipFileNameList []string for _, image := range fullImageNameList { if !SaveToTarGZ(image, gzipFolder) { errorGzipImageList = append(errorGzipImageList, image) continue } tarGzipFileNameList = append(tarGzipFileNameList, convertImageGzipFileName(image)) } utils.BeautifulPrintListWithTitle(tarGzipFileNameList, "image gzip name list") return errorPullImageList, errorGzipImageList } func LoadFromGzipFilePath(gzipFullPath string) bool { openFile, err := os.OpenFile(gzipFullPath, 0, fs.ModePerm) if err != nil { log.ErrorF("[ImageLoadFromFile] - failed to open file %s, error is %s", gzipFullPath, err.Error()) return false } loadResponse, err := apiClient.ImageLoad(context.TODO(), openFile, true) if err != nil { log.ErrorF("[ImageLoadFromFile] - load error %s, error is %s", gzipFullPath, err.Error()) return false } log.InfoF("[ImageLoadFromFile] - load of %s, result is %s", gzipFullPath, strconv.FormatBool(loadResponse.JSON)) scanner := bufio.NewScanner(loadResponse.Body) for scanner.Scan() { line := scanner.Text() fmt.Println(line) } return true } func LoadFromFolderPath(folderPath string) (errorLoadImageNameList []string) { if !strings.HasSuffix(folderPath, "/") { folderPath += "/" } dirEntries, err := os.ReadDir(folderPath) if err != nil { log.ErrorF("[ImageLoadFromFolderPath] - error read folder %s error is %s", folderPath, err.Error()) return } // load gzip file for _, dirEntry := range dirEntries { if strings.HasSuffix(dirEntry.Name(), ".tar.gz") { if !LoadFromGzipFilePath(folderPath + dirEntry.Name()) { errorLoadImageNameList = append(errorLoadImageNameList, folderPath+dirEntry.Name()) } } } return errorLoadImageNameList } func SaveToTarGZ(targetImageName, folderPathPrefix string) bool { imageGetByName := GetByName(targetImageName) if imageGetByName == nil { log.ErrorF("[ImageSaveToTarGZ] - %s not exits", targetImageName) return false } imageSaveTarStream, err := apiClient.ImageSave(context.TODO(), imageGetByName.RepoTags) if err != nil { log.ErrorF("[ImageSaveToTarGZ] - image save error %s", err.Error()) return false } var realImageTag string for _, repoTag := range imageGetByName.RepoTags { if !strings.Contains(repoTag, "8033") { realImageTag = repoTag break } } gzipImageFile := convertImageGzipFileName(realImageTag) if !strings.HasSuffix(folderPathPrefix, "/") { folderPathPrefix += "/" } _ = os.MkdirAll(folderPathPrefix, os.ModeDir) gzipImageFile = folderPathPrefix + gzipImageFile log.InfoF("[ImageSaveToTarGZ] - start to save [%s] to [%s]", realImageTag, gzipImageFile) _ = os.Remove(gzipImageFile) tarFile, err := os.Create(gzipImageFile) if err != nil { log.ErrorF("[ImageSaveToTarGZ] - error create gzip %s file ! => %s ", gzipImageFile, err.Error()) return false } defer tarFile.Close() // 创建pgzip writer gw, err := pgzip.NewWriterLevel(tarFile, pgzip.DefaultCompression) // 你可以调整压缩级别 if err != nil { log.ErrorF("[ImageSaveToTarGZ] - pgzip create writer error: %s", err.Error()) } defer gw.Close() // Copy the tar archive to the gzip writer. if _, err := io.Copy(gw, imageSaveTarStream); err != nil { log.ErrorF("[ImageSaveToTarGZ] - failed to copy tar archive to gzip writer: %s", err.Error()) return false } return true } // convertImageGzipFileName 必须输出长度为4的内容 =出现得次数为3 func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) { // harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 ==> cmlc=cmii=cmii-uav-platform=5.4.0.tar.gz // rancher/fleet:v0.3.4 // ossr/srs:v5.0.1 ==> docker=cmii=srs=v5.0.1.tar.gz // nginx:latest // bitnami/minio:2022.5.4 // simonrupf/chronyd:0.4.3 // 10.1.1.1:8033/cmii/ok:1.2 不支持 不允许存在 split := strings.Split(imageRepoTag, ":") //log.DebugF(" %s to %s", imageRepoTag, split) if len(split) == 1 { // nginx return "docker=library=" + imageRepoTag + "=latest.tar.gz" } first := strings.Split(split[0], "/") //log.DebugF(" split[0] %s to %s", split[0], first) if len(first) == 3 { // harbor.cdcyy.cn/cmii/cmii-uav-platform:5.4.0 // docker.io/ossr/srs:v5.0.1 if strings.HasPrefix(split[0], CmiiHarborPrefix) { gzipFileName += "cmlc=cmii=" } else { gzipFileName += "docker=cmii=" } gzipFileName += first[2] gzipFileName += "=" } else if len(first) == 4 { // harbor.cdcyy.cn/cmii/ossr/srs:v5.0.1 if !strings.HasPrefix(split[0], CmiiHarborPrefix) { return imageRepoTag } gzipFileName += "cmlc=cmii=" gzipFileName += first[3] gzipFileName += "=" } else if len(first) == 2 { // bitnami/redis // ossrs/srs gzipFileName += "docker=" gzipFileName += first[0] gzipFileName += "=" gzipFileName += first[1] gzipFileName += "=" } else if len(first) == 1 { // nginx:latest return "docker=library=" + split[0] + "=" + split[1] + ".tar.gz" } gzipFileName += split[1] gzipFileName += ".tar.gz" return gzipFileName } func ConvertCMiiImageMapToList(cmiiImageVersionMap map[string]string) (fullImageNameList []string) { for image, tag := range cmiiImageVersionMap { s := CmiiHarborPrefix + image + ":" + tag fullImageNameList = append(fullImageNameList, s) } return fullImageNameList } func loginToDockerHub(HarborFullHost string) { if HarborFullHost == "" { HarborFullHost = "https://registry-1.docker.io" } login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{ Username: "icederce", Password: "loveff.cxc.23", Auth: "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==", ServerAddress: HarborFullHost, }) if err != nil { log.ErrorF("[loginToDockerHub] - login to %s failed !", HarborFullHost) } log.DebugF("[loginToDockerHub] - login is %s", login.Status) } func WriteDependencyImageToFile() { imageFilePrefix := "C:\\Users\\wddsh\\Documents\\IdeaProjects\\ProjectOctopus\\cmii_operator\\image\\" middleFile := imageFilePrefix + "middle-image.txt" _ = os.Remove(middleFile) for _, image := range MiddlewareAmd64 { executor.BasicAppendContentToFile(image+"\n", middleFile) } rkeFile := imageFilePrefix + "rke-image.txt" _ = os.Remove(rkeFile) for _, image := range Rancher1204Amd64 { executor.BasicAppendContentToFile(image+"\n", rkeFile) } }