Merge remote-tracking branch 'origin/local-dev'

cmii_operator/CmiiImageSync.go

@@ -3,6 +3,8 @@ package cmii_operator
 import (
 	"bufio"
 	"context"
+	"encoding/json"
+	"errors"
 	"fmt"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/klauspost/pgzip"
@@ -188,6 +190,82 @@ func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode
 
 }
 
+func ImagePullFromFileJson(filePathName string) {
+
+	readFile, err := os.ReadFile(filePathName)
+	if err != nil {
+		log.ErrorF("[ImagePullFromFileJson] - file %s read error ! %s", filePathName, err.Error())
+		return
+	}
+
+	var resultMap map[string]string
+	err = json.Unmarshal(readFile, &readFile)
+	if err != nil {
+		log.ErrorF("[ImagePullFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
+		return
+	}
+
+	for image, tag := range resultMap {
+		pullResult := ImagePullFromCmiiHarbor(image + ":" + tag)
+		if pullResult == nil {
+			continue
+		}
+		scanner := bufio.NewScanner(pullResult)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.Contains(line, "\"status\":\"Pulling from") {
+				fmt.Println(line)
+			}
+			if strings.Contains(line, "Status: Image is up to date for") {
+				fmt.Println(line)
+			}
+		}
+		fmt.Println()
+	}
+
+}
+
+func ImagePullFromList(depContainerList []string) {
+	for _, dep := range depContainerList {
+
+		loginToDockerHub()
+
+		pullResult := ImagePullFromCmiiHarbor(dep)
+		if pullResult == nil {
+			continue
+		}
+		scanner := bufio.NewScanner(pullResult)
+		for scanner.Scan() {
+			line := scanner.Text()
+			if strings.Contains(line, "\"status\":\"Pulling from") {
+				fmt.Println(line)
+			}
+			if strings.Contains(line, "Status: Image is up to date for") {
+				fmt.Println(line)
+			}
+		}
+		fmt.Println()
+	}
+}
+
+func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string) {
+
+	ImagePullFromList(imageNameList)
+
+	// generate a project folder
+	err := os.MkdirAll(gzipFolder, os.ModeDir)
+	if err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error())
+		}
+	}
+
+	for _, image := range imageNameList {
+		ImageSaveToTarGZ(image, gzipFolder)
+	}
+
+}
+
 func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
 
 	imageGetByName := ImageGetByName(targetImageName)
@@ -236,8 +314,13 @@ func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
 func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
 
 	split := strings.Split(imageRepoTag, ":")
-	first := strings.Split(split[0], "/")
+	//log.DebugF(" %s to %s", imageRepoTag, split)
+	if len(split) == 1 {
+		return "docker=" + imageRepoTag + "=latest.tar.gz"
+	}
 
+	first := strings.Split(split[0], "/")
+	//log.DebugF(" split[0] %s to %s", split[0], first)
 	if len(first) == 3 {
 		if strings.Contains(first[0], "cdcyy") {
 			gzipFileName += "cmlc="
@@ -250,11 +333,15 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
 		gzipFileName += first[2]
 		gzipFileName += "="
 	} else if len(first) == 2 {
+		//
 		gzipFileName += "docker="
 		gzipFileName += first[0]
 		gzipFileName += "="
 		gzipFileName += first[1]
 		gzipFileName += "="
+	} else if len(first) == 1 {
+		//
+		return "docker=" + split[0] + "=" + split[1] + ".tar.gz"
 	}
 
 	gzipFileName += split[1]
@@ -262,3 +349,18 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
 
 	return gzipFileName
 }
+
+func loginToDockerHub() {
+
+	login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{
+		Username:      "icederce",
+		Password:      "loveff.cxc.23",
+		Auth:          "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==",
+		ServerAddress: "https://registry-1.docker.io",
+	})
+	if err != nil {
+		log.ErrorF("[loginToDockerHub] - login failed !")
+	}
+
+	log.DebugF("[loginToDockerHub] - login is %s", login.Status)
+}

cmii_operator/CmiiImageSync_test.go

@@ -104,3 +104,39 @@ func TestImageSaveToTarGZ(t *testing.T) {
 
 	assert.Equal(t, imageSaveToTarGZ, true, "image save to tar gz file error !")
 }
+
+func TestConvertImageGzipFileName(t *testing.T) {
+
+	test := []string{
+		"bitnami/redis:6.2.6-debian-10-r0",
+		"bitnami/redis:6.2.14-debian-11-r1",
+		"bitnami/mysql:8.0.35-debian-11-r1",
+		"bitnami/mysql:8.1.0-debian-11-r42",
+		"simonrupf/chronyd:0.4.3",
+		"bitnami/bitnami-shell:10-debian-10-r140",
+		"bitnami/bitnami-shell:11-debian-11-r136",
+		"bitnami/rabbitmq:3.9.12-debian-10-r3",
+		"bitnami/rabbitmq:3.11.26-debian-11-r2",
+		"ossrs/srs:v4.0.136",
+		"emqx/emqx:4.2.12",
+		"nacos/nacos-server:v2.1.2",
+		"nacos/nacos-server:v2.1.2-slim",
+		"mongo:5.0",
+		"rabbitmq:3.9-management",
+		"bitnami/minio:2022.5.4",
+		"bitnami/minio:2023.5.4",
+		"kubernetesui/dashboard:v2.0.1",
+		"kubernetesui/metrics-scraper:v1.0.4",
+		"ossrs/srs:v4.0-r3",
+		"nginx:1.21.3",
+		"redis:6.0.20-alpine",
+		"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
+		"busybox:latest",
+		"busybox",
+	}
+
+	for _, s := range test {
+		gzipFileName := convertImageGzipFileName(s)
+		t.Logf(" %s to %s", s, gzipFileName)
+	}
+}

cmii_operator/CmiiOperator.go

@@ -3,21 +3,19 @@ package cmii_operator
 import (
 	"errors"
 	"os"
-	"strings"
 	"wdd.io/agent-go/executor"
 	"wdd.io/agent-go/utils"
 )
 
-func FetchDemoImages(projectName, projectFolderPath string) bool {
+const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
-	if !strings.HasSuffix(projectFolderPath, "/") {
+
-		projectFolderPath += "/"
+func FetchDemoImages(projectName string, gzipSplit bool) bool {
-	}
 
 	// generate a project folder
-	err := os.Mkdir(projectFolderPath, os.ModeDir)
+	err := os.Mkdir(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
 	if err != nil {
 		if !errors.Is(err, os.ErrExist) {
-			log.ErrorF("[FetchDemoImages] - create folder of %s error %s", projectFolderPath, err.Error())
+			log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
 			return false
 		}
 	}
@@ -29,8 +27,8 @@ func FetchDemoImages(projectName, projectFolderPath string) bool {
 	utils.BeautifulPrint(frontendMap)
 
 	// save map to file
-	backendMapFile := projectFolderPath + projectName + "-backend-app.json"
+	backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
-	frontendMapFile := projectFolderPath + projectName + "-frontend-app.json"
+	frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
 	_ = os.Remove(backendMapFile)
 	_ = os.Remove(frontendMapFile)
 
@@ -48,10 +46,31 @@ func FetchDemoImages(projectName, projectFolderPath string) bool {
 	ImagePullFromCmiiHarborByMap(frontendMap, true)
 
 	// compress image
-
+	if gzipSplit {
+		for image, tag := range backendMap {
+			ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
+		}
+		for image, tag := range frontendMap {
+			ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
+		}
+	}
 	// upload to harbor
 
 	// clean up images
 
 	return false
 }
+
+func FetchDependencyRepos(gzipSplit bool) {
+	err := os.Mkdir(OfflineImageGzipFolderPrefix, os.ModeDir)
+	if err != nil {
+		if !errors.Is(err, os.ErrExist) {
+			log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error())
+		}
+	}
+
+	ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
+
+	ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
+
+}

cmii_operator/CmiiOperator_test.go

@@ -3,5 +3,9 @@ package cmii_operator
 import "testing"
 
 func TestFetchDemoImages(t *testing.T) {
-	FetchDemoImages("cqga", "/root/Downloads/cqga")
+	FetchDemoImages("cqga", true)
+}
+
+func TestFetchDependencyRepos(t *testing.T) {
+	FetchDependencyRepos(true)
 }

cmii_operator/DependencyConfig.go

@@ -0,0 +1,92 @@
+package cmii_operator
+
+var MiddlewareAmd64 = []string{
+	"bitnami/redis:6.2.6-debian-10-r0",
+	"bitnami/redis:6.2.14-debian-11-r1",
+	"bitnami/mysql:8.0.35-debian-11-r1",
+	"bitnami/mysql:8.1.0-debian-11-r42",
+	"simonrupf/chronyd:0.4.3",
+	"bitnami/bitnami-shell:10-debian-10-r140",
+	"bitnami/bitnami-shell:11-debian-11-r136",
+	"bitnami/rabbitmq:3.9.12-debian-10-r3",
+	"bitnami/rabbitmq:3.11.26-debian-11-r2",
+	"ossrs/srs:v4.0.136",
+	"emqx/emqx:4.2.12",
+	"nacos/nacos-server:v2.1.2",
+	"nacos/nacos-server:v2.1.2-slim",
+	"mongo:5.0",
+	"rabbitmq:3.9-management",
+	"bitnami/minio:2022.5.4",
+	"bitnami/minio:2023.5.4",
+	"kubernetesui/dashboard:v2.0.1",
+	"kubernetesui/metrics-scraper:v1.0.4",
+	"ossrs/srs:v4.0-r3",
+	"nginx:1.21.3",
+	"redis:6.0.20-alpine",
+	"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
+	"busybox:latest",
+}
+
+var Rancher1204Amd64 = []string{
+	"rancher/backup-restore-operator:v1.0.3",
+	"rancher/calico-cni:v3.17.2",
+	"rancher/calico-ctl:v3.17.2",
+	"rancher/calico-kube-controllers:v3.17.2",
+	"rancher/calico-node:v3.17.2",
+	"rancher/calico-pod2daemon-flexvol:v3.17.2",
+	"rancher/cis-operator:v1.0.3",
+	"rancher/cluster-proportional-autoscaler:1.7.1",
+	"rancher/coredns-coredns:1.8.0",
+	"rancher/coreos-etcd:v3.4.14-rancher1",
+	"rancher/coreos-kube-state-metrics:v1.9.7",
+	"rancher/coreos-prometheus-config-reloader:v0.39.0",
+	"rancher/coreos-prometheus-operator:v0.39.0",
+	"rancher/externalip-webhook:v0.1.6",
+	"rancher/flannel-cni:v0.3.0-rancher6",
+	"rancher/coreos-flannel:v0.13.0-rancher1",
+	"rancher/fleet-agent:v0.3.4",
+	"rancher/fleet:v0.3.4",
+	"rancher/fluentd:v0.1.24",
+	"rancher/grafana-grafana:7.1.5",
+	"rancher/hyperkube:v1.20.4-rancher1",
+	"rancher/jimmidyson-configmap-reload:v0.3.0",
+	"rancher/k8s-dns-dnsmasq-nanny:1.15.2",
+	"rancher/k8s-dns-kube-dns:1.15.2",
+	"rancher/k8s-dns-node-cache:1.15.13",
+	"rancher/k8s-dns-sidecar:1.15.2",
+	"rancher/klipper-lb:v0.1.2",
+	"rancher/kube-api-auth:v0.1.4",
+	"rancher/kubectl:v1.20.4",
+	"rancher/kubernetes-external-dns:v0.7.3",
+	"rancher/cluster-proportional-autoscaler:1.8.1",
+	"rancher/library-busybox:1.31.1",
+	"rancher/library-busybox:1.32.1",
+	"rancher/library-nginx:1.19.2-alpine",
+	"rancher/library-traefik:1.7.19",
+	"rancher/local-path-provisioner:v0.0.11",
+	"rancher/local-path-provisioner:v0.0.14",
+	"rancher/local-path-provisioner:v0.0.19",
+	"rancher/log-aggregator:v0.1.7",
+	"rancher/istio-kubectl:1.5.10",
+	"rancher/metrics-server:v0.4.1",
+	"rancher/configmap-reload:v0.3.0-rancher4",
+	"rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1",
+	"rancher/nginx-ingress-controller:nginx-0.43.0-rancher1",
+	"rancher/opa-gatekeeper:v3.1.0-beta.7",
+	"rancher/openzipkin-zipkin:2.14.2",
+	"rancher/pause:3.2",
+	"rancher/plugins-docker:18.09",
+	"rancher/prom-alertmanager:v0.21.0",
+	"rancher/prom-node-exporter:v1.0.1",
+	"rancher/prom-prometheus:v2.18.2",
+	"rancher/prometheus-auth:v0.2.1",
+	"rancher/rancher-agent:v2.5.7",
+	"rancher/rancher-webhook:v0.1.0-beta9",
+	"rancher/rancher:v2.5.7",
+	"rancher/rke-tools:v0.1.72",
+	"rancher/security-scan:v0.1.14",
+	"rancher/security-scan:v0.2.2",
+	"rancher/shell:v0.1.6",
+	"rancher/sonobuoy-sonobuoy:v0.16.3",
+	"rancher/system-upgrade-controller:v0.6.2",
+}

cmii_operator/config/middleware-amd64.json

@@ -0,0 +1,26 @@
+{
+  "docker.io/bitnami/redis": "6.2.6-debian-10-r0",
+  "docker.io/bitnami/redis": "6.2.14-debian-11-r1",
+  "docker.io/bitnami/mysql": "8.0.35-debian-11-r1",
+  "docker.io/bitnami/mysql": "8.1.0-debian-11-r42",
+  "docker.io/simonrupf/chronyd": "0.4.3",
+  "docker.io/bitnami/bitnami-shell": "10-debian-10-r140",
+  "docker.io/bitnami/bitnami-shell": "11-debian-11-r136",
+  "docker.io/bitnami/rabbitmq": "3.9.12-debian-10-r3",
+  "docker.io/bitnami/rabbitmq": "3.11.26-debian-11-r2",
+  "docker.io/ossrs/srs": "v4.0.136",
+  "docker.io/emqx/emqx": "4.2.12",
+  "docker.io/nacos/nacos-server": "v2.1.2",
+  "docker.io/nacos/nacos-server": "v2.1.2-slim",
+  "docker.io/mongo": "5.0",
+  "docker.io/rabbitmq": "3.9-management",
+  "docker.io/bitnami/minio": "2022.5.4",
+  "docker.io/bitnami/minio": "2023.5.4",
+  "docker.io/simonrupf/chronyd": "0.4.3",
+  "docker.io/kubernetesui/dashboard": "v2.0.1",
+  "docker.io/kubernetesui/metrics-scraper": "v1.0.4",
+  "docker.io/ossrs/srs": "v4.0-r3",
+  "docker.io/nginx": "1.21.3",
+  "docker.io/redis": "6.0.20-alpine",
+  "docker.io/dyrnq/nfs-subdir-external-provisioner": "v4.0.2"
+}

cmii_operator/config/rancher-1.20.4-amd64.json

@@ -0,0 +1,58 @@
+{
+  "rancher/backup-restore-operator": "v1.0.3",
+  "rancher/calico-cni": "v3.17.2",
+  "rancher/calico-ctl": "v3.17.2",
+  "rancher/calico-kube-controllers": "v3.17.2",
+  "rancher/calico-node": "v3.17.2",
+  "rancher/calico-pod2daemon-flexvol": "v3.17.2",
+  "rancher/cis-operator": "v1.0.3",
+  "rancher/coredns-coredns": "1.8.0",
+  "rancher/coreos-etcd": "v3.4.14-rancher1",
+  "rancher/coreos-kube-state-metrics": "v1.9.7",
+  "rancher/coreos-prometheus-config-reloader": "v0.39.0",
+  "rancher/coreos-prometheus-operator": "v0.39.0",
+  "rancher/externalip-webhook": "v0.1.6",
+  "rancher/flannel-cni": "v0.3.0-rancher6",
+  "rancher/coreos-flannel": "v0.13.0-rancher1",
+  "rancher/fleet-agent": "v0.3.4",
+  "rancher/fleet": "v0.3.4",
+  "rancher/fluentd": "v0.1.24",
+  "rancher/grafana-grafana": "7.1.5",
+  "rancher/hyperkube": "v1.20.4-rancher1",
+  "rancher/jimmidyson-configmap-reload": "v0.3.0",
+  "rancher/k8s-dns-dnsmasq-nanny": "1.15.2",
+  "rancher/k8s-dns-kube-dns": "1.15.2",
+  "rancher/k8s-dns-node-cache": "1.15.13",
+  "rancher/k8s-dns-sidecar": "1.15.2",
+  "rancher/klipper-lb": "v0.1.2",
+  "rancher/kube-api-auth": "v0.1.4",
+  "rancher/kubectl": "v1.20.4",
+  "rancher/kubernetes-external-dns": "v0.7.3",
+  "rancher/cluster-proportional-autoscaler": "1.8.1",
+  "rancher/library-busybox": "1.32.1",
+  "rancher/library-nginx": "1.19.2-alpine",
+  "rancher/library-traefik": "1.7.19",
+  "rancher/local-path-provisioner": "v0.0.14",
+  "rancher/log-aggregator": "v0.1.7",
+  "rancher/istio-kubectl": "1.5.10",
+  "rancher/metrics-server": "v0.4.1",
+  "rancher/configmap-reload": "v0.3.0-rancher4",
+  "rancher/nginx-ingress-controller-defaultbackend": "1.5-rancher1",
+  "rancher/nginx-ingress-controller": "nginx-0.43.0-rancher1",
+  "rancher/opa-gatekeeper": "v3.1.0-beta.7",
+  "rancher/openzipkin-zipkin": "2.14.2",
+  "rancher/pause": "3.2",
+  "rancher/plugins-docker": "18.09",
+  "rancher/prom-alertmanager": "v0.21.0",
+  "rancher/prom-node-exporter": "v1.0.1",
+  "rancher/prom-prometheus": "v2.18.2",
+  "rancher/prometheus-auth": "v0.2.1",
+  "rancher/rancher-agent": "v2.5.7",
+  "rancher/rancher-webhook": "v0.1.0-beta9",
+  "rancher/rancher": "v2.5.7",
+  "rancher/rke-tools": "v0.1.72",
+  "rancher/security-scan": "v0.1.14",
+  "rancher/shell": "v0.1.6",
+  "rancher/sonobuoy-sonobuoy": "v0.16.3",
+  "rancher/system-upgrade-controller": "v0.6.2"
+}