zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/local-dev'

Browse Source
This commit is contained in:
zeaslity
2024-01-25 11:37:49 +08:00
parent 4ffd48500c 5069372348
commit a09515c82d
7 changed files with 349 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
cmii_operator/CmiiImageSync.go
View File

@@ -3,6 +3,8 @@ package cmii_operator
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types/filters"
"github.com/klauspost/pgzip"
@@ -188,6 +190,82 @@ func ImagePullFromCmiiHarborByMap(imageVersionMap map[string]string, silentMode
}
func ImagePullFromFileJson(filePathName string) {
readFile, err := os.ReadFile(filePathName)
if err != nil {
log.ErrorF("[ImagePullFromFileJson] - file %s read error ! %s", filePathName, err.Error())
return
}
var resultMap map[string]string
err = json.Unmarshal(readFile, &readFile)
if err != nil {
log.ErrorF("[ImagePullFromFileJson] - file %s un marshal error ! %s", filePathName, err.Error())
return
}
for image, tag := range resultMap {
pullResult := ImagePullFromCmiiHarbor(image + ":" + tag)
if pullResult == nil {
continue
}
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
fmt.Println()
}
}
func ImagePullFromList(depContainerList []string) {
for _, dep := range depContainerList {
loginToDockerHub()
pullResult := ImagePullFromCmiiHarbor(dep)
if pullResult == nil {
continue
}
scanner := bufio.NewScanner(pullResult)
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "\"status\":\"Pulling from") {
fmt.Println(line)
}
if strings.Contains(line, "Status: Image is up to date for") {
fmt.Println(line)
}
}
fmt.Println()
}
}
func ImagePullFromListAndCompressSplit(imageNameList []string, gzipFolder string) {
ImagePullFromList(imageNameList)
// generate a project folder
err := os.MkdirAll(gzipFolder, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[ImagePullFromListAndCompressSplit] - create folder of %s error %s", gzipFolder, err.Error())
}
}
for _, image := range imageNameList {
ImageSaveToTarGZ(image, gzipFolder)
}
}
func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
imageGetByName := ImageGetByName(targetImageName)
@@ -236,8 +314,13 @@ func ImageSaveToTarGZ(targetImageName, folderPathPrefix string) bool {
func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
split := strings.Split(imageRepoTag, ":")
first := strings.Split(split[0], "/")
//log.DebugF(" %s to %s", imageRepoTag, split)
if len(split) == 1 {
return "docker=" + imageRepoTag + "=latest.tar.gz"
}
first := strings.Split(split[0], "/")
//log.DebugF(" split[0] %s to %s", split[0], first)
if len(first) == 3 {
if strings.Contains(first[0], "cdcyy") {
gzipFileName += "cmlc="
@@ -250,11 +333,15 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
gzipFileName += first[2]
gzipFileName += "="
} else if len(first) == 2 {
//
gzipFileName += "docker="
gzipFileName += first[0]
gzipFileName += "="
gzipFileName += first[1]
gzipFileName += "="
} else if len(first) == 1 {
//
return "docker=" + split[0] + "=" + split[1] + ".tar.gz"
}
gzipFileName += split[1]
@@ -262,3 +349,18 @@ func convertImageGzipFileName(imageRepoTag string) (gzipFileName string) {
return gzipFileName
}
func loginToDockerHub() {
login, err := apiClient.RegistryLogin(context.TODO(), types.AuthConfig{
Username: "icederce",
Password: "loveff.cxc.23",
Auth: "aWNlZGVyY2U6bG92ZWZmLmN4Yy4yMw==",
ServerAddress: "https://registry-1.docker.io",
})
if err != nil {
log.ErrorF("[loginToDockerHub] - login failed !")
}
log.DebugF("[loginToDockerHub] - login is %s", login.Status)
}

36
cmii_operator/CmiiImageSync_test.go
View File

@@ -104,3 +104,39 @@ func TestImageSaveToTarGZ(t *testing.T) {
assert.Equal(t, imageSaveToTarGZ, true, "image save to tar gz file error !")
}
func TestConvertImageGzipFileName(t *testing.T) {
test := []string{
"bitnami/redis:6.2.6-debian-10-r0",
"bitnami/redis:6.2.14-debian-11-r1",
"bitnami/mysql:8.0.35-debian-11-r1",
"bitnami/mysql:8.1.0-debian-11-r42",
"simonrupf/chronyd:0.4.3",
"bitnami/bitnami-shell:10-debian-10-r140",
"bitnami/bitnami-shell:11-debian-11-r136",
"bitnami/rabbitmq:3.9.12-debian-10-r3",
"bitnami/rabbitmq:3.11.26-debian-11-r2",
"ossrs/srs:v4.0.136",
"emqx/emqx:4.2.12",
"nacos/nacos-server:v2.1.2",
"nacos/nacos-server:v2.1.2-slim",
"mongo:5.0",
"rabbitmq:3.9-management",
"bitnami/minio:2022.5.4",
"bitnami/minio:2023.5.4",
"kubernetesui/dashboard:v2.0.1",
"kubernetesui/metrics-scraper:v1.0.4",
"ossrs/srs:v4.0-r3",
"nginx:1.21.3",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"busybox:latest",
"busybox",
}
for _, s := range test {
gzipFileName := convertImageGzipFileName(s)
t.Logf(" %s to %s", s, gzipFileName)
}
}

39
cmii_operator/CmiiOperator.go
View File

@@ -3,21 +3,19 @@ package cmii_operator
import (
"errors"
"os"
"strings"
"wdd.io/agent-go/executor"
"wdd.io/agent-go/utils"
)
func FetchDemoImages(projectName, projectFolderPath string) bool {
if !strings.HasSuffix(projectFolderPath, "/") {
projectFolderPath += "/"
}
const OfflineImageGzipFolderPrefix = "/root/octopus_image/"
func FetchDemoImages(projectName string, gzipSplit bool) bool {
// generate a project folder
err := os.Mkdir(projectFolderPath, os.ModeDir)
err := os.Mkdir(OfflineImageGzipFolderPrefix+projectName, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", projectFolderPath, err.Error())
log.ErrorF("[FetchDemoImages] - create folder of %s error %s", OfflineImageGzipFolderPrefix+projectName, err.Error())
return false
}
}
@@ -29,8 +27,8 @@ func FetchDemoImages(projectName, projectFolderPath string) bool {
utils.BeautifulPrint(frontendMap)
// save map to file
backendMapFile := projectFolderPath + projectName + "-backend-app.json"
frontendMapFile := projectFolderPath + projectName + "-frontend-app.json"
backendMapFile := OfflineImageGzipFolderPrefix + projectName + "-backend-app.json"
frontendMapFile := OfflineImageGzipFolderPrefix + projectName + "-frontend-app.json"
_ = os.Remove(backendMapFile)
_ = os.Remove(frontendMapFile)
@@ -48,10 +46,31 @@ func FetchDemoImages(projectName, projectFolderPath string) bool {
ImagePullFromCmiiHarborByMap(frontendMap, true)
// compress image
if gzipSplit {
for image, tag := range backendMap {
ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
}
for image, tag := range frontendMap {
ImageSaveToTarGZ(image+":"+tag, OfflineImageGzipFolderPrefix+projectName+"/app/")
}
}
// upload to harbor
// clean up images
return false
}
func FetchDependencyRepos(gzipSplit bool) {
err := os.Mkdir(OfflineImageGzipFolderPrefix, os.ModeDir)
if err != nil {
if !errors.Is(err, os.ErrExist) {
log.ErrorF("[FetchDependencyRepos] - create folder of %s error %s", OfflineImageGzipFolderPrefix, err.Error())
}
}
ImagePullFromListAndCompressSplit(MiddlewareAmd64, OfflineImageGzipFolderPrefix+"middle/")
ImagePullFromListAndCompressSplit(Rancher1204Amd64, OfflineImageGzipFolderPrefix+"rke/")
}

6
cmii_operator/CmiiOperator_test.go
View File

@@ -3,5 +3,9 @@ package cmii_operator
import "testing"
func TestFetchDemoImages(t *testing.T) {
FetchDemoImages("cqga", "/root/Downloads/cqga")
FetchDemoImages("cqga", true)
}
func TestFetchDependencyRepos(t *testing.T) {
FetchDependencyRepos(true)
}

92
cmii_operator/DependencyConfig.go Normal file
View File

@@ -0,0 +1,92 @@
package cmii_operator
var MiddlewareAmd64 = []string{
"bitnami/redis:6.2.6-debian-10-r0",
"bitnami/redis:6.2.14-debian-11-r1",
"bitnami/mysql:8.0.35-debian-11-r1",
"bitnami/mysql:8.1.0-debian-11-r42",
"simonrupf/chronyd:0.4.3",
"bitnami/bitnami-shell:10-debian-10-r140",
"bitnami/bitnami-shell:11-debian-11-r136",
"bitnami/rabbitmq:3.9.12-debian-10-r3",
"bitnami/rabbitmq:3.11.26-debian-11-r2",
"ossrs/srs:v4.0.136",
"emqx/emqx:4.2.12",
"nacos/nacos-server:v2.1.2",
"nacos/nacos-server:v2.1.2-slim",
"mongo:5.0",
"rabbitmq:3.9-management",
"bitnami/minio:2022.5.4",
"bitnami/minio:2023.5.4",
"kubernetesui/dashboard:v2.0.1",
"kubernetesui/metrics-scraper:v1.0.4",
"ossrs/srs:v4.0-r3",
"nginx:1.21.3",
"redis:6.0.20-alpine",
"dyrnq/nfs-subdir-external-provisioner:v4.0.2",
"busybox:latest",
}
var Rancher1204Amd64 = []string{
"rancher/backup-restore-operator:v1.0.3",
"rancher/calico-cni:v3.17.2",
"rancher/calico-ctl:v3.17.2",
"rancher/calico-kube-controllers:v3.17.2",
"rancher/calico-node:v3.17.2",
"rancher/calico-pod2daemon-flexvol:v3.17.2",
"rancher/cis-operator:v1.0.3",
"rancher/cluster-proportional-autoscaler:1.7.1",
"rancher/coredns-coredns:1.8.0",
"rancher/coreos-etcd:v3.4.14-rancher1",
"rancher/coreos-kube-state-metrics:v1.9.7",
"rancher/coreos-prometheus-config-reloader:v0.39.0",
"rancher/coreos-prometheus-operator:v0.39.0",
"rancher/externalip-webhook:v0.1.6",
"rancher/flannel-cni:v0.3.0-rancher6",
"rancher/coreos-flannel:v0.13.0-rancher1",
"rancher/fleet-agent:v0.3.4",
"rancher/fleet:v0.3.4",
"rancher/fluentd:v0.1.24",
"rancher/grafana-grafana:7.1.5",
"rancher/hyperkube:v1.20.4-rancher1",
"rancher/jimmidyson-configmap-reload:v0.3.0",
"rancher/k8s-dns-dnsmasq-nanny:1.15.2",
"rancher/k8s-dns-kube-dns:1.15.2",
"rancher/k8s-dns-node-cache:1.15.13",
"rancher/k8s-dns-sidecar:1.15.2",
"rancher/klipper-lb:v0.1.2",
"rancher/kube-api-auth:v0.1.4",
"rancher/kubectl:v1.20.4",
"rancher/kubernetes-external-dns:v0.7.3",
"rancher/cluster-proportional-autoscaler:1.8.1",
"rancher/library-busybox:1.31.1",
"rancher/library-busybox:1.32.1",
"rancher/library-nginx:1.19.2-alpine",
"rancher/library-traefik:1.7.19",
"rancher/local-path-provisioner:v0.0.11",
"rancher/local-path-provisioner:v0.0.14",
"rancher/local-path-provisioner:v0.0.19",
"rancher/log-aggregator:v0.1.7",
"rancher/istio-kubectl:1.5.10",
"rancher/metrics-server:v0.4.1",
"rancher/configmap-reload:v0.3.0-rancher4",
"rancher/nginx-ingress-controller-defaultbackend:1.5-rancher1",
"rancher/nginx-ingress-controller:nginx-0.43.0-rancher1",
"rancher/opa-gatekeeper:v3.1.0-beta.7",
"rancher/openzipkin-zipkin:2.14.2",
"rancher/pause:3.2",
"rancher/plugins-docker:18.09",
"rancher/prom-alertmanager:v0.21.0",
"rancher/prom-node-exporter:v1.0.1",
"rancher/prom-prometheus:v2.18.2",
"rancher/prometheus-auth:v0.2.1",
"rancher/rancher-agent:v2.5.7",
"rancher/rancher-webhook:v0.1.0-beta9",
"rancher/rancher:v2.5.7",
"rancher/rke-tools:v0.1.72",
"rancher/security-scan:v0.1.14",
"rancher/security-scan:v0.2.2",
"rancher/shell:v0.1.6",
"rancher/sonobuoy-sonobuoy:v0.16.3",
"rancher/system-upgrade-controller:v0.6.2",
}

26
cmii_operator/config/middleware-amd64.json Normal file
View File

@@ -0,0 +1,26 @@
{
"docker.io/bitnami/redis": "6.2.6-debian-10-r0",
"docker.io/bitnami/redis": "6.2.14-debian-11-r1",
"docker.io/bitnami/mysql": "8.0.35-debian-11-r1",
"docker.io/bitnami/mysql": "8.1.0-debian-11-r42",
"docker.io/simonrupf/chronyd": "0.4.3",
"docker.io/bitnami/bitnami-shell": "10-debian-10-r140",
"docker.io/bitnami/bitnami-shell": "11-debian-11-r136",
"docker.io/bitnami/rabbitmq": "3.9.12-debian-10-r3",
"docker.io/bitnami/rabbitmq": "3.11.26-debian-11-r2",
"docker.io/ossrs/srs": "v4.0.136",
"docker.io/emqx/emqx": "4.2.12",
"docker.io/nacos/nacos-server": "v2.1.2",
"docker.io/nacos/nacos-server": "v2.1.2-slim",
"docker.io/mongo": "5.0",
"docker.io/rabbitmq": "3.9-management",
"docker.io/bitnami/minio": "2022.5.4",
"docker.io/bitnami/minio": "2023.5.4",
"docker.io/simonrupf/chronyd": "0.4.3",
"docker.io/kubernetesui/dashboard": "v2.0.1",
"docker.io/kubernetesui/metrics-scraper": "v1.0.4",
"docker.io/ossrs/srs": "v4.0-r3",
"docker.io/nginx": "1.21.3",
"docker.io/redis": "6.0.20-alpine",
"docker.io/dyrnq/nfs-subdir-external-provisioner": "v4.0.2"
}

58
cmii_operator/config/rancher-1.20.4-amd64.json Normal file
View File

@@ -0,0 +1,58 @@
{
"rancher/backup-restore-operator": "v1.0.3",
"rancher/calico-cni": "v3.17.2",
"rancher/calico-ctl": "v3.17.2",
"rancher/calico-kube-controllers": "v3.17.2",
"rancher/calico-node": "v3.17.2",
"rancher/calico-pod2daemon-flexvol": "v3.17.2",
"rancher/cis-operator": "v1.0.3",
"rancher/coredns-coredns": "1.8.0",
"rancher/coreos-etcd": "v3.4.14-rancher1",
"rancher/coreos-kube-state-metrics": "v1.9.7",
"rancher/coreos-prometheus-config-reloader": "v0.39.0",
"rancher/coreos-prometheus-operator": "v0.39.0",
"rancher/externalip-webhook": "v0.1.6",
"rancher/flannel-cni": "v0.3.0-rancher6",
"rancher/coreos-flannel": "v0.13.0-rancher1",
"rancher/fleet-agent": "v0.3.4",
"rancher/fleet": "v0.3.4",
"rancher/fluentd": "v0.1.24",
"rancher/grafana-grafana": "7.1.5",
"rancher/hyperkube": "v1.20.4-rancher1",
"rancher/jimmidyson-configmap-reload": "v0.3.0",
"rancher/k8s-dns-dnsmasq-nanny": "1.15.2",
"rancher/k8s-dns-kube-dns": "1.15.2",
"rancher/k8s-dns-node-cache": "1.15.13",
"rancher/k8s-dns-sidecar": "1.15.2",
"rancher/klipper-lb": "v0.1.2",
"rancher/kube-api-auth": "v0.1.4",
"rancher/kubectl": "v1.20.4",
"rancher/kubernetes-external-dns": "v0.7.3",
"rancher/cluster-proportional-autoscaler": "1.8.1",
"rancher/library-busybox": "1.32.1",
"rancher/library-nginx": "1.19.2-alpine",
"rancher/library-traefik": "1.7.19",
"rancher/local-path-provisioner": "v0.0.14",
"rancher/log-aggregator": "v0.1.7",
"rancher/istio-kubectl": "1.5.10",
"rancher/metrics-server": "v0.4.1",
"rancher/configmap-reload": "v0.3.0-rancher4",
"rancher/nginx-ingress-controller-defaultbackend": "1.5-rancher1",
"rancher/nginx-ingress-controller": "nginx-0.43.0-rancher1",
"rancher/opa-gatekeeper": "v3.1.0-beta.7",
"rancher/openzipkin-zipkin": "2.14.2",
"rancher/pause": "3.2",
"rancher/plugins-docker": "18.09",
"rancher/prom-alertmanager": "v0.21.0",
"rancher/prom-node-exporter": "v1.0.1",
"rancher/prom-prometheus": "v2.18.2",
"rancher/prometheus-auth": "v0.2.1",
"rancher/rancher-agent": "v2.5.7",
"rancher/rancher-webhook": "v0.1.0-beta9",
"rancher/rancher": "v2.5.7",
"rancher/rke-tools": "v0.1.72",
"rancher/security-scan": "v0.1.14",
"rancher/shell": "v0.1.6",
"rancher/sonobuoy-sonobuoy": "v0.16.3",
"rancher/system-upgrade-controller": "v0.6.2"
}