[ Agent ] [ APP ] - accomplish rke and dashboard

2023-11-20 16:13:42 +08:00
parent 72269894db
commit c47e6b5ed1
7 changed files with 416 additions and 22 deletions
--- a/agent-go/executor/AppFunction.go
+++ b/agent-go/executor/AppFunction.go
@@ -1,6 +1,7 @@
 package executor

 import (
+	"net"
 	"time"
 )

@@ -89,6 +90,16 @@ func (op *AgentOsOperator) Deploy(appFuncName string, funcArgs ...string) (bool,
 	return resultOK, result
 }

+func CheckAppInstallFolder() (bool, []string) {
+	if !BasicCreateFolder("/root/wdd/install") {
+		return false, []string{
+			"[CheckAppInstallFolder] - install folder create failed !",
+		}
+	}
+
+	return true, nil
+}
+
 func (op *AgentOsOperator) deployRke(funcArgs []string) (bool, []string) {

 	var ok bool
@@ -115,6 +126,49 @@ func (op *AgentOsOperator) deployRke(funcArgs []string) (bool, []string) {
 }

 func (op *AgentOsOperator) deployK8sDashboard(funcArgs []string) (bool, []string) {
+
+	restartHarborExec, strings := op.restartHarborExec()
+	if !restartHarborExec {
+		return false, strings
+	}
+
+	// check folder
+	folder, i := CheckAppInstallFolder()
+	if !folder {
+		return false, i
+	}
+
+	// download template file
+	k8sDashBoardYamlFile := "/root/wdd/install/k8s-dashboard.yaml"
+	ok, resultLog := BasicDownloadFile(op.OssOfflinePrefix+"k8s-dashboard-template.yaml", k8sDashBoardYamlFile)
+	if !ok {
+		return false, resultLog
+	}
+
+	// kubectl
+
+	// replace
+	parseIP := net.ParseIP(funcArgs[0])
+	if parseIP == nil {
+		return false, []string{
+			"[deployK8sDashboard] - ip args error !",
+		}
+	}
+	BasicReplace(k8sDashBoardYamlFile, "A1C2IP", funcArgs[0])
+
+	// up
+	applyExec, resultLog := KubectlApplyExec(k8sDashBoardYamlFile)
+	if !applyExec {
+		return false, resultLog
+	}
+
+	// check deployment ok
+	if !CheckDeploymentStatusTimeout("kubernetes-dashboard", "kube-system", 120) {
+		return false, []string{
+			"[deployK8sDashboard] - deployment run error !",
+		}
+	}
+
 	return true, nil
 }

@@ -379,27 +433,13 @@ func (op *AgentOsOperator) loadMysqlInitScript(funcArgs []string) (bool, []strin

 func (op *AgentOsOperator) checkMySQL(funcArgs []string) (bool, []string) {

-	// 设置超时时间为 120 秒
-	timeout := time.After(120 * time.Second)
-
-	for {
-		select {
-		case <-timeout:
-			msg := "检查MySQL时间Pod的运行状态超时! 120s"
-			log.Error(msg)
-			return false, []string{msg}
-		default:
-			msg := "MySQL  Pod已经启动!"
-			if CheckPodStatus("helm-mysql-0", funcArgs[0]) {
-				log.Info(msg)
-				return true, []string{msg}
-			}
+	if !CheckPodStatusTimeout("helm-mysql-0", funcArgs[0], 180) {
+		return false, []string{
+			"helm-mysql 启动失败!",
 		}
-
-		// 每隔5s检查一次
-		time.Sleep(5 * time.Second)
 	}

+	return true, nil
 }

 func (op *AgentOsOperator) deployMiddlewares(funcArgs []string) (bool, []string) {
--- a/agent-go/executor/BaseFunction.go
+++ b/agent-go/executor/BaseFunction.go
@@ -1103,6 +1103,107 @@ func (op *AgentOsOperator) installHarborExec() (bool, []string) {
 	return true, []string{msg}
 }

+func (op *AgentOsOperator) restartHarborExec() (bool, []string) {
+
+	// check pods all run
+	allHarborContainerName := []string{
+		"harbor-log",
+		"harbor-db",
+		"harbor-portal",
+		"redis",
+		"registryctl",
+		"registry",
+		"registry",
+		"harbor-core",
+		"nginx",
+		"harbor-jobservice",
+	}
+	harborServiceRunningHealthy := true
+
+	for _, container := range allHarborContainerName {
+		ok, _ := PipelineCommandExecutor([][]string{
+			{
+				"docker",
+				"ps",
+			},
+			{
+				"grep",
+				"-c",
+				container,
+			},
+		})
+		if !ok {
+			harborServiceRunningHealthy = false
+			msg := fmt.Sprintf("[restartHarborExec] - harbor service unhealthy, container [ %s ] not running, prepare to restart !", container)
+			log.Warn(msg)
+			break
+		}
+	}
+
+	if harborServiceRunningHealthy {
+		return true, []string{
+			"[restartHarborExec] - harbor container run healthy !",
+		}
+	}
+
+	// docker-compose.yml exists
+	harborDockerComposeFile := "/root/wdd/harbor/docker-compose.yml"
+	if !BasicFileExistAndNotNull(harborDockerComposeFile) {
+		errorLog := fmt.Sprintf("[restartHarborExec] - harborDockerComposeFile %s not exist !", harborDockerComposeFile)
+		log.Error(errorLog)
+		return false, []string{
+			errorLog,
+		}
+	}
+
+	// execute docker-compose up
+	if !BasicCommandExists("docker-compose") {
+		return false, []string{
+			"[restartHarborExec] - docker-compose not exist !",
+		}
+	}
+
+	log.InfoF("[restartHarborExec] - prepare to restart harbor docker compose !")
+	ok, resultLog := AllCommandExecutor([]string{
+		"docker-compose",
+		"-f",
+		harborDockerComposeFile,
+		"up",
+		"-d",
+	})
+	if !ok {
+		return false, resultLog
+	}
+
+	// wait
+
+	// check pods all run
+	for _, container := range allHarborContainerName {
+		ok, _ := PipelineCommandExecutor([][]string{
+			{
+				"docker",
+				"ps",
+			},
+			{
+				"grep",
+				"-c",
+				container,
+			},
+		})
+		if !ok {
+			errLog := fmt.Sprintf("[restartHarborExec] - harbor service [ %s ] restart failed !", container)
+			log.Error(errLog)
+			return false, []string{
+				errLog,
+			}
+		}
+	}
+
+	return true, []string{
+		"[restartHarborExec] - harbor container restarted and run healthy !",
+	}
+}
+
 func (op *AgentOsOperator) chronyToPublicNTP() [][]string {

 	serverIPInV4 := op.AgentServerInfo.ServerIPInV4
--- a/agent-go/executor/BasicFunction.go
+++ b/agent-go/executor/BasicFunction.go
@@ -78,6 +78,25 @@ func BasicFileExistAndNotNull(filename string) bool {
 	}
 }

+func BasicCreateFolder(folderName string) bool {
+	cmd := exec.Command("test", "-d", folderName)
+	err := cmd.Run()
+	if err == nil {
+		log.DebugF("folder %s already exists !", folderName)
+		return true
+	}
+
+	cmd2 := exec.Command("mkdir", "-p", folderName)
+	err = cmd2.Run()
+	if err != nil {
+		log.ErrorF("folder %s create error!", folderName)
+		return false
+	} else {
+		log.DebugF("folder %s create success!", folderName)
+		return true
+	}
+}
+
 // BasicPrettyPrint 打印执行结果
 func BasicPrettyPrint(resultOk bool, resultLog []string) {
 	fmt.Printf("resultOK is => %#v\n", resultOk)
--- a/agent-go/executor/K8sFunction.go
+++ b/agent-go/executor/K8sFunction.go
@@ -1,8 +1,46 @@
 package executor

-import "strings"
+import (
+	"fmt"
+	"strings"
+	"sync"
+	"time"

-var configFilePath = "/root/.kube/config"
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var k8sConfigFilePath = "/root/.kube/config"
+var k8sClient = newK8sClientInstance()
+
+func newK8sClientInstance() *kubernetes.Clientset {
+	once := sync.Once{}
+
+	if !BasicFileExistAndNotNull(k8sConfigFilePath) {
+		log.WarnF("[newK8sClientInstance] -  k8s config %s does not exist ! ", k8sConfigFilePath)
+		return nil
+	}
+
+	var client *kubernetes.Clientset
+	once.Do(func() {
+		// 使用kubeconfig文件初始化客户端
+		config, err := clientcmd.BuildConfigFromFlags("", k8sConfigFilePath)
+		if err != nil {
+			log.ErrorF("[newK8sClientInstance] - load from %s error !", k8sConfigFilePath)
+
+		}
+
+		client, err = kubernetes.NewForConfig(config)
+		if err != nil {
+			log.Error("[newK8sClientInstance] - create k8s client error !")
+		}
+	})
+
+	return client
+}

 func CheckPodStatus(specific string, supreme string) bool {

@@ -25,3 +63,119 @@ func CheckPodStatus(specific string, supreme string) bool {

 	return false
 }
+
+func CheckPodStatusTimeout(specificPod string, supreme string, waitTimeOut int) bool {
+
+	// 设置超时时间和时间间隔
+	timeout := time.After(time.Duration(waitTimeOut) * time.Second)
+	tick := time.Tick(5 * time.Second)
+
+	// 监控Pod状态
+	for {
+		select {
+		case <-timeout:
+			log.ErrorF("[CheckPodStatusTimeout] - 命名空间: %s, Pod名称: %s, 状态: 失败!\n", supreme, specificPod)
+			return false
+		case <-tick:
+			pod, err := k8sClient.CoreV1().Pods(supreme).Get(context.TODO(), specificPod, metav1.GetOptions{})
+			if err != nil {
+				log.ErrorF("[CheckPodStatusTimeout] - 获取Pod信息失败: %v\n", err)
+			} else {
+				log.DebugF("[CheckPodStatusTimeout] - 命名空间: %s, Pod名称: %s, 状态: %s\n", supreme, pod.Name, pod.Status.Phase)
+				if pod.Status.Phase == "Running" {
+					return true
+				}
+			}
+		}
+	}
+}
+
+func CheckDeploymentStatusTimeout(specificDeployment string, supreme string, waitTimeOut int) bool {
+
+	// 设置超时时间和时间间隔
+	timeout := time.After(time.Duration(waitTimeOut) * time.Second)
+	tick := time.Tick(5 * time.Second)
+
+	// 监控Pod状态
+	for {
+		select {
+		case <-timeout:
+			log.ErrorF("[CheckDeploymentStatusTimeout] - 命名空间: %s, Deployment名称: %s, 状态: 失败!\n", supreme, specificDeployment)
+			return false
+		case <-tick:
+			deployment, err := k8sClient.AppsV1().Deployments(supreme).Get(context.TODO(), specificDeployment, metav1.GetOptions{})
+			if err != nil {
+				log.ErrorF("[CheckDeploymentStatusTimeout] - 获取deployment信息失败: %v\n", err)
+			} else {
+				log.DebugF("[CheckDeploymentStatusTimeout] - 命名空间: %s, Deployment %s 还有Pods未处于Running状态 (Ready: %d, Total: %d)\n", supreme, deployment.Name, deployment.Status.ReadyReplicas, deployment.Status.Replicas)
+
+				if deployment.Status.ReadyReplicas == deployment.Status.Replicas {
+					return true
+				}
+
+			}
+		}
+	}
+}
+
+func KubectlApplyExec(resourcesYamlFile string) (bool, []string) {
+
+	// check kubectl
+	if !BasicCommandExists("kubectl") {
+		return false, []string{
+			"[KubectlApplyExec] - kubectl command not exist !",
+		}
+	}
+
+	// check resourcesYamlFile
+	if !BasicFileExistAndNotNull(resourcesYamlFile) {
+		return false, []string{
+			fmt.Sprintf("[KubectlApplyExec] - wrong resourcesYamlFile %s not exist !", resourcesYamlFile),
+		}
+	}
+
+	// apply -f
+	ok, resultLog := AllCommandExecutor([]string{
+		"kubectl",
+		"apply",
+		"-f",
+		resourcesYamlFile,
+	})
+	if !ok {
+		return false, resultLog
+	}
+
+	return true, append(resultLog,
+		fmt.Sprintf("[KubectlApplyExec] - %s apply success!", resourcesYamlFile))
+}
+
+func KubectlDeleteExec(resourcesYamlFile string) (bool, []string) {
+
+	// check kubectl
+	if !BasicCommandExists("kubectl") {
+		return false, []string{
+			"[KubectlDeleteExec] - kubectl command not exist !",
+		}
+	}
+
+	// check resourcesYamlFile
+	if !BasicFileExistAndNotNull(resourcesYamlFile) {
+		return false, []string{
+			fmt.Sprintf("[KubectlDeleteExec] - wrong resourcesYamlFile %s not exist !", resourcesYamlFile),
+		}
+	}
+
+	// apply -f
+	ok, resultLog := AllCommandExecutor([]string{
+		"kubectl",
+		"delete",
+		"-f",
+		resourcesYamlFile,
+	})
+	if !ok {
+		return false, resultLog
+	}
+
+	return true, append(resultLog,
+		fmt.Sprintf("[KubectlDeleteExec] - %s delete success!", resourcesYamlFile))
+}