From 1f7138d541d35a44d78c641762a44b348ed79a5b Mon Sep 17 00:00:00 2001 From: zeaslity Date: Wed, 22 Nov 2023 10:49:11 +0800 Subject: [PATCH] [ Agent ] [ App ] - k8s-pvc --- agent-go/executor/AppFunction.go | 64 ++++-- agent-go/executor/K8sFunction.go | 186 ++++++++++++------ .../func/auto/service/AppFuncScheduler.java | 6 +- 3 files changed, 180 insertions(+), 76 deletions(-) diff --git a/agent-go/executor/AppFunction.go b/agent-go/executor/AppFunction.go index 2464571..119bd60 100644 --- a/agent-go/executor/AppFunction.go +++ b/agent-go/executor/AppFunction.go @@ -44,7 +44,7 @@ func (op *AgentOsOperator) Deploy(appFuncName string, funcArgs ...string) (bool, case "DEPLOY_TEST_NFS": resultOK, result = op.deployTestNFS(funcArgs) break - case "deployPVC": + case "DEPLOY_K8S_PVC": resultOK, result = op.deployPVC(funcArgs) break case "DEPLOY_STORAGE_CLASS": @@ -198,7 +198,7 @@ func (op *AgentOsOperator) deployK8sDashboard(funcArgs []string) (bool, []string } // check deployment ok - if !CheckDeploymentStatusTimeout("kubernetes-dashboard", "kube-system", 120) { + if !K8sCheckDeploymentStatusTimeout("kubernetes-dashboard", "kube-system", 120) { return false, []string{ "[deployK8sDashboard] - deployment run error !", } @@ -209,14 +209,14 @@ func (op *AgentOsOperator) deployK8sDashboard(funcArgs []string) (bool, []string func (op *AgentOsOperator) deployK8sNamespace(funcArgs []string) (bool, []string) { - if !CreateK8sNamespace(funcArgs[1]) { + if !K8sCreateNamespace(funcArgs[1]) { return false, []string{ - fmt.Sprintf("Namespace of %s create error!", funcArgs[1]), + fmt.Sprintf("Namespace of [%s] create error!", funcArgs[1]), } } return true, []string{ - fmt.Sprintf("Namespace of %s create success !", funcArgs[1]), + fmt.Sprintf("Namespace of [%s]create success !", funcArgs[1]), } } @@ -313,7 +313,7 @@ func (op *AgentOsOperator) deployNFS(funcArgs []string) (bool, []string) { } // check running status - if !CheckDeploymentStatusTimeout("nfs-client-provisioner", "kube-system", 60) { + if !K8sCheckDeploymentStatusTimeout("nfs-client-provisioner", "kube-system", 60) { return false, []string{ "[deployNFS] - nfs running error !", } @@ -365,7 +365,7 @@ func (op *AgentOsOperator) deployTestNFS(funcArgs []string) (bool, []string) { result = append(result, "NFS 文件写入 异常！！") } - if !CheckPodStatusTimeout("test-pod", "default", 30) { + if !K8sCheckPodStatusTimeout("test-pod", "default", 30) { return false, []string{ "[deployTestNFS] - test pod create failed !", } @@ -389,27 +389,41 @@ func (op *AgentOsOperator) deployPVC(funcArgs []string) (bool, []string) { } // 下载模板文件 - k8sNFSYamlFile := "/root/wdd/install/k8s-pvc.yaml" - ok, resultLog := BasicDownloadFile(op.OssOfflinePrefix+pvcTemplateFile, k8sNFSYamlFile) + k8sPvcYamlFile := "/root/wdd/install/k8s-pvc.yaml" + ok, resultLog := BasicDownloadFile(op.OssOfflinePrefix+pvcTemplateFile, k8sPvcYamlFile) if !ok { return false, resultLog } // 根据参数 A1C2IP 替换 - if !BasicReplace(k8sNFSYamlFile, "SUPREME", funcArgs[1]) { + if !BasicReplace(k8sPvcYamlFile, "SUPREME", funcArgs[1]) { result = append(result, "替换SUPREME信息") return false, result } // 启动服务 - exec, strings := KubectlApplyExec(k8sNFSYamlFile) + exec, strings := KubectlApplyExec(k8sPvcYamlFile) if !exec { return false, strings } - // 成功启动 - return true, []string{ - "PVC部署成功！", + // check status + ok, pvcList := K8sListPVCInNamespace(funcArgs[1]) + if !ok { + return false, []string{ + "[deployPVC] - pvc list error ! can not check status !", + } } + + for _, pvcName := range pvcList { + if !K8sCheckPVCStatusTimeOut(pvcName, funcArgs[1], 30) { + return false, []string{ + fmt.Sprintf("[deployPVC] - pvc [%s] in namepsace [%s] running error !", pvcName, funcArgs[1]), + } + } + } + + // 成功启动 + return true, append(pvcList, "PVC部署成功！") } func (op *AgentOsOperator) deployMySQL(funcArgs []string) (bool, []string) { @@ -452,6 +466,13 @@ func (op *AgentOsOperator) deployMySQL(funcArgs []string) (bool, []string) { return false, append(result, strings...) } + // check mysql + if !K8sCheckPodStatusTimeout("helm-mysql-0", funcArgs[0], 180) { + return false, []string{ + "helm-mysql-0 启动失败!", + } + } + // 成功启动 return true, []string{ "MySQL部署成功！", @@ -464,7 +485,7 @@ func (op *AgentOsOperator) loadMysqlInitScript(funcArgs []string) (bool, []strin func (op *AgentOsOperator) checkMySQL(funcArgs []string) (bool, []string) { - if !CheckPodStatusTimeout("helm-mysql-0", funcArgs[0], 180) { + if !K8sCheckPodStatusTimeout("helm-mysql-0", funcArgs[0], 180) { return false, []string{ "helm-mysql 启动失败!", } @@ -560,6 +581,19 @@ func (op *AgentOsOperator) deployRedis(funcArgs []string) (bool, []string) { return false, append(result, strings...) } + // check status + if !K8sCheckPodStatusTimeout("helm-redis-master-0", funcArgs[1], 120) { + return false, []string{ + "[deployRedis] - redis-master running failed ! please cheak !", + } + } + + if !K8sCheckPodStatusTimeout("helm-redis-replicas-0", funcArgs[1], 120) { + return false, []string{ + "[deployRedis] - redis-replicas running failed ! please check !", + } + } + // 成功启动 return true, []string{ "Redis 部署成功！", diff --git a/agent-go/executor/K8sFunction.go b/agent-go/executor/K8sFunction.go index eec51cd..a5510b5 100644 --- a/agent-go/executor/K8sFunction.go +++ b/agent-go/executor/K8sFunction.go @@ -45,7 +45,117 @@ func newK8sClientInstance() *kubernetes.Clientset { return client } -func CheckPodStatus(specific string, supreme string) bool { +func K8sCheckPodStatusTimeout(specificPod string, supreme string, waitTimeOut int) bool { + + // 设置超时时间和时间间隔 + timeout := time.After(time.Duration(waitTimeOut) * time.Second) + tick := time.Tick(5 * time.Second) + + // 监控Pod状态 + for { + select { + case <-timeout: + log.ErrorF("[K8sCheckPodStatusTimeout] - 命名空间: [%s], Pod名称: [%s], 状态: 失败!", supreme, specificPod) + return false + case <-tick: + pod, err := k8sClient.CoreV1().Pods(supreme).Get(context.TODO(), specificPod, metav1.GetOptions{}) + if err != nil { + log.ErrorF("[K8sCheckPodStatusTimeout] - 获取Pod信息失败: %s", err.Error()) + } else { + log.DebugF("[K8sCheckPodStatusTimeout] - 命名空间: [%s], Pod名称: [%s], 状态: [%s]", supreme, pod.Name, pod.Status.Phase) + if pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodSucceeded { + return true + } + } + } + } +} + +func K8sCheckDeploymentStatusTimeout(specificDeployment string, supreme string, waitTimeOut int) bool { + + // 设置超时时间和时间间隔 + timeout := time.After(time.Duration(waitTimeOut) * time.Second) + tick := time.Tick(5 * time.Second) + + // 监控Pod状态 + for { + select { + case <-timeout: + log.ErrorF("[K8sCheckDeploymentStatusTimeout] - 命名空间: %s, Deployment名称: %s, 状态: 失败!\n", supreme, specificDeployment) + return false + case <-tick: + deployment, err := k8sClient.AppsV1().Deployments(supreme).Get(context.TODO(), specificDeployment, metav1.GetOptions{}) + if err != nil { + log.ErrorF("[K8sCheckDeploymentStatusTimeout] - 获取deployment信息失败: %v ", err) + } else { + log.DebugF("[K8sCheckDeploymentStatusTimeout] - 命名空间: [ %s ], Deployment: [ %s ] 还有Pods未处于Running状态 (Ready: %d, Total: %d)\n", supreme, deployment.Name, deployment.Status.ReadyReplicas, deployment.Status.Replicas) + + if deployment.Status.ReadyReplicas == deployment.Status.Replicas { + return true + } + + } + } + } +} + +func K8sListPVCInNamespace(supreme string) (bool, []string) { + + if k8sClient == nil { + log.ErrorF("k8s client is nil, run k8s function error !") + return false, nil + } + + pvcs, err := k8sClient.CoreV1().PersistentVolumeClaims(supreme).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + log.ErrorF("[K8sListPVCInNamespace] - error list pvc list in namespace %s", supreme) + return false, nil + } + + var pvcList []string + for _, pvc := range pvcs.Items { + pvcList = append(pvcList, pvc.Name) + } + + log.DebugF("K8sListPVCInNamespace - all pvc in namespace of [ %s ] are %v", supreme, pvcList) + return true, pvcList +} + +func K8sCheckPVCStatusTimeOut(specificPvcName string, supreme string, waitTimeOut int) bool { + + if k8sClient == nil { + log.ErrorF("k8s client is nil, run k8s function error !") + return false + } + + // 设置超时时间和时间间隔 + timeout := time.After(time.Duration(waitTimeOut) * time.Second) + tick := time.Tick(5 * time.Second) + + // 监控Pod状态 + for { + select { + case <-timeout: + log.ErrorF("[K8sCheckPVCStatusTimeOut] - 命名空间: %s, PVC 名称: %s, 状态: 失败! ", supreme, specificPvcName) + return false + case <-tick: + pvc, err := k8sClient.CoreV1().PersistentVolumeClaims(supreme).Get(context.TODO(), specificPvcName, metav1.GetOptions{}) + if err != nil { + log.ErrorF("[K8sCheckPVCStatusTimeOut] - 获取 PVC 信息失败: %v ", err) + } + + if pvc.Status.Phase == "Running" { + log.DebugF("K8sCheckPVCStatusTimeOut] - PVC %s in namespace %s is running", specificPvcName, supreme) + return true + } else { + log.WarnF("K8sCheckPVCStatusTimeOut] - PVC %s in namespace %s run failed !", specificPvcName, supreme) + return false + } + } + } +} + +func KubectlCheckPodStatus(specific string, supreme string) bool { if !BasicCommandExists("kubectl") { log.Error("kubectl命令不存在，无法查看Pod状态，请查看！") @@ -67,60 +177,6 @@ func CheckPodStatus(specific string, supreme string) bool { return false } -func CheckPodStatusTimeout(specificPod string, supreme string, waitTimeOut int) bool { - - // 设置超时时间和时间间隔 - timeout := time.After(time.Duration(waitTimeOut) * time.Second) - tick := time.Tick(5 * time.Second) - - // 监控Pod状态 - for { - select { - case <-timeout: - log.ErrorF("[CheckPodStatusTimeout] - 命名空间: [%s], Pod名称: [%s], 状态: 失败!\n", supreme, specificPod) - return false - case <-tick: - pod, err := k8sClient.CoreV1().Pods(supreme).Get(context.TODO(), specificPod, metav1.GetOptions{}) - if err != nil { - log.ErrorF("[CheckPodStatusTimeout] - 获取Pod信息失败: %s", err.Error()) - } else { - log.DebugF("[CheckPodStatusTimeout] - 命名空间: [%s], Pod名称: [%s], 状态: [%s]\n", supreme, pod.Name, pod.Status.Phase) - if pod.Status.Phase == corev1.PodRunning || pod.Status.Phase == corev1.PodSucceeded { - return true - } - } - } - } -} - -func CheckDeploymentStatusTimeout(specificDeployment string, supreme string, waitTimeOut int) bool { - - // 设置超时时间和时间间隔 - timeout := time.After(time.Duration(waitTimeOut) * time.Second) - tick := time.Tick(5 * time.Second) - - // 监控Pod状态 - for { - select { - case <-timeout: - log.ErrorF("[CheckDeploymentStatusTimeout] - 命名空间: %s, Deployment名称: %s, 状态: 失败!\n", supreme, specificDeployment) - return false - case <-tick: - deployment, err := k8sClient.AppsV1().Deployments(supreme).Get(context.TODO(), specificDeployment, metav1.GetOptions{}) - if err != nil { - log.ErrorF("[CheckDeploymentStatusTimeout] - 获取deployment信息失败: %v\n", err) - } else { - log.DebugF("[CheckDeploymentStatusTimeout] - 命名空间: [ %s ], Deployment: [ %s ] 还有Pods未处于Running状态 (Ready: %d, Total: %d)\n", supreme, deployment.Name, deployment.Status.ReadyReplicas, deployment.Status.Replicas) - - if deployment.Status.ReadyReplicas == deployment.Status.Replicas { - return true - } - - } - } - } -} - func KubectlApplyExec(resourcesYamlFile string) (bool, []string) { // check kubectl @@ -183,23 +239,33 @@ func KubectlDeleteExec(resourcesYamlFile string) (bool, []string) { fmt.Sprintf("[KubectlDeleteExec] - %s delete success!", resourcesYamlFile)) } -func CreateK8sNamespace(namespaceName string) bool { +func K8sCreateNamespace(namespaceName string) bool { if k8sClient == nil { log.ErrorF("k8s client is nil, run k8s function error !") return false } + namespace, err := k8sClient.CoreV1().Namespaces().Get(context.TODO(), namespaceName, metav1.GetOptions{}) + if err != nil { + log.ErrorF("Error getting namespace: %s ", err.Error()) + return false + } + if namespace != nil { + log.InfoF("[K8sCreateNamespace] - namespace of [%s] already exists!", namespaceName) + return true + } + // create namespace // 创建命名空间对象 - namespace := &corev1.Namespace{ + namespace = &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: namespaceName, }, } // 使用客户端创建命名空间 - _, err := k8sClient.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) + namespace, err = k8sClient.CoreV1().Namespaces().Create(context.TODO(), namespace, metav1.CreateOptions{}) if err != nil { - log.ErrorF("Error creating namespace: %s \n", err.Error()) + log.ErrorF("Error getting namespace: %s ", err.Error()) return false } @@ -222,7 +288,7 @@ func CreateK8sNamespace(namespaceName string) bool { return true } -func GetK8sDashBoardAuthKey() { +func K8sGetDashBoardAuthKey() { // 获取 kube-system 命名空间的 secrets 列表 secrets, err := k8sClient.CoreV1().Secrets("kube-system").List(context.TODO(), metav1.ListOptions{}) diff --git a/server/src/main/java/io/wdd/func/auto/service/AppFuncScheduler.java b/server/src/main/java/io/wdd/func/auto/service/AppFuncScheduler.java index 6da99c9..8c7572b 100644 --- a/server/src/main/java/io/wdd/func/auto/service/AppFuncScheduler.java +++ b/server/src/main/java/io/wdd/func/auto/service/AppFuncScheduler.java @@ -95,7 +95,11 @@ public class AppFuncScheduler { List appFunctionEnumList = List.of( // AppFunctionEnum.DEPLOY_RKE - AppFunctionEnum.DEPLOY_TEST_NFS +// AppFunctionEnum.DEPLOY_TEST_NFS +// AppFunctionEnum.DEPLOY_K8S_NAMESPACE, +// AppFunctionEnum.DEPLOY_K8S_MYSQL, + + AppFunctionEnum.DEPLOY_K8S_REDIS );