From 487e808c58e2f59b0db586971a1f731dc5695a6b Mon Sep 17 00:00:00 2001 From: zeaslity Date: Fri, 3 Feb 2023 14:58:10 +0800 Subject: [PATCH] [ server ] [ status ]- update some code - 2 --- .../beans/status/OctopusStatusMessage.java | 1 + .../wdd/rpc/init/ServerBootUpEnvironment.java | 28 +++++++++++++------ .../service/status/MonitorAllAgentStatus.java | 23 +++++++-------- .../io/wdd/source/shell/lib/wdd-lib-sys.sh | 6 ++-- 4 files changed, 35 insertions(+), 23 deletions(-) diff --git a/common/src/main/java/io/wdd/common/beans/status/OctopusStatusMessage.java b/common/src/main/java/io/wdd/common/beans/status/OctopusStatusMessage.java index fb1c677..d58d77a 100644 --- a/common/src/main/java/io/wdd/common/beans/status/OctopusStatusMessage.java +++ b/common/src/main/java/io/wdd/common/beans/status/OctopusStatusMessage.java @@ -12,6 +12,7 @@ import lombok.experimental.SuperBuilder; public class OctopusStatusMessage { // below two will be used by both server and agent + // 存储所有Agent的实时健康状态， 1代表健康 0代表失败 public static final String ALL_AGENT_STATUS_REDIS_KEY = "ALL_AGENT_HEALTHY_STATUS"; public static final String HEALTHY_STATUS_MESSAGE_TYPE = "ping"; public static final String ALL_STATUS_MESSAGE_TYPE = "all"; diff --git a/server/src/main/java/io/wdd/rpc/init/ServerBootUpEnvironment.java b/server/src/main/java/io/wdd/rpc/init/ServerBootUpEnvironment.java index cd66ccd..6d86615 100644 --- a/server/src/main/java/io/wdd/rpc/init/ServerBootUpEnvironment.java +++ b/server/src/main/java/io/wdd/rpc/init/ServerBootUpEnvironment.java @@ -6,9 +6,9 @@ import io.wdd.common.utils.TimeUtils; import io.wdd.server.beans.vo.ServerInfoVO; import io.wdd.server.coreService.CoreServerService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; -import org.springframework.util.Assert; import javax.annotation.PostConstruct; import javax.annotation.Resource; @@ -56,7 +56,7 @@ public class ServerBootUpEnvironment { /** * 保存所有健康运行的Agent Topic Name */ - public static List ALL_HEALTHY_AGENT_TOPIC_NAME_LIST; + public static final List ALL_HEALTHY_AGENT_TOPIC_NAME_LIST = new ArrayList<>(); @Resource CoreServerService coreServerService; @@ -68,21 +68,21 @@ public class ServerBootUpEnvironment { public void GenerateAllCache() { //所有Agent的TopicName ALL_AGENT_TOPIC_NAME_SET - updateAllAgentTopicNameSetCache(); + updateAllAgentTopicNameCache(); // Agent状态信息的两个Map updateAgentStatusMapCache(); } - public void updateAllAgentTopicNameSetCache() { + public void updateAllAgentTopicNameCache() { List allAgentInfo = coreServerService.serverGetAll(); - Assert.notEmpty( - allAgentInfo, - "not agent registered ! skip the agent healthy status check !" - ); + if (CollectionUtils.isEmpty(allAgentInfo)) { + log.warn("[Serer Boot Up] Octopus Serer First Boot Up ! No Agent Registered Ever!"); + return; + } ALL_AGENT_TOPIC_NAME_LIST.clear(); ALL_AGENT_TOPIC_NAME_SET.clear(); @@ -101,13 +101,23 @@ public class ServerBootUpEnvironment { public void updateAgentStatusMapCache() { + if (CollectionUtils.isEmpty(ALL_AGENT_TOPIC_NAME_LIST)) { + log.warn("[Serer Boot Up] Octopus Serer First Boot Up ! No Agent Registered Ever!"); + return; + } + List statusList = redisTemplate .opsForHash() .multiGet( ALL_AGENT_STATUS_REDIS_KEY, ALL_AGENT_TOPIC_NAME_LIST ); - // current log to console is ok + + // 初始话 还没有状态的情况，直接return + if (CollectionUtils.isEmpty(statusList)) { + log.warn("agent status from redis is empty !"); + return; + } // 结构保存为agentStatusMap ==> agent-topic-name : STATUS(healthy, failed, unknown) HashMap agentStatusMap = new HashMap<>(32); diff --git a/server/src/main/java/io/wdd/rpc/scheduler/service/status/MonitorAllAgentStatus.java b/server/src/main/java/io/wdd/rpc/scheduler/service/status/MonitorAllAgentStatus.java index 4af407d..2267547 100644 --- a/server/src/main/java/io/wdd/rpc/scheduler/service/status/MonitorAllAgentStatus.java +++ b/server/src/main/java/io/wdd/rpc/scheduler/service/status/MonitorAllAgentStatus.java @@ -1,15 +1,14 @@ package io.wdd.rpc.scheduler.service.status; -import io.wdd.common.beans.status.AgentHealthyStatusEnum; import io.wdd.common.beans.status.OctopusStatusMessage; import io.wdd.common.utils.TimeUtils; import io.wdd.rpc.init.ServerBootUpEnvironment; import io.wdd.rpc.scheduler.service.BuildStatusScheduleTask; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; import org.springframework.context.annotation.Lazy; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; -import org.springframework.util.CollectionUtils; import javax.annotation.Resource; import java.util.HashMap; @@ -20,7 +19,6 @@ import java.util.stream.Collectors; import static io.wdd.common.beans.status.OctopusStatusMessage.ALL_AGENT_STATUS_REDIS_KEY; import static io.wdd.common.beans.status.OctopusStatusMessage.HEALTHY_STATUS_MESSAGE_TYPE; import static io.wdd.rpc.init.ServerBootUpEnvironment.ALL_AGENT_TOPIC_NAME_LIST; -import static io.wdd.rpc.init.ServerBootUpEnvironment.STATUS_AGENT_LIST_MAP; /** * 更新频率被类 BuildStatusScheduleTask.class控制 @@ -59,6 +57,12 @@ public class MonitorAllAgentStatus { try { // 1. 获取所有注册的Agent 手动更新 + serverBootUpEnvironment.updateAllAgentTopicNameCache(); + if (CollectionUtils.isEmpty(ALL_AGENT_TOPIC_NAME_LIST)) { + log.warn("[Scheduler] No Agent Registered ! End Up Status Monitor !"); + return; + } + // 1.1 检查 Agent状态保存数据结构是否正常 checkOrCreateRedisHealthyKey(); @@ -126,6 +130,7 @@ public class MonitorAllAgentStatus { ) .collect(Collectors.toList()); + // 发送信息 collectAgentStatus.statusMessageToAgent(collect); } @@ -139,17 +144,13 @@ public class MonitorAllAgentStatus { // 执行Metric上报定时任务 buildStatusScheduleTask.buildAgentMetricScheduleTask(); - // update time - AGENT_HEALTHY_INIT_MAP.put( - "updateTime", - currentTimeString - ); - + // 这里仅仅是更新时间 redisTemplate .opsForHash() - .putAll( + .put( ALL_AGENT_STATUS_REDIS_KEY, - AGENT_HEALTHY_INIT_MAP + "updateTime", + currentTimeString ); } diff --git a/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh b/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh index c9c722c..b68862a 100644 --- a/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh +++ b/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh @@ -138,9 +138,9 @@ tmp () { gcloud compute instances create octopus-agent-2c-4g-1 --project=compact-lacing-371804 --zone=asia-northeast1-b --machine-type=n2d-custom-2-4096 --network-interface=network-tier=PREMIUM,subnet=default --metadata=startup-script=wget\ https://raw.githubusercontent.com/zeaslity/ProjectOctopus/main/source/src/main/java/io/wdd/source/shell/agent-bootup.sh\ \&\&\ chmod\ \+x\ agent-bootup.sh\ \&\&\ /bin/bash\ agent-bootup.sh --can-ip-forward --maintenance-policy=MIGRATE --provisioning-model=STANDARD --service-account=172889627951-compute@developer.gserviceaccount.com --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --create-disk=auto-delete=yes,boot=yes,device-name=octopus-agent-2c-4g,image=projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221213,mode=rw,size=20,type=projects/compact-lacing-371804/zones/us-west4-b/diskTypes/pd1-ssd --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --reservation-affinity=any -gcloud compute instances create tokyo-amd64-03 --project=compact-lacing-371804 --zone=asia-northeast1-b --machine-type=n2d-custom-2-4096 --network-interface=network-tier=PREMIUM,subnet=default --can-ip-forward --maintenance-policy=MIGRATE --provisioning-model=STANDARD --service-account=172889627951-compute@developer.gserviceaccount.com --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --create-disk=auto-delete=yes,boot=yes,device-name=octopus-agent-2c-4g,image=projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221213,mode=rw,size=20,type=projects/compact-lacing-371804/zones/us-west4-b/diskTypes/pd-ssd --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --reservation-affinity=any +gcloud compute instances create tokyo-amd64-13 --project=compact-lacing-371804 --zone=asia-northeast1-b --machine-type=n2d-custom-2-4096 --network-interface=network-tier=PREMIUM,subnet=default --can-ip-forward --maintenance-policy=MIGRATE --provisioning-model=STANDARD --service-account=172889627951-compute@developer.gserviceaccount.com --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --create-disk=auto-delete=yes,boot=yes,device-name=octopus-agent-2c-4g,image=projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221213,mode=rw,size=20,type=projects/compact-lacing-371804/zones/us-west4-b/diskTypes/pd-ssd --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --reservation-affinity=any -gcloud compute ssh --zone "asia-northeast1-b" "tokyo-amd64-03" --project "compact-lacing-371804" +gcloud compute ssh --zone "asia-northeast1-b" "tokyo-amd64-13" --project "compact-lacing-371804" gcloud compute instances list --project "compact-lacing-371804" @@ -149,7 +149,7 @@ wget https://raw.githubusercontent.com/zeaslity/ProjectOctopus/main/source/src/m echo "y -" | gcloud compute instances delete tokyo-amd64-03 --project=compact-lacing-371804 --zone=asia-northeast1-b +" | gcloud compute instances delete tokyo-amd64-13 --project=compact-lacing-371804 --zone=asia-northeast1-b apt-cache madison openjdk-11-jdk | head -n 1 | awk '{print$3}'