zeaslity/ProjectOctopus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[server] [ scheduler.script ] - 优化定时脚本任务部分的代码

Browse Source
This commit is contained in:
zeaslity
2023-02-16 14:21:17 +08:00
parent 864d290fe7
commit 2f8f586b0e
13 changed files with 133 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
agent/all-agent-hosts.txt
View File

@@ -34,7 +34,9 @@
141.147.106.62 London-amd64-02 london-2 l2
132.145.12.135 London-arm64-01 london-3 l3
132.145.28.48 London-arm64-02 london-4 l4
# 所有的主机
Phoenix-amd64-02,Phoenix-arm64-01,Phoenix-arm64-02,London-amd64-01,London-amd64-02,London-arm64-02
Phoenix-amd64-02,Phoenix-arm64-01,Phoenix-arm64-02,London-amd64-01,London-amd64-02,London-arm64-01,London-arm64-02

15
server/src/main/java/io/wdd/func/controller/XrayController.java
View File

@@ -40,7 +40,7 @@ public class XrayController {
)
);
/*ArrayList<ProxyNode> pathB = new ArrayList<>(
ArrayList<ProxyNode> pathB = new ArrayList<>(
Arrays.asList(
shanghai,
seoul2,
@@ -53,9 +53,16 @@ public class XrayController {
seoul2,
phoenix2
)
);*/
);
allNetworkPathList.add(pathA);
ArrayList<ProxyNode> pathD = new ArrayList<>(
Arrays.asList(
chengduAgent
)
);
// allNetworkPathList.add(pathA);
allNetworkPathList.add(pathD);
// allNetworkPathList.add(pathB);
// allNetworkPathList.add(pathC);
@@ -63,6 +70,8 @@ public class XrayController {
xrayConfigDistribute.uploadXrayConfigToOSS(allNetworkPathList.get(0));
xrayConfigDistribute.buildXrayUpdateResult(allNetworkPathList.get(0));
System.out.println("结束！");
}

7
server/src/main/java/io/wdd/func/xray/beans/node/ProxyNode.java
View File

@@ -7,8 +7,6 @@ import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.SuperBuilder;
import java.io.File;
@Data
@AllArgsConstructor
@NoArgsConstructor
@@ -26,6 +24,9 @@ public class ProxyNode {
*/
int num;
String agentName;
String agentTopicName;
String location;
@@ -41,6 +42,6 @@ public class ProxyNode {
@Override
public String toString() {
return "ProxyNode: [ " + agentTopicName + " ]";
return "ProxyNode: [ " + agentName + " ]";
}
}

29
server/src/main/java/io/wdd/func/xray/beans/node/ProxyNodeSet.java
View File

@@ -14,6 +14,8 @@ public class ProxyNodeSet {
public static ProxyNode phoenix2;
public static ProxyNode london2;
public static ProxyNode chengduAgent;
static {
@@ -24,7 +26,7 @@ public class ProxyNodeSet {
.publicIPv4("43.136.177.228")
.proxyNodeType(ProxyNodeType.INTERFACE)
.name("cd-agent")
.agentTopicName("Chengdu-amd-01")
.agentName("Chengdu-amd-01")
.build();
@@ -35,7 +37,7 @@ public class ProxyNodeSet {
.publicIPv4("42.192.52.227")
.proxyNodeType(ProxyNodeType.RELAY)
.name("tc-sh")
.agentTopicName("Shanghai-amd64-01")
.agentName("Shanghai-amd64-01")
.build();
hongkong = ProxyNode
@@ -45,7 +47,7 @@ public class ProxyNodeSet {
.publicIPv4("43.154.83.213")
.proxyNodeType(ProxyNodeType.EXTERNAL)
.name("tc-hk")
.agentTopicName("Hongkong-amd64-01")
.agentName("Hongkong-amd64-01")
.build();
@@ -56,7 +58,7 @@ public class ProxyNodeSet {
.publicIPv4("140.238.30.110")
.proxyNodeType(ProxyNodeType.EXTERNAL)
.name("seoul-2")
.agentTopicName("Seoul-amd64-02")
.agentName("Seoul-amd64-02")
.build();
@@ -67,7 +69,7 @@ public class ProxyNodeSet {
.publicIPv4("140.238.52.228")
.proxyNodeType(ProxyNodeType.EXTERNAL)
.name("tokyo-2")
.agentTopicName("Tokyo-amd64-02")
.agentName("Tokyo-amd64-02")
.build();
phoenix2 = ProxyNode
@@ -77,7 +79,7 @@ public class ProxyNodeSet {
.publicIPv4("129.146.171.163")
.proxyNodeType(ProxyNodeType.EXTERNAL)
.name("phoenix-2")
.agentTopicName("Phoenix-amd64-02")
.agentName("Phoenix-amd64-02")
.build();
london2 = ProxyNode
@@ -87,7 +89,18 @@ public class ProxyNodeSet {
.publicIPv4("141.147.106.62")
.proxyNodeType(ProxyNodeType.EXTERNAL)
.name("london-2")
.agentTopicName("London-amd64-02")
.agentName("London-amd64-02")
.build();
chengduAgent = ProxyNode
.builder()
.location("Chengdu")
.num(99)
.publicIPv4("183.220.112.113")
.proxyNodeType(ProxyNodeType.EXTERNAL)
.name("chengdu-agent")
.agentName("Chengdu-amd64-77")
.agentTopicName("Chengdu-amd64-77-remote")
.build();
ProxyNodeMap.put(chengdu.getNum(), chengdu);
@@ -97,7 +110,7 @@ public class ProxyNodeSet {
ProxyNodeMap.put(tokyo2.getNum(), tokyo2);
ProxyNodeMap.put(phoenix2.getNum(), phoenix2);
ProxyNodeMap.put(london2.getNum(), london2);
ProxyNodeMap.put(chengduAgent.getNum(), chengduAgent);
}

30
server/src/main/java/io/wdd/func/xray/service/XrayConfigDistribute.java
View File

@@ -1,5 +1,6 @@
package io.wdd.func.xray.service;
import io.wdd.common.utils.TimeUtils;
import io.wdd.func.oss.config.OctopusObjectSummary;
import io.wdd.func.oss.config.OssConfig;
import io.wdd.func.oss.service.OSSCoreService;
@@ -41,10 +42,9 @@ public class XrayConfigDistribute {
static {
ArrayList<String> first = new ArrayList<>(
List.of(
"cp",
"mv",
"/usr/local/etc/xray/config.json",
"/usr/local/etc/xray/config-bak-$(date +%Y-%m-%d-%H-%M-%S)"
"/usr/local/etc/xray/config-bak-TIME"
)
);
@@ -126,6 +126,7 @@ public class XrayConfigDistribute {
public void buildXrayUpdateResult(ArrayList<ProxyNode> networkPathList) {
String formatTimeString = TimeUtils.currentFormatTimeString();
List<String> resultKeyList = networkPathList
.stream()
@@ -136,13 +137,25 @@ public class XrayConfigDistribute {
.getObjectSummary()
.getAccess();
String realUrl = access
.getOriginUrl()
.getPath();
String realUrl = String.valueOf(access
.getOriginUrl());
if (StringUtils.isNotEmpty(access.getCloudflareUrl())) {
// 优先使用 被cloudflare包装过的的下载网址
realUrl = access.getCloudflareUrl();
}
// 修改命令中的时间
String s = updateXrayCommandList
.get(0)
.get(2);
updateXrayCommandList
.get(0)
.set(
2,
s.replace("TIME",
formatTimeString)
);
// 修改命令中的下载url
updateXrayCommandList
.get(1)
@@ -165,7 +178,10 @@ public class XrayConfigDistribute {
)
.collect(Collectors.toList());
log.debug("发送所有的配置到各个Agent成功， 结果查看为 => {}", resultKeyList);
log.info(
"发送所有的配置到各个Agent成功， 结果查看为 => {}",
resultKeyList
);
}

17
server/src/main/java/io/wdd/func/xray/service/XrayCoreServiceImpl.java
View File

@@ -23,7 +23,6 @@ import javax.annotation.Resource;
import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
@@ -81,18 +80,10 @@ public class XrayCoreServiceImpl implements XrayCoreService {
private void generateXrayJsonSinglePath(ArrayList<ProxyNode> networkPathList) {
int pathLength = networkPathList.size();
if (pathLength == 1) {
log.error("网络路径节点仅为一个");
return;
}
// 忽略掉 null的字段
objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
ArrayList<XrayConfig> tmpXrayConfigList = new ArrayList<>();
// 采用 VMESS + websocket的形式形成 链状代理
// 由于 Vlss+XTLS的形式形成 链状结构
String tag = generatePathTag(networkPathList);
@@ -119,7 +110,10 @@ public class XrayCoreServiceImpl implements XrayCoreService {
ProxyNode proxyNode = networkPathList.get(pos);
// 最后一个节点需要进行 FreeOutbound的构造
boolean isOutBoundFree = pos == pathLength - 1;
// 构造 xrayConfig
XrayConfig xrayConfig = doBuildXrayConfig(
isOutBoundFree,
tag,
@@ -129,9 +123,6 @@ public class XrayCoreServiceImpl implements XrayCoreService {
pos
);
// 添加到临时缓存中
tmpXrayConfigList.add(xrayConfig);
// 持久化
try {
String resultContent = objectMapper
@@ -169,7 +160,7 @@ public class XrayCoreServiceImpl implements XrayCoreService {
private String buildXrayConfigFileName(ProxyNode proxyNode, String timeString) {
return proxyNode.getNum() + "-" + proxyNode.getAgentTopicName() + "-" + timeString + ".json";
return proxyNode.getNum() + "-" + proxyNode.getAgentName() + "-" + timeString + ".json";
}

4
server/src/main/java/io/wdd/rpc/execute/service/CoreExecutionServiceImpl.java
View File

@@ -84,8 +84,8 @@ public class CoreExecutionServiceImpl implements CoreExecutionService {
// 检查agentTopicName是否存在
if (!ALL_AGENT_TOPIC_NAME_SET.contains(agentTopicName)) {
log.error("agentTopicName异常!");
throw new MyRuntimeException("agentTopicName异常!");
log.error("agentTopicName异常! 输入为 => {}", agentTopicName);
throw new MyRuntimeException("agentTopicName异常!" + agentTopicName);
}
// 归一化type类型 不行

13
server/src/main/java/io/wdd/rpc/init/ServerBootUpEnvironment.java
View File

@@ -99,6 +99,12 @@ public class ServerBootUpEnvironment {
}
/**
*
* 更新Agent状态信息的全局缓存
*
* 由定时任务或者初始化服务触发
*/
public void updateAgentStatusMapCache() {
if (CollectionUtils.isEmpty(ALL_AGENT_TOPIC_NAME_LIST)) {
@@ -128,12 +134,13 @@ public class ServerBootUpEnvironment {
);
}
String currentTimeString = TimeUtils.currentTimeString();
log.info(
// 由于太多了,所以关闭此处log输出
/* String currentTimeString = TimeUtils.currentTimeString();
log.debug(
"[ AGENT HEALTHY CHECK ] time is {} , result are => {}",
currentTimeString,
agentStatusMap
);
);*/
// 2023-01-16
ALL_AGENT_STATUS_MAP.clear();

2
server/src/main/java/io/wdd/rpc/scheduler/config/QuartzLogOperator.java
View File

@@ -12,7 +12,7 @@ public class QuartzLogOperator {
public boolean save(){
log.info("QuartzLogOperator pretend to have saved the log !");
log.debug("QuartzLogOperator pretend to have saved the log !");
return true;

13
server/src/main/java/io/wdd/rpc/scheduler/job/AgentScriptSchedulerJob.java
View File

@@ -1,8 +1,10 @@
package io.wdd.rpc.scheduler.job;
import io.wdd.common.utils.TimeUtils;
import io.wdd.rpc.scheduler.beans.ScriptSchedulerDTO;
import io.wdd.rpc.scheduler.service.script.AgentApplyScheduledScript;
import lombok.extern.slf4j.Slf4j;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
@@ -12,6 +14,10 @@ import javax.annotation.Resource;
import static io.wdd.rpc.scheduler.service.QuartzSchedulerServiceImpl.SCRIPT_SCHEDULER_MISSION_KEY;
/**
* 定时脚本任务核心类，Quartz框架定时调用该类
*/
@Slf4j
public class AgentScriptSchedulerJob extends QuartzJobBean {
@Resource
@@ -20,6 +26,11 @@ public class AgentScriptSchedulerJob extends QuartzJobBean {
@Override
protected void executeInternal(JobExecutionContext jobExecutionContext) throws JobExecutionException {
log.info("[ 定时脚本任务 ]被调用了，调用时间为: {}",
TimeUtils.currentTimeString()
);
// 解析 Scheduler 模块传递过来的参数
JobDataMap jobDataMap = jobExecutionContext
.getJobDetail()
@@ -28,7 +39,7 @@ public class AgentScriptSchedulerJob extends QuartzJobBean {
// ScriptScheduleDTO
ScriptSchedulerDTO scriptSchedulerDTO = (ScriptSchedulerDTO) jobDataMap.get(SCRIPT_SCHEDULER_MISSION_KEY);
// 调用实际任务执行器
// 调用实际任务执行器, 实际执行定时任务的核心业务内容
agentApplyScheduledScript.apply(scriptSchedulerDTO);
}

11
server/src/main/java/io/wdd/rpc/scheduler/service/QuartzSchedulerService.java
View File

@@ -17,7 +17,7 @@ public interface QuartzSchedulerService {
boolean addMission(OctopusQuartzJob quartzJob);
/**
* 增加一个任务job
* 增加一个任务job, 重复特定的次数
*
* @param jobClass 任务job实现类
* @param jobName 任务job名称（保证唯一性）
@@ -37,8 +37,9 @@ public interface QuartzSchedulerService {
* @param startTime
* @param cronJobExpression 任务时间表达式
* @param jobData 任务参数
* @return
*/
void addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, int startTime, String cronJobExpression, Map jobData);
JobDetail addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, int startTime, String cronJobExpression, Map jobData);
/**
* 增加一个任务job
@@ -46,11 +47,12 @@ public interface QuartzSchedulerService {
* @param jobClass 任务job实现类
* @param jobName 任务job名称（保证唯一性）
* @param jobGroupName 任务job组名
* @param startTime
* @param startTime 任务开始时间,单位间隔为秒
* @param cronJobExpression 任务时间表达式
* @param jobData 任务参数
* @return
*/
void addMission(
JobDetail addMission(
Class<? extends QuartzJobBean> jobClass,
String jobName,
String jobGroupName,
@@ -125,7 +127,6 @@ public interface QuartzSchedulerService {
/**
* --------------------------------------------------------------
* 页面定时任务部分
* 应该只有脚本功能才可以定时，目前一阶段的功能
*

57
server/src/main/java/io/wdd/rpc/scheduler/service/QuartzSchedulerServiceImpl.java
View File

@@ -28,13 +28,8 @@ import java.util.*;
import java.util.stream.Collectors;
import static io.wdd.rpc.scheduler.beans.ScriptSchedulerVO.SCRIPT_SCHEDULE_MISSION_GROUP_NAME;
import static io.wdd.rpc.scheduler.service.BuildStatusScheduleTask.JOB_GROUP_NAME;
import static org.quartz.TriggerBuilder.newTrigger;
/**
* @author Andya
* @date 2021/4/01
*/
@Slf4j
@Service
public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
@@ -52,15 +47,8 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
* 保存 定时脚本任务的中间DTO，在Quartz框架中JobDataMap的Key名称
*/
public static final String SCRIPT_SCHEDULER_MISSION_KEY = "scriptSchedulerDTO";
/**
* --------------------------------------------------------------
* 页面定时任务部分
* 应该只有脚本功能才可以定时，目前一阶段的功能
* */
/**
* @return
*/
@Override
public HashMap<String, String> createScriptScheduledMission(ScriptSchedulerVO scriptSchedulerVO) {
@@ -75,7 +63,9 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
SCRIPT_SCHEDULER_MISSION_KEY,
scriptSchedulerDTO
);
this.addMission(
// 核心部分，使用Quartz框架，新增定时任务
JobDetail jobDetail = this.addMission(
AgentScriptSchedulerJob.class,
// 方便删除任务，比较好定位
scriptSchedulerDTO.getSchedulerUuid(),
@@ -89,20 +79,23 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
HashMap<String, String> futureExecutionResultKey = quartzSchedulerUtils.generateAndSetFutureExecutionResultKey(scriptSchedulerDTO);
log.info(
log.debug(
"futureExecutionResultKey is => {}",
futureExecutionResultKey
);
log.info("[ 定时脚本任务 ] - 新建任务成功, 船舰任务DTO为 => {}", scriptSchedulerDTO);
log.info("[ 定时脚本任务 ] - 新建任务成功, 任务内容为 => {}", jobDetail);
// persistent the script scheduled mission
// dto should store more info
ScriptSchedulerPO scriptSchedulerPO = convertToScriptSchedulerPO(scriptSchedulerDTO);
log.info(
log.debug(
"scriptSchedulerPO is => {}",
scriptSchedulerPO
);
// scriptSchedulerService.save(scriptSchedulerPO);
// 持久化保存定时脚本任务至数据库中
scriptSchedulerService.save(scriptSchedulerPO);
return futureExecutionResultKey;
}
@@ -205,16 +198,7 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
return false;
}
/**
* 增加一个job
*
* @param jobClass 任务实现类
* @param jobName 任务名称
* @param jobGroupName 任务组名
* @param jobRunTimePinch 时间表达式 (这是每隔多少秒为一次任务)
* @param jobRunRepeatTimes 运行的次数 （<0:表示不限次数）
* @param jobData 参数
*/
@Override
public void addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, int jobRunTimePinch, int jobRunRepeatTimes, Map jobData) {
try {
@@ -278,10 +262,11 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
* 表示立即开始
* @param cronJobExpression 时间表达式 （如：0/5 * * * * ? ）
* @param jobData 参数
* @return
*/
@Override
public void addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, int startTime, String cronJobExpression, Map jobData) {
addMission(
public JobDetail addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, int startTime, String cronJobExpression, Map jobData) {
return addMission(
jobClass,
jobName,
jobGroupName,
@@ -293,7 +278,7 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
}
@Override
public void addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, String jobDescription, int startTime, String cronJobExpression, Map jobData) {
public JobDetail addMission(Class<? extends QuartzJobBean> jobClass, String jobName, String jobGroupName, String jobDescription, int startTime, String cronJobExpression, Map jobData) {
try {
// 创建jobDetail实例，绑定Job实现类
// 指明job的名称，所在组的名称，以及绑定job类
@@ -306,6 +291,7 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
)
.withDescription(jobDescription)
.build();
// 设置job参数
if (jobData != null && jobData.size() > 0) {
jobDetail
@@ -316,7 +302,6 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
// 定义调度触发规则
// 使用cornTrigger规则
// 触发器key
// uniform the start time
if (ObjectUtils.isEmpty(startTime)) {
startTime = 1;
@@ -330,13 +315,12 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
.startAt(
DateBuilder.futureDate(
startTime,
IntervalUnit.MILLISECOND
IntervalUnit.SECOND
)
)
.withSchedule(
CronScheduleBuilder.cronSchedule(cronJobExpression)
)
.startNow()
.build();
// 把作业和触发器注册到任务调度中
@@ -345,10 +329,7 @@ public class QuartzSchedulerServiceImpl implements QuartzSchedulerService {
trigger
);
log.debug(
"jobDataMap: {}",
jobDetail.getJobDataMap()
);
return jobDetail;
} catch (Exception e) {

24
server/src/main/java/io/wdd/rpc/scheduler/service/script/AgentApplyScheduledScript.java
View File

@@ -25,11 +25,17 @@ public class AgentApplyScheduledScript {
@Resource
QuartzSchedulerUtils quartzSchedulerUtils;
/**
* 实际执行定时脚本任务的核心方法
*
* @param scriptSchedulerDTO
*/
public void apply(ScriptSchedulerDTO scriptSchedulerDTO) {
List<List<String>> completeCommandList = scriptSchedulerDTO.getCompleteCommandList();
List<String> targetMachineList = scriptSchedulerDTO.getTargetMachineList();
// 拿到执行任务的 resultKey
HashMap<String, String> futureResultKeyMap = scriptSchedulerDTO.getAgentTopicNameToFutureResultKeyMap();
// 新增-定时调用Nacos中预编辑脚本的内容
@@ -38,12 +44,14 @@ public class AgentApplyScheduledScript {
scriptType = "Scheduled Script";
}
List<String> resultKeyList = coreExecutionService.SendCommandToAgentComplete(
targetMachineList,
scriptType,
completeCommandList,
futureResultKeyMap
);
// 发送命令到Agent中
List<String> resultKeyList = coreExecutionService
.SendCommandToAgentComplete(
targetMachineList,
scriptType,
completeCommandList,
futureResultKeyMap
);
// 将 resultKeyList 放入这个DTO中
scriptSchedulerDTO.setResultKeyList(resultKeyList);
@@ -51,7 +59,7 @@ public class AgentApplyScheduledScript {
// 更新DTO中的下次执行时间 否则会出错
HashMap<String, String> futureExecutionResultKey = quartzSchedulerUtils.generateAndSetFutureExecutionResultKey(scriptSchedulerDTO);
log.info(
log.debug(
"futureExecutionResultKey in scheduled work is => {}",
futureExecutionResultKey
);
@@ -59,7 +67,7 @@ public class AgentApplyScheduledScript {
// 需要更新数据库
// 关联性数据库
log.info("[定时脚本任务]-[业务类]-实际执行完成!");
}