diff --git a/.github/workflows/build-push-docker.yml b/.github/workflows/build-push-docker.yml index dc2433d..3d3b7c8 100644 --- a/.github/workflows/build-push-docker.yml +++ b/.github/workflows/build-push-docker.yml @@ -2,7 +2,7 @@ # good reference -name: Maven Package +name: Octopus Complete CI on: # Triggers the workflow on push request events but only for the main branch @@ -128,7 +128,10 @@ jobs: echo "--------------------------------------" cd ./agent/target ls + echo "start to change timezone to Asia/Shanghai" + timedatectl set-timezone Asia/Shanghai && timedatectl set-ntp true echo "current time is $(date --rfc-3339=seconds | cut -d"+" -f1 | sed "s/ /-/g" | sed "s/:/-/g")" + echo "current time is $(date)" echo "start to copy target jar" cp ./agent-*.jar ./octopus-agent-$(date --rfc-3339=seconds | cut -d"+" -f1 | sed "s/ /-/g" | sed "s/:/-/g").jar cd /home/runner/work/ProjectOctopus/ProjectOctopus diff --git a/agent/src/main/java/io/wdd/agent/executor/CommandExecutor.java b/agent/src/main/java/io/wdd/agent/executor/CommandExecutor.java index 72b6759..ff477ed 100644 --- a/agent/src/main/java/io/wdd/agent/executor/CommandExecutor.java +++ b/agent/src/main/java/io/wdd/agent/executor/CommandExecutor.java @@ -6,6 +6,7 @@ import io.wdd.agent.executor.thread.LogToArrayListCache; import io.wdd.common.beans.executor.ExecutionMessage; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.ObjectUtils; import org.springframework.context.annotation.Configuration; import javax.annotation.Resource; @@ -86,12 +87,18 @@ public class CommandExecutor { // start to send the result log streamSender.startToWaitLog(streamKey); - // get the command result - processResult = process.waitFor(); + // todo this will stuck the process and rabbitmq message will reentry the queue + // get the command result must also be a timeout smaller than the process + boolean waitFor = process.waitFor(50, TimeUnit.SECONDS); // end send logs streamSender.endWaitLog(streamKey); + // get the process result + if (ObjectUtils.isNotEmpty(waitFor) && ObjectUtils.isNotEmpty(process)) { + processResult = process.exitValue(); + } + log.debug("current shell command {} result is {}", processBuilder.command(), processResult); @@ -106,7 +113,6 @@ public class CommandExecutor { return () -> { try { - log.debug("daemon thread start to wait for {} s for the result", processMaxWaitSeconds); TimeUnit.SECONDS.sleep(processMaxWaitSeconds); diff --git a/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java b/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java index c833623..e8c2998 100644 --- a/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java +++ b/agent/src/main/java/io/wdd/agent/executor/function/CollectAllExecutorFunction.java @@ -31,7 +31,7 @@ public class CollectAllExecutorFunction { /** - * store the Octopus Agent Functions and Reflection Class Path + * store the Octopus Agent Functions and Function Command List * key: function name * value: function shell List contend */ diff --git a/server/src/main/java/io/wdd/rpc/execute/result/CreateStreamReader.java b/server/src/main/java/io/wdd/rpc/execute/result/CreateStreamReader.java index 4532196..9373180 100644 --- a/server/src/main/java/io/wdd/rpc/execute/result/CreateStreamReader.java +++ b/server/src/main/java/io/wdd/rpc/execute/result/CreateStreamReader.java @@ -38,6 +38,12 @@ public class CreateStreamReader implements ApplicationContextAware { // prepare the environment prepareEnv(); + // oldStreamKey equals streamKey don't need to do anything , just return + if (redisStreamReaderConfig.getStreamKey().equals(streamKey)) { + log.debug("redis listener container not change !"); + return; + } + // destroy the REDIS_STREAM_LISTENER_CONTAINER destroyStreamReader(streamKey); diff --git a/source/src/main/java/io/wdd/source/shell/agent-bootup.sh b/source/src/main/java/io/wdd/source/shell/agent-bootup.sh index a6b2347..d9b3800 100644 --- a/source/src/main/java/io/wdd/source/shell/agent-bootup.sh +++ b/source/src/main/java/io/wdd/source/shell/agent-bootup.sh @@ -2,7 +2,6 @@ ##### environment variables ###### - JAVA_OPTS="-Xms128m -Xmx512m -Dfile.encoding=utf-8 --spring.profiles.active=k3s --spring.cloud.nacos.config.group=k3s --spring.cloud.nacos.config.extension-configs[0].dataId=common-k3s.yaml --spring.cloud.nacos.config.extension-configs[0].group=k3s" DependLibFiles=( @@ -13,7 +12,6 @@ DependLibFiles=( wdd-lib-sys.sh ) - OctopusAgentUrl=https://happybirthday.107421.xyz/octopus-agent/ RepoSourcePath=https://raw.githubusercontent.com/zeaslity/ProjectOctopus/main/source/src/main/java/io/wdd/source/shell @@ -42,6 +40,15 @@ colorEcho() { echo -e "\033[${1}${@:2}\033[0m" 1>&2 } +####################################### +# description +# Globals: +# EUID +# RED +# YELLOW +# Arguments: +# None +####################################### check_root() { if [[ $EUID != 0 ]]; then colorEcho ${RED} "当前非root账号(或没有root权限)，无法继续操作，请更换root账号!" @@ -50,17 +57,41 @@ check_root() { fi } +####################################### +# description +# Globals: +# PURPLE +# SplitLine +# Arguments: +# None +####################################### FunctionStart() { colorEcho ${PURPLE} ${SplitLine} colorEcho ${PURPLE} ${SplitLine} echo "" } +####################################### +# description +# Globals: +# GREEN +# SplitLine +# Arguments: +# None +####################################### FunctionSuccess() { colorEcho ${GREEN} ${SplitLine} echo "" } +####################################### +# description +# Globals: +# BlinkGreen +# SplitLine +# Arguments: +# None +####################################### FunctionEnd() { echo "" colorEcho ${BlinkGreen} ${SplitLine} @@ -80,32 +111,32 @@ check_sys() { sys_bit=$(uname -m) case $sys_bit in - i[36]86) - os_bit="32" - LinuxRelease="386" - ;; - x86_64) - os_bit="64" - LinuxRelease="amd64" - ;; - *armv6*) - os_bit="arm" - LinuxRelease="arm6" - ;; - *armv7*) - os_bit="arm" - LinuxRelease="arm7" - ;; - *aarch64* | *armv8*) - os_bit="arm64" - LinuxRelease="arm64" - ;; - *) - colorEcho ${RED} " + i[36]86) + os_bit="32" + LinuxRelease="386" + ;; + x86_64) + os_bit="64" + LinuxRelease="amd64" + ;; + *armv6*) + os_bit="arm" + LinuxRelease="arm6" + ;; + *armv7*) + os_bit="arm" + LinuxRelease="arm7" + ;; + *aarch64* | *armv8*) + os_bit="arm64" + LinuxRelease="arm64" + ;; + *) + colorEcho ${RED} " 哈哈……这个 辣鸡脚本 不支持你的系统。 (-_-) \n 备注: 仅支持 Ubuntu 16+ / Debian 8+ / CentOS 7+ 系统 " && exit 1 - ;; + ;; esac ## 判定Linux的发行版本 if [ -f /etc/redhat-release ]; then @@ -188,6 +219,18 @@ installDemandSoftwares() { return 0 } +####################################### +# description +# Globals: +# BLUE +# DependLibFiles +# OctopusAgentPath +# RED +# RepoSourcePath +# libfile +# Arguments: +# None +####################################### DownloadAllFile() { FunctionStart @@ -203,7 +246,7 @@ DownloadAllFile() { for libfile in ${DependLibFiles[*]}; do colorEcho $BLUE "lib file is $libfile" - wget "$RepoSourcePath/lib/$libfile" -Oq $OctopusAgentPath/lib/$libfile + wget "$RepoSourcePath/lib/$libfile" -qO $OctopusAgentPath/lib/$libfile FunctionSuccess done @@ -216,25 +259,26 @@ DownloadAllFile() { . ./lib/wdd-lib-os.sh CheckAndDownloadLatestVersion - FunctionSuccess FunctionEnd } - -ModifySystemConfig(){ +####################################### +# description +# Globals: +# BLUE +# GREEN +# Arguments: +# None +####################################### +ModifySystemConfig() { FunctionStart colorEcho ${BLUE} "开始修改系统内核参数…………" ## 配置内核参数 cat >/etc/sysctl.d/k8s.conf </etc/rsyslog.d/octopus-agent.conf </etc/rsyslog.d/octopus-agent.conf <&1 | awk -F, '{io=$NF} END { print io}' | sed 's/^[ \t]*//;s/[ \t]*$//' } +####################################### +# description +# Arguments: +# 1 +# Returns: +# ... +####################################### calc_size() { local raw=$1 local total_size=0 @@ -72,12 +81,35 @@ calc_size() { echo "${total_size} ${unit}" } +####################################### +# description +# Arguments: +# None +# Returns: +# ... +####################################### GethostArchInfo() { [ -f /etc/redhat-release ] && awk '{print $0}' /etc/redhat-release && return [ -f /etc/os-release ] && awk -F'[= "]' '/PRETTY_NAME/{print $3,$4,$5}' /etc/os-release && return [ -f /etc/lsb-release ] && awk -F'[="]+' '/DESCRIPTION/{print $2}' /etc/lsb-release && return } +####################################### +# description +# Globals: +# freespace +# io1 +# io2 +# io3 +# ioall +# ioavg +# ioraw1 +# ioraw2 +# ioraw3 +# writemb +# Arguments: +# None +####################################### StartIOTest() { log "start IO speed test !" freespace=$(df -m . | awk 'NR==2 {print $4}') @@ -105,12 +137,22 @@ StartIOTest() { echo " $(_red "Not enough space for I/O Speed test!")" fi - } +####################################### +# description +# Globals: +# cpuName +# sys_manu +# sys_product +# sys_ver +# virt +# virtualx +# Arguments: +# None +####################################### Check_Virtualization() { - log "start to check host virtualization !" command_exists "dmesg" && virtualx="$(dmesg 2>/dev/null)" @@ -131,21 +173,21 @@ Check_Virtualization() { virt="LXC" elif [[ -f /proc/user_beancounters ]]; then virt="OpenVZ" - elif [[ "${virtualx}" == *kvm-clock* ]]; then + elif [[ ${virtualx} == *kvm-clock* ]]; then virt="KVM" - elif [[ "${sys_product}" == *KVM* ]]; then + elif [[ ${sys_product} == *KVM* ]]; then virt="KVM" - elif [[ "${cpuName}" == *KVM* ]]; then + elif [[ ${cpuName} == *KVM* ]]; then virt="KVM" - elif [[ "${cpuName}" == *QEMU* ]]; then + elif [[ ${cpuName} == *QEMU* ]]; then virt="KVM" - elif [[ "${virtualx}" == *"VMware Virtual Platform"* ]]; then + elif [[ ${virtualx} == *"VMware Virtual Platform"* ]]; then virt="VMware" - elif [[ "${sys_product}" == *"VMware Virtual Platform"* ]]; then + elif [[ ${sys_product} == *"VMware Virtual Platform"* ]]; then virt="VMware" - elif [[ "${virtualx}" == *"Parallels Software International"* ]]; then + elif [[ ${virtualx} == *"Parallels Software International"* ]]; then virt="Parallels" - elif [[ "${virtualx}" == *VirtualBox* ]]; then + elif [[ ${virtualx} == *VirtualBox* ]]; then virt="VirtualBox" elif [[ -e /proc/xen ]]; then if grep -q "control_d" "/proc/xen/capabilities" 2>/dev/null; then @@ -155,9 +197,9 @@ Check_Virtualization() { fi elif [ -f "/sys/hypervisor/type" ] && grep -q "xen" "/sys/hypervisor/type"; then virt="Xen" - elif [[ "${sys_manu}" == *"Microsoft Corporation"* ]]; then - if [[ "${sys_product}" == *"Virtual Machine"* ]]; then - if [[ "${sys_ver}" == *"7.0"* || "${sys_ver}" == *"Hyper-V" ]]; then + elif [[ ${sys_manu} == *"Microsoft Corporation"* ]]; then + if [[ ${sys_product} == *"Virtual Machine"* ]]; then + if [[ ${sys_ver} == *"7.0"* || ${sys_ver} == *"Hyper-V" ]]; then virt="Hyper-V" else virt="Microsoft Virtual Machine" @@ -167,12 +209,21 @@ Check_Virtualization() { virt="Dedicated" fi - } +####################################### +# description +# Globals: +# city +# country +# org +# public_ipv4 +# region +# Arguments: +# None +####################################### GetIpv4Info() { - log "start to get system public ip info !" org="$(wget -q -T10 -O- ipinfo.io/org)" @@ -181,10 +232,8 @@ GetIpv4Info() { region="$(wget -q -T10 -O- ipinfo.io/region)" public_ipv4="$(wget -q -T10 -O- ipinfo.io/ip)" - } - log "start to collect system info !" check_sys @@ -247,7 +296,6 @@ disk_used_size=$( disk_used_size=$(calc_size $disk_used_size) tcpctrl=$(sysctl net.ipv4.tcp_congestion_control | awk -F ' ' '{print $3}') - # todo # StartIOTest @@ -262,7 +310,7 @@ if [[ $(cat /etc/hostname | cut -d"-" -f 3 | grep -c '^[0-9][0-9]') -gt 0 ]]; th else machineNumber=99 fi -cat >/etc/environment.d/octopus-agent.conf</etc/environment.d/octopus-agent.conf <" -f2 | cut -d"<" -f1) + local latestVersion=$(curl $OctopusAgentUrl | grep -v h1 | grep "a href=" | awk '{print$2}' | cut -d">" -f2 | cut -d"<" -f1) + log "" log "octopus agent latest version is => [ $latestVersion ]" - + log "" rm -rf /octopus-agent/*.jar cd /octopus-agent log "start to download the latest version !" - wget "$OctopusAgentUrl$latestVersion" cp "$latestVersion" agent.jar - log "" - echo "" - log "----------------------------------------------" + log "" + log "---------------- ls the /octopus-agent ----------------------" ls /octopus-agent/ | grep jar log "----------------------------------------------" -} \ No newline at end of file +} diff --git a/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh b/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh index 43cd23c..7adab2d 100644 --- a/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh +++ b/source/src/main/java/io/wdd/source/shell/lib/wdd-lib-sys.sh @@ -138,9 +138,6 @@ tmp () { gcloud compute instances create octopus-agent-2c-4g-1 --project=compact-lacing-371804 --zone=asia-northeast1-b --machine-type=n2d-custom-2-4096 --network-interface=network-tier=PREMIUM,subnet=default --metadata=startup-script=wget\ https://raw.githubusercontent.com/zeaslity/ProjectOctopus/main/source/src/main/java/io/wdd/source/shell/agent-bootup.sh\ \&\&\ chmod\ \+x\ agent-bootup.sh\ \&\&\ /bin/bash\ agent-bootup.sh --can-ip-forward --maintenance-policy=MIGRATE --provisioning-model=STANDARD --service-account=172889627951-compute@developer.gserviceaccount.com --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --create-disk=auto-delete=yes,boot=yes,device-name=octopus-agent-2c-4g,image=projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221213,mode=rw,size=20,type=projects/compact-lacing-371804/zones/us-west4-b/diskTypes/pd-ssd --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --reservation-affinity=any -gcloud compute instances delete tokyo-amd64-03 --project=compact-lacing-371804 --zone=asia-northeast1-b - - gcloud compute instances create tokyo-amd64-03 --project=compact-lacing-371804 --zone=asia-northeast1-b --machine-type=n2d-custom-2-4096 --network-interface=network-tier=PREMIUM,subnet=default --can-ip-forward --maintenance-policy=MIGRATE --provisioning-model=STANDARD --service-account=172889627951-compute@developer.gserviceaccount.com --scopes=https://www.googleapis.com/auth/devstorage.read_only,https://www.googleapis.com/auth/logging.write,https://www.googleapis.com/auth/monitoring.write,https://www.googleapis.com/auth/servicecontrol,https://www.googleapis.com/auth/service.management.readonly,https://www.googleapis.com/auth/trace.append --create-disk=auto-delete=yes,boot=yes,device-name=octopus-agent-2c-4g,image=projects/ubuntu-os-cloud/global/images/ubuntu-2004-focal-v20221213,mode=rw,size=20,type=projects/compact-lacing-371804/zones/us-west4-b/diskTypes/pd-ssd --no-shielded-secure-boot --shielded-vtpm --shielded-integrity-monitoring --reservation-affinity=any gcloud compute ssh --zone "asia-northeast1-b" "tokyo-amd64-03" --project "compact-lacing-371804" @@ -148,6 +145,11 @@ gcloud compute ssh --zone "asia-northeast1-b" "tokyo-amd64-03" --project "compac wget https://raw.githubusercontent.com/zeaslity/ProjectOctopus/main/source/src/main/java/io/wdd/source/shell/agent-bootup.sh && chmod +x agent-bootup.sh && /bin/bash agent-bootup.sh + +echo "y +" | gcloud compute instances delete tokyo-amd64-03 --project=compact-lacing-371804 --zone=asia-northeast1-b + + apt-cache madison openjdk-11-jdk | head -n 1 | awk '{print$3}' java -jar /octopus-agent/agent.jar -Xms128m -Xmx512m -Dfile.encoding=utf-8 --spring.profiles.active=k3s --spring.cloud.nacos.config.group=k3s --spring.cloud.nacos.config.extension-configs[0].dataId=common-k3s.yaml --spring.cloud.nacos.config.extension-configs[0].group=k3s