CmiiDeploy/84-202605-北京三河测试/install-rmdc-watchdog-node.sh

#!/usr/bin/env bash
set -euo pipefail

###############################################################################
# Configuration
###############################################################################

# Space separated host IPs. Include the master IP if the master node also needs
# rmdc-watchdog-node installed.
INSTALL_HOSTS="192.168.3.31 192.168.3.32 192.168.3.33"

# Absolute binary path on the master node. Remote nodes will receive the binary
# at the same absolute path.
RMDC_WATCHDOG_NODE_BIN="/root/wdd/rmdc-watchdog-node"

# Replace WDD_BOOTSTRAP_PSK in the systemd service.
WDD_BOOTSTRAP_PSK="hO8dKlh9hSXZ25Bv9xsySxDe2rh7XikV"

# Main NIC name. The script reads each node's IPv4 address on this NIC and uses
# it as WDD_LISTEN_IP.
MAIN_NIC="eno3"

# Master node private IP. This replaces WDD_BOOTSTRAP_ALLOW_IPS.
MASTER_IP="192.168.3.31"

# Systemd service install path on every node.
SERVICE_PATH="/usr/lib/systemd/system/rmdc-watchdog-node.service"

# SSH settings for remote nodes.
SSH_PORT="22"
SSH_CONNECT_TIMEOUT_SEC="10"
SSH_CONTROL_PERSIST="10m"

###############################################################################
# Implementation
###############################################################################

SERVICE_NAME="$(basename "$SERVICE_PATH")"
REMOTE_TMP_DIR="/tmp/rmdc-watchdog-node-deploy"
REMOTE_SERVICE_TMP="${REMOTE_TMP_DIR}/${SERVICE_NAME}"
SSH_CONTROL_DIR=""
SSH_CONTROL_PATH=""

log() {
  printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
}

die() {
  printf 'error: %s\n' "$*" >&2
  exit 1
}

shell_quote() {
  printf "'"
  printf '%s' "${1-}" | sed "s/'/'\\\\''/g"
  printf "'"
}

require_command() {
  command -v "$1" >/dev/null 2>&1 || die "required command not found: $1"
}

validate_config() {
  [[ "$(id -u)" == "0" ]] || die "this script must run as root"
  [[ "$#" == "0" ]] || die "this script does not accept command line arguments; edit the configuration section instead"
  [[ -n "$INSTALL_HOSTS" ]] || die "INSTALL_HOSTS is empty"
  [[ -f "$RMDC_WATCHDOG_NODE_BIN" ]] || die "binary not found: $RMDC_WATCHDOG_NODE_BIN"
  [[ -n "$WDD_BOOTSTRAP_PSK" && "$WDD_BOOTSTRAP_PSK" != "change-me" ]] || die "WDD_BOOTSTRAP_PSK must be configured"
  [[ -n "$MAIN_NIC" ]] || die "MAIN_NIC must be configured"
  [[ -n "$MASTER_IP" ]] || die "MASTER_IP must be configured"
  [[ "$SERVICE_PATH" = /* ]] || die "SERVICE_PATH must be an absolute path"
  [[ "$RMDC_WATCHDOG_NODE_BIN" = /* ]] || die "RMDC_WATCHDOG_NODE_BIN must be an absolute path"

  require_command awk
  require_command basename
  require_command dirname
  require_command install
  require_command ip
  require_command mktemp
  require_command sed
  require_command systemctl
}

get_local_ipv4s() {
  {
    hostname -I 2>/dev/null | tr ' ' '\n' || true
    ip -o -4 addr show 2>/dev/null | awk '{split($4, a, "/"); print a[1]}' || true
  } | awk 'NF' | sort -u
}

is_local_host() {
  local host="$1"

  case "$host" in
    localhost|127.0.0.1|::1)
      return 0
      ;;
  esac

  get_local_ipv4s | awk -v host="$host" '$0 == host {found = 1} END {exit found ? 0 : 1}'
}

local_nic_ip() {
  ip -o -4 addr show dev "$MAIN_NIC" scope global |
    awk '{split($4, a, "/"); print a[1]; exit}'
}

remote_sh() {
  local host="$1"
  local command="$2"

  ssh \
    -p "$SSH_PORT" \
    -o ConnectTimeout="$SSH_CONNECT_TIMEOUT_SEC" \
    -o StrictHostKeyChecking=no \
    -o UserKnownHostsFile=/dev/null \
    -o LogLevel=ERROR \
    -o ControlMaster=auto \
    -o ControlPersist="$SSH_CONTROL_PERSIST" \
    -o ControlPath="$SSH_CONTROL_PATH" \
    "root@${host}" \
    "bash -lc $(shell_quote "$command")"
}

remote_scp() {
  local source_path="$1"
  local host="$2"
  local target_path="$3"

  scp \
    -P "$SSH_PORT" \
    -o ConnectTimeout="$SSH_CONNECT_TIMEOUT_SEC" \
    -o StrictHostKeyChecking=no \
    -o UserKnownHostsFile=/dev/null \
    -o LogLevel=ERROR \
    -o ControlMaster=auto \
    -o ControlPersist="$SSH_CONTROL_PERSIST" \
    -o ControlPath="$SSH_CONTROL_PATH" \
    "$source_path" \
    "root@${host}:${target_path}"
}

cleanup_ssh_control() {
  local host

  if [[ -n "${SSH_CONTROL_DIR:-}" && -d "$SSH_CONTROL_DIR" ]]; then
    for host in $INSTALL_HOSTS; do
      if ! is_local_host "$host"; then
        ssh \
          -p "$SSH_PORT" \
          -o StrictHostKeyChecking=no \
          -o UserKnownHostsFile=/dev/null \
          -o LogLevel=ERROR \
          -o ControlPath="$SSH_CONTROL_PATH" \
          -O exit \
          "root@${host}" >/dev/null 2>&1 || true
      fi
    done
    rm -rf "$SSH_CONTROL_DIR"
  fi
}

remote_nic_ip() {
  local host="$1"
  remote_sh "$host" "ip -o -4 addr show dev $(shell_quote "$MAIN_NIC") scope global | awk '{split(\$4, a, \"/\"); print a[1]; exit}'"
}

render_service() {
  local listen_ip="$1"
  local output_path="$2"
  local working_dir

  working_dir="$(dirname "$RMDC_WATCHDOG_NODE_BIN")"

  cat > "$output_path" <<EOF
[Unit]
Description=RMDC Watchdog Node Daemon
After=network.target

[Service]
Type=simple
User=root
Group=root
WorkingDirectory=${working_dir}
ExecStart=${RMDC_WATCHDOG_NODE_BIN}
Restart=on-failure
RestartSec=3

Environment="WDD_LISTEN_IP=${listen_ip}"
Environment="WDD_BOOTSTRAP_TIMEOUT_SEC=300"
Environment="WDD_BOOTSTRAP_ALLOW_IPS=${MASTER_IP}"
Environment="WDD_BOOTSTRAP_RE_ALLOW=false"
Environment="WDD_RUNTIME_MAX_CONCURRENT=4"
Environment="WDD_RUNTIME_QUEUE_CAPACITY=128"
Environment="WDD_RUNTIME_TASK_TIMEOUT_SEC=1800"
Environment="WDD_SECURITY_REQUIRE_ROOT=true"
Environment="WDD_SECURITY_TOTP_WINDOW=1"
Environment="WDD_BOOTSTRAP_PSK=${WDD_BOOTSTRAP_PSK}"

[Install]
WantedBy=multi-user.target
EOF
}

install_local() {
  local host="$1"
  local listen_ip
  local rendered_service

  listen_ip="$(local_nic_ip)"
  [[ -n "$listen_ip" ]] || die "failed to get local IPv4 address from NIC ${MAIN_NIC}"

  rendered_service="$(mktemp)"
  render_service "$listen_ip" "$rendered_service"

  log "${host}: installing local service with WDD_LISTEN_IP=${listen_ip}"
  chmod 0755 "$RMDC_WATCHDOG_NODE_BIN"
  chown root:root "$RMDC_WATCHDOG_NODE_BIN"
  install -d -m 0755 "$(dirname "$SERVICE_PATH")"
  install -m 0644 -o root -g root "$rendered_service" "$SERVICE_PATH"
  rm -f "$rendered_service"

  systemctl daemon-reload
  systemctl enable "$SERVICE_NAME"
  systemctl restart "$SERVICE_NAME"
  systemctl is-active --quiet "$SERVICE_NAME"
}

install_remote() {
  local host="$1"
  local listen_ip
  local rendered_service
  local binary_dir
  local service_dir

  listen_ip="$(remote_nic_ip "$host")"
  [[ -n "$listen_ip" ]] || die "${host}: failed to get IPv4 address from NIC ${MAIN_NIC}"

  rendered_service="$(mktemp)"
  render_service "$listen_ip" "$rendered_service"

  binary_dir="$(dirname "$RMDC_WATCHDOG_NODE_BIN")"
  service_dir="$(dirname "$SERVICE_PATH")"

  log "${host}: preparing remote directories"
  remote_sh "$host" "mkdir -p $(shell_quote "$binary_dir") $(shell_quote "$service_dir") $(shell_quote "$REMOTE_TMP_DIR")"

  log "${host}: uploading binary to ${RMDC_WATCHDOG_NODE_BIN}"
  remote_scp "$RMDC_WATCHDOG_NODE_BIN" "$host" "$RMDC_WATCHDOG_NODE_BIN"

  log "${host}: uploading service with WDD_LISTEN_IP=${listen_ip}"
  remote_scp "$rendered_service" "$host" "$REMOTE_SERVICE_TMP"
  rm -f "$rendered_service"

  log "${host}: reloading and restarting ${SERVICE_NAME}"
  remote_sh "$host" "set -euo pipefail
chmod 0755 $(shell_quote "$RMDC_WATCHDOG_NODE_BIN")
chown root:root $(shell_quote "$RMDC_WATCHDOG_NODE_BIN")
install -m 0644 -o root -g root $(shell_quote "$REMOTE_SERVICE_TMP") $(shell_quote "$SERVICE_PATH")
systemctl daemon-reload
systemctl enable $(shell_quote "$SERVICE_NAME")
systemctl restart $(shell_quote "$SERVICE_NAME")
systemctl is-active --quiet $(shell_quote "$SERVICE_NAME")
rm -f $(shell_quote "$REMOTE_SERVICE_TMP")"
}

print_failure_diagnostics() {
  local host="$1"

  if is_local_host "$host"; then
    systemctl --no-pager --full status "$SERVICE_NAME" || true
    journalctl -u "$SERVICE_NAME" -n 80 --no-pager || true
  else
    remote_sh "$host" "systemctl --no-pager --full status $(shell_quote "$SERVICE_NAME") || true; journalctl -u $(shell_quote "$SERVICE_NAME") -n 80 --no-pager || true" || true
  fi
}

main() {
  validate_config "$@"

  local has_remote="false"
  local host
  for host in $INSTALL_HOSTS; do
    if ! is_local_host "$host"; then
      has_remote="true"
      break
    fi
  done

  if [[ "$has_remote" == "true" ]]; then
    require_command ssh
    require_command scp
    SSH_CONTROL_DIR="$(mktemp -d -t rmdc-watchdog-node-ssh.XXXXXX)"
    SSH_CONTROL_PATH="${SSH_CONTROL_DIR}/%r@%h:%p"
    trap cleanup_ssh_control EXIT
  fi

  for host in $INSTALL_HOSTS; do
    log "${host}: deployment started"
    if is_local_host "$host"; then
      if ! install_local "$host"; then
        print_failure_diagnostics "$host"
        die "${host}: deployment failed"
      fi
    else
      if ! install_remote "$host"; then
        print_failure_diagnostics "$host"
        die "${host}: deployment failed"
      fi
    fi
    log "${host}: deployment finished"
  done

  log "all deployments finished"
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
  main "$@"
fi