系统进程监控与自动重启脚本

#!/bin/bash # ========================================== # 系统进程监控与自动重启脚本 # 用途：监控关键服务进程，异常时自动重启并告警 # 使用方法：bash process_monitor.sh # 定时任务：*/5 * * * * /path/to/process_monitor.sh # ========================================== # 配置 MONITOR_SERVICES="${MONITOR_SERVICES:-nginx mysql redis docker}" RESTART_CMD_NGINX="${RESTART_CMD_NGINX:-systemctl restart nginx}" RESTART_CMD_MYSQL="${RESTART_CMD_MYSQL:-systemctl restart mysql}" RESTART_CMD_REDIS="${RESTART_CMD_REDIS:-systemctl restart redis}" RESTART_CMD_DOCKER="${RESTART_CMD_DOCKER:-systemctl restart docker}" LOG_FILE="${LOG_FILE:-/var/log/process_monitor.log}" ALERT_EMAIL="${ALERT_EMAIL:-}" ALERT_WEBHOOK="${ALERT_WEBHOOK:-}" MAX_RESTART="${MAX_RESTART:-3}" RESTART_INTERVAL="${RESTART_INTERVAL:-300}" # 创建日志目录 mkdir -p "$(dirname "$LOG_FILE")" echo "========== 进程监控开始 ==========" echo "监控时间: $(date '+%Y-%m-%d %H:%M:%S')" echo "监控服务: $MONITOR_SERVICES" echo "" # 日志函数 log_msg() { local level=$1 local msg=$2 echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $msg" | tee -a "$LOG_FILE" } # 发送告警 send_alert() { local service=$1 local action=$2 local msg="【进程监控告警】服务: $service, 动作: $action, 时间: $(date '+%Y-%m-%d %H:%M:%S'), 主机: $(hostname)" if [ -n "$ALERT_EMAIL" ]; then echo "$msg" | mail -s "进程监控告警 - $service" "$ALERT_EMAIL" 2>/dev/null log_msg "INFO" "已发送邮件告警到: $ALERT_EMAIL" fi if [ -n "$ALERT_WEBHOOK" ]; then curl -s -X POST "$ALERT_WEBHOOK" -H "Content-Type: application/json" -d "{"content": "$msg"}" >/dev/null 2>&1 log_msg "INFO" "已发送Webhook告警" fi } # 检查服务状态 check_service() { local service=$1 log_msg "INFO" "检查服务: $service" # 检查进程是否存在 if pgrep -x "$service" > /dev/null 2>&1; then log_msg "INFO" "服务 $service 运行正常" return 0 fi # 检查systemd服务状态 if systemctl list-units --type=service --all | grep -q "${service}.service"; then if systemctl is-active --quiet "$service" 2>/dev/null; then log_msg "INFO" "服务 $service 运行正常(systemd)" return 0 fi fi log_msg "ERROR" "服务 $service 未运行!" return 1 } # 重启服务 restart_service() { local service=$1 local restart_cmd_var="RESTART_CMD_$(echo $service | tr '[:lower:]' '[:upper:]')" local restart_cmd=${!restart_cmd_var} if [ -z "$restart_cmd" ]; then restart_cmd="systemctl restart $service" fi log_msg "WARN" "尝试重启服务: $service" log_msg "INFO" "执行命令: $restart_cmd" # 检查重启次数 local restart_count_file="/tmp/process_monitor_${service}_count" local restart_time_file="/tmp/process_monitor_${service}_time" local current_time=$(date +%s) if [ -f "$restart_count_file" ] && [ -f "$restart_time_file" ]; then local last_time=$(cat "$restart_time_file") local time_diff=$((current_time - last_time)) if [ "$time_diff" -lt "$RESTART_INTERVAL" ]; then local count=$(cat "$restart_count_file") if [ "$count" -ge "$MAX_RESTART" ]; then log_msg "ERROR" "服务 $service 在 $RESTART_INTERVAL 秒内已重启 $count 次，超过最大限制 $MAX_RESTART 次，停止自动重启!" send_alert "$service" "重启次数超限，停止自动重启" return 1 fi echo $((count + 1)) > "$restart_count_file" else echo 1 > "$restart_count_file" echo "$current_time" > "$restart_time_file" fi else echo 1 > "$restart_count_file" echo "$current_time" > "$restart_time_file" fi # 执行重启 if eval "$restart_cmd" 2>&1 | tee -a "$LOG_FILE"; then sleep 3 if check_service "$service"; then log_msg "INFO" "服务 $service 重启成功" send_alert "$service" "自动重启成功" return 0 else log_msg "ERROR" "服务 $service 重启后仍未运行" send_alert "$service" "重启失败" return 1 fi else log_msg "ERROR" "服务 $service 重启命令执行失败" send_alert "$service" "重启命令执行失败" return 1 fi } # 主循环 TOTAL=0 NORMAL=0 ABNORMAL=0 RESTARTED=0 for service in $MONITOR_SERVICES; do TOTAL=$((TOTAL + 1)) if check_service "$service"; then NORMAL=$((NORMAL + 1)) else ABNORMAL=$((ABNORMAL + 1)) if restart_service "$service"; then RESTARTED=$((RESTARTED + 1)) fi fi echo "" done # 监控汇总 echo "========== 监控汇总 ==========" echo " 总服务数: $TOTAL" echo " 正常: $NORMAL" echo " 异常: $ABNORMAL" echo " 成功重启: $RESTARTED" echo " 日志文件: $LOG_FILE" echo "" if [ "$ABNORMAL" -gt 0 ]; then echo "🔴 发现 $ABNORMAL 个服务异常，请查看日志: $LOG_FILE" else echo "✅ 所有服务运行正常" fi echo "" echo "========== 监控完成 =========="

[{"name": "MONITOR_SERVICES", "label": "\u76d1\u63a7\u670d\u52a1\u5217\u8868(\u7a7a\u683c\u5206\u9694)", "default": "nginx mysql redis"}, {"name": "LOG_FILE", "label": "\u65e5\u5fd7\u6587\u4ef6\u8def\u5f84", "default": "/var/log/process_monitor.log"}, {"name": "MAX_RESTART", "label": "\u6700\u5927\u91cd\u542f\u6b21\u6570", "default": "3"}, {"name": "ALERT_EMAIL", "label": "\u544a\u8b66\u90ae\u7bb1", "default": ""}, {"name": "ALERT_WEBHOOK", "label": "Webhook\u5730\u5740", "default": ""}]