Monit-基于非容器服务自恢复程序实践

康器 · 2025-11-23 15:40:01

1.需求：因为历史原因和软件程序原因，有上百台服务和所在服务未运行在容器中，需要在程序奔溃自动拉起(以Java Python C++为主)。
2.目的：能够非人为干预快速自动恢复，要求检测频率在10s一次
3.实现方式
3.1 根据不同语言自己开发脚本实现自动拉起和通知(不够标准化-弃用)

1 #!/bin/bash
2
3 # ========== 使用说明 ==========
4 # 1. 自定配置env: fat/prod等环境变量, nacos地址, lark webhook等
5 # 2. 确保每个Java服务有对应的启动脚本（如 deploy-xxx.sh）
6 # 3. 设置需要监控的Java服务列表（见下方SERVICES_PROCESS_ID等变量）
7 # 4. 将此脚本添加到crontab中定期执行，例如每1分钟检测一次monitor是否在运行，如果在运行不会重复运行，不在运行则会后台运行：
8 # */1 * * * * /bin/bash /path/to/this/monitor.sh >> /path/to/monitor.log 2>&1
9
10 # ========== 环境变量加载 ==========
11 # 在 crontab 中执行时，需要显式加载环境变量
12 # 按优先级加载多个可能的环境变量文件
13 ENV_FILES=(
14 "/etc/profile"
15 "/etc/bashrc"
16 "/root/.bash_profile"
17 "/root/.bashrc"
18 "$HOME/.bash_profile"
19 "$HOME/.bashrc"
20 )
21
22 for env_file in "${ENV_FILES[@]}"; do
23 if [ -f "$env_file" ]; then
24 echo "[$(date '+%Y-%m-%d %H:%M:%S')] 加载环境变量文件: $env_file" >> /tmp/monitor_env.log
25 source "$env_file"
26 fi
27 done
28
29 # 如果上述文件都没有加载到关键环境变量，则手动设置
30 if [ -z "$JAVA_HOME" ]; then
31 export JAVA_HOME="/opt/jdk-17.0.8"
32 export CLASSPATH=".:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar"
33 export PATH="$JAVA_HOME/bin:$PATH"
34 echo "[$(date '+%Y-%m-%d %H:%M:%S')] 手动设置 JAVA_HOME: $JAVA_HOME" >> /tmp/monitor_env.log
35 fi
36
37 # 设置必要的环境变量
38 export env_nacos_address="nacos.test.com:8848"
39 export env_nacos_namespace="fat"
40 export ENV="fat"
41
42 # 记录环境变量状态（用于调试）
43 {
44 echo "=== 环境变量检查 ==="
45 echo "时间: $(date)"
46 echo "JAVA_HOME: $JAVA_HOME"
47 echo "PATH: $PATH"
48 echo "ENV: $ENV"
49 echo "env_nacos_address: $env_nacos_address"
50 which java
51 java -version 2>&1
52 echo "=== 环境变量检查结束 ==="
53 } >> /tmp/monitor_env.log 2>&1
54
55 # ========== 脚本配置 ==========
56 target_folder="./"
57
58 LARK_WEBHOOK="https://open.larksuite.com/open-apis/bot/v2/hook/token"
59 LARK_LOG_FILE="$target_folder/lark.log"
60 MONITOR_LOG_FILE="/tmp/check.logs"
61
62 LARK_MAX_RETRY=3
63 LARK_RETRY_DELAY=10
64 LARK_ENV="fat"
65
66 # ========== 文件锁机制 ==========
67 LOCK_FILE="/tmp/service_monitor.lock"
68
69 # 尝试获取锁，如果失败则退出
70 exec 200>"$LOCK_FILE"
71 flock -n 200 || {
72 echo "[$(date '+%Y-%m-%d %H:%M:%S')] 监控脚本已在运行，退出本次执行 (PID: $(cat $LOCK_FILE 2>/dev/null || echo '未知'))" >> "$MONITOR_LOG_FILE"
73 exit 0
74 }
75
76 # 将当前PID写入锁文件
77 echo $$ > "$LOCK_FILE"
78
79 # 设置退出时清理锁文件
80 trap "rm -f $LOCK_FILE; exit" INT TERM EXIT
81
82 echo "[$(date '+%Y-%m-%d %H:%M:%S')] 启动服务监控进程 (PID: $$)" | tee -a "$MONITOR_LOG_FILE"
83
84 # ========== Lark 报警函数 ==========
85 send_lark_alert() {
86 local message="$1"
87 local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
88
89 local full_message="[$LARK_ENV][进程监控] $message\n"
90 full_message+="服务器名: $(hostname)\n"
91 full_message+="报警时间: $timestamp\n"
92 #full_message+="Jame Mei</at>"
93 #full_message+="Levi Li</at>"
94 #full_message+="Bingbing Sun</at>"
95
96 local json_data="{
97 "msg_type": "interactive",
98 "card": {
99 "elements": [{
100 "tag": "div",
101 "text": {
102 "content": "$full_message",
103 "tag": "lark_md"
104 }
105 }]
106 }
107 }"
108
109 local attempt=1
110 while [[ $attempt -le $LARK_MAX_RETRY ]]; do
111 http_status=$(curl -s -o /dev/null -w "%{http_code}" \
112 -m 3 \
113 -X POST "$LARK_WEBHOOK" \
114 -H "Content-Type: application/json" \
115 -d "$json_data")
116
117 if [[ $http_status -eq 200 ]]; then
118 echo "[$(date "+%F %T")] Lark报警发送成功" >> "$LARK_LOG_FILE"
119 return 0
120 else
121 sleep $LARK_RETRY_DELAY
122 ((attempt++))
123 fi
124 done
125 echo "[$(date "+%F %T")] 错误: Lark报警发送失败,已达最大重试次数: $attempt" >> "$LARK_LOG_FILE"
126 return 1
127 }
128
129 send_alert_async() {
130 (
131 send_lark_alert "$1"
132 ) &> /dev/null &
133 }
134
135 # ========== 服务配置 ==========
136 WORK_DIR="/data/scripts"
137
138 # 服务列表
139 #SERVICES_PROCESS_ID[0]="pb-trading-engine-1.0-SNAPSHOT.jar"
140 #SERVICES_FRIENDLY_NAME[0]="交易引擎 (Trading Engine)"
141 #SERVICES_START_CMD[0]="./deploy-engine-new.sh start"
142
143 SERVICES_PROCESS_ID[1]="pb-trading-market-1.0-SNAPSHOT.jar"
144 SERVICES_FRIENDLY_NAME[1]="行情服务 (Trading Market)"
145 SERVICES_START_CMD[1]="./pb-trading-market.sh start"
146
147 SERVICES_PROCESS_ID[2]="rapidx-trading-query-realtime-1.0-SNAPSHOT.jar"
148 SERVICES_FRIENDLY_NAME[2]="实时查询 (Query Realtime)"
149 SERVICES_START_CMD[2]="./deploy-realtime-new.sh start"
150
151 SERVICES_PROCESS_ID[3]="router-server-1.0.0-SNAPSHOT.jar"
152 SERVICES_FRIENDLY_NAME[3]="路由服务 (Router Server)"
153 SERVICES_START_CMD[3]="./router-server.sh start"
154
155 SERVICES_PROCESS_ID[4]="pb-trading-query-1.0-SNAPSHOT.jar"
156 SERVICES_FRIENDLY_NAME[4]="交易查询 (Trading Query)"
157 SERVICES_START_CMD[4]="./pb-trading-query.sh start"
158
159 SERVICES_PROCESS_ID[5]="rapidx-trading-market-gateway-1.0-SNAPSHOT.jar"
160 SERVICES_FRIENDLY_NAME[5]="行情网关 (Market Gateway)"
161 SERVICES_START_CMD[5]="./rapidx-trading-market-gateway.sh start"
162
163 SERVICES_PROCESS_ID[6]="ltp-ems-1.0.0-master-SNAPSHOT.jar"
164 SERVICES_FRIENDLY_NAME[6]="订单管理 (EMS)"
165 SERVICES_START_CMD[6]="./deploy-ems-new.sh start"
166
167 SERVICES_PROCESS_ID[7]="pb-trading-push-0.0.1-SNAPSHOT.jar"
168 SERVICES_FRIENDLY_NAME[7]="推送服务 (Trading Push)"
169 SERVICES_START_CMD[7]="./pb-trading-push.sh start"
170
171 SERVICES_PROCESS_ID[8]="pb-trading-gateway-1.0-SNAPSHOT.jar"
172 SERVICES_FRIENDLY_NAME[8]="交易网关 (Trading Gateway)"
173 SERVICES_START_CMD[8]="./pb-trading-gateway.sh start"
174
175 SERVICES_PROCESS_ID[9]="exchange-data-server-1.0.0-SNAPSHOT.jar"
176 SERVICES_FRIENDLY_NAME[9]="exchange-data-server"
177 SERVICES_START_CMD[9]="./ltp-exchange-data-server.sh start"
178
179 SERVICES_PROCESS_ID[10]="rapidtrade-storage-1.0-SNAPSHOT.jar"
180 SERVICES_FRIENDLY_NAME[10]="rapidtrade-storage"
181 SERVICES_START_CMD[10]="./rapidtrade-storage.sh start"
182
183 SERVICES_PROCESS_ID[11]="rapidx-trading-algo-server-1.0-SNAPSHOT.jar"
184 SERVICES_FRIENDLY_NAME[11]="rapidx-trading-algo-server"
185 SERVICES_START_CMD[11]="./rapidx-trading-algo-server.sh start"
186
187 SERVICES_PROCESS_ID[12]="rapidtrade-mock-1.0-SNAPSHOT.jar"
188 SERVICES_FRIENDLY_NAME[12]="rapidtrade-mock"
189 SERVICES_START_CMD[12]="./rapidtrade-mock.sh start"
190
191 SERVICES_PROCESS_ID[13]="rapidx-ws-simulator-1.0-SNAPSHOT.jar"
192 SERVICES_FRIENDLY_NAME[13]="rapidx-ws-simulator"
193 SERVICES_START_CMD[13]="./rapidx-ws-simulator.sh start"
194
195 SERVICES_PROCESS_ID[14]="pb-trading-statistics-1.0-SNAPSHOT.jar"
196 SERVICES_FRIENDLY_NAME[14]="pb-trading-statistics"
197 SERVICES_START_CMD[14]="./pb-trading-statistics.sh start"
198
199 SERVICES_PROCESS_ID[15]="rapidx-trading-onezero-maker-1.0-SNAPSHOT.jar"
200 SERVICES_FRIENDLY_NAME[15]="rapidx-trading-onezero-maker"
201 SERVICES_START_CMD[15]="./rapidx-trading-onezero-maker.sh start"
202
203 SERVICES_PROCESS_ID[16]="pb-trading-transfer-1.0-SNAPSHOT.jar"
204 SERVICES_FRIENDLY_NAME[16]="pb-trading-transfer"
205 SERVICES_START_CMD[16]="./pb-trading-transfer.sh start"
206
207 SERVICES_PROCESS_ID[17]="rapidx-trading-clearing-1.0-SNAPSHOT.jar"
208 SERVICES_FRIENDLY_NAME[17]="rapidx-trading-clearing"
209 SERVICES_START_CMD[17]="./rapidx-trading-clearing.sh start"
210
211 SERVICES_PROCESS_ID[18]="pb-trading-monitor-1.0-SNAPSHOT.jar"
212 SERVICES_FRIENDLY_NAME[18]="pb-trading-monitor"
213 SERVICES_START_CMD[18]="./pb-trading-monitor.sh start"
214
215 SERVICES_PROCESS_ID[19]="rapidx-trading-query-persistent-1.0-SNAPSHOT.jar"
216 SERVICES_FRIENDLY_NAME[19]="rapidx-trading-query-persistent"
217 SERVICES_START_CMD[19]="./deploy-persistent-new.sh start"
218
219 SERVICES_PROCESS_ID[20]="rapidtrade-storage-1.0-SNAPSHOT.jar"
220 SERVICES_FRIENDLY_NAME[20]="rapidtrade-storage"
221 SERVICES_START_CMD[20]="./rapidtrade-storage.sh start"
222
223 SERVICES_PROCESS_ID[21]="bitu-trade-1.0-SNAPSHOT.jar"
224 SERVICES_FRIENDLY_NAME[21]="bitu-trade"
225 SERVICES_START_CMD[21]=bitu-trade.sh
226 #SERVICES_START_CMD[21]="./deploy.sh start"
227
228 SERVICES_PROCESS_ID[22]="ltp-data-integration-1.0-SNAPSHOT.jar"
229 SERVICES_FRIENDLY_NAME[22]="ltp-data-integration"
230 SERVICES_START_CMD[22]="./ltp-data-integration.sh start"
231
232 SERVICES_PROCESS_ID[23]="ingest-server-app-1.0-SNAPSHOT.jar"
233 SERVICES_FRIENDLY_NAME[23]="ingest-server-app"
234 SERVICES_START_CMD[23]="./data-ingest-server.sh start"
235
236 SERVICES_PROCESS_ID[24]="ltp-data-visual-1.0-SNAPSHOT.jar"
237 SERVICES_FRIENDLY_NAME[24]="ltp-data-visual"
238 SERVICES_START_CMD[24]="./data-cam-visual.sh start"
239
240
241 # ========== 主循环 ==========
242 mkdir -p "$(dirname "$MONITOR_LOG_FILE")"
243
244 while true; do
245 for i in "${!SERVICES_PROCESS_ID[@]}"; do
246 process_id="${SERVICES_PROCESS_ID[$i]}"
247 friendly_name="${SERVICES_FRIENDLY_NAME[$i]}"
248 start_cmd="${SERVICES_START_CMD[$i]}"
249
250 if ! pgrep -f "$process_id" > /dev/null; then
251 echo "[$(date '+%Y-%m-%d %H:%M:%S')] - 服务 [$friendly_name] (进程标识: $process_id) 未运行，正在重启..." | tee -a "$MONITOR_LOG_FILE"
252
253 # 在子shell中启动服务，确保环境变量正确传递
254 (
255 # 再次加载环境变量确保子shell中有正确的环境
256 for env_file in "${ENV_FILES[@]}"; do
257 if [ -f "$env_file" ]; then
258 source "$env_file"
259 fi
260 done
261
262 # 手动设置关键环境变量作为备用
263 export JAVA_HOME="/opt/jdk-17.0.8"
264 export CLASSPATH=".:$JAVA_HOME/lib/tools.jar:$JAVA_HOME/lib/dt.jar"
265 export PATH="$JAVA_HOME/bin:$PATH"
266 export env_nacos_address="nacos.test.com:8848"
267 export env_nacos_namespace="fat"
268 export ENV="fat"
269
270 cd "$WORK_DIR"
271 echo "[$(date '+%Y-%m-%d %H:%M:%S')] - 启动目录: $(pwd), JAVA_HOME: $JAVA_HOME" >> "$MONITOR_LOG_FILE"
272 $start_cmd >> "$MONITOR_LOG_FILE" 2>&1
273 ) &
274
275 send_alert_async "服务 [$friendly_name] (进程标识: $process_id) 已停止运行！正在尝试重启。"
276 else
277 echo "[$(date '+%Y-%m-%d %H:%M:%S')] - 服务 [$friendly_name] 运行正常." >> "$MONITOR_LOG_FILE"
278 fi
279 done
280
281 sleep 10
282 done

复制代码

View Code

3.2 使用开源通用软件统一维护自动拉起(开发+运维都可以简单维护和使用)　　　　　　　　

1 #程序安装
2 dnf install -y gcc make openssl-devel bison flex zlib-devel
3 #apt install -y gcc make libssl-dev bison flex zlib1g-dev
4 #yum install -y gcc make openssl-devel bison flex zlib-devel
5 wget https://mmonit.com/monit/dist/monit-5.34.0.tar.gz
6 tar xf monit-5.34.0.tar.gz && cd monit-5.34.0/
7 ./configure --prefix=/usr/local/monit --without-pam && make && make install
8 mkdir /usr/local/monit/etc -p && mkdir -p /usr/local/monit/etc/
9 cp monitrc /usr/local/monit/etc/
10 chmod 600 /usr/local/monit/etc/monitrc #配置文件定义检测时间，检测配置文件
11 ln -s /usr/local/monit/bin/monit /usr/sbin/monit
12 monit --version
13 mkdir /etc/monit/conf.d/ -p #所有进程检测配置文件，如启动脚本变动修改这里即可
14 cp /usr/local/monit/bin/monit /usr/bin/
15
16
17 #systemd配置：vi /etc/systemd/system/monit.service
18 [Unit]
19 Description=Monit process monitor
20 Documentation=https://mmonit.com/monit/
21 After=network.target
22
23 [Service]
24 Type=forking
25 ExecStart=/usr/bin/monit -c /usr/local/monit/etc/monitrc
26 ExecReload=/usr/bin/monit -c /usr/local/monit/etc/monitrc reload
27 ExecStop=/usr/bin/monit -c /usr/local/monit/etc/monitrc quit
28 PIDFile=/var/run/monit.pid
29 Restart=on-failure
30 User=root
31 Group=root
32
33 [Install]
34 WantedBy=multi-user.target # 多用户模式下开机自启
35
36 systemctl reload monit
37 systemctl enable monit
38 systemctl start monit

复制代码

3.2.1程序安装.sh

1 #进程配置方式vim /usr/local/monit/etc/monitrc：
2 set daemon 10
3 set logfile /var/log/monit.log
4
5 # 服务配置 /etc/monit/conf.d/rapidtrade-mock.conf
6 check process rapidtrade_mock matching "rapidtrade-mock"
7 start program = "/data/scripts/rapidtrade-mock.sh start"
8 stop program = "/data/scripts/rapidtrade-mock.sh stop"
9 if does not exist then start
10
11
12
13 #port配置方式：
14 check host rapidtrade_mock with address 127.0.0.1
15 if failed
16 port 7040
17 type tcp
18 timeout 5 seconds
19 for 2 cycles
20 then start
21 start program = "/data/scripts/rapidtrade-mock.sh start" as uid root and gid root
22 stop program = "/data/scripts/rapidtrade-mock.sh stop" as uid root and gid root
23 if 3 restarts within 5 cycles then timeout
24
25
26
27
28 #健康监测端口+路径:经过测试有问题，无法启动服务并恢复正常。
29 check host my_web_service with address 127.0.0.1
30 if failed
31 port 80
32 protocol http
33 request "/actuator/prometheus" # 指定要检查的健康检查端点路径
34 with timeout 10 seconds
35 for 3 cycles
36 then restart
37 start program = "/usr/bin/systemctl start my-service"
38 stop program = "/usr/bin/systemctl stop my-service"

复制代码

3.2.2 配置使用

4.测试和使用

这样基于传统服务，只需要改程序有对应的start.sh stop.sh脚本就可以简单配置进程健康检测来维护服务自动拉起，简单高效不需要每个团队开发很多自己的脚本去维护。
来源：程序园用户自行投稿发布，如果侵权，请联系站长删除
免责声明：如果侵犯了您的权益，请联系站长，我们会及时删除侵权内容，谢谢合作！

账号		自动登录	找回密码
密码			立即注册

Monit-基于非容器服务自恢复程序实践

相关帖子

签约作者

Monit-基于非容器服务自恢复程序实践

相关帖子

相关推荐

签约作者