服务器资源监控告警设置

# Ubuntu 24.04 LTS 部署 node_exporter 1.8（截至 2026 年稳定版本）
sudo useradd --no-create-home --shell /bin/false node_exporter
wget https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz
tar xzf node_exporter-1.8.2.linux-amd64.tar.gz
sudo cp node_exporter-1.8.2.linux-amd64/node_exporter /usr/local/bin/
sudo chown node_exporter:node_exporter /usr/local/bin/node_exporter

sudo tee /etc/systemd/system/node_exporter.service >/dev/null <<'EOF'
[Unit]
Description=Prometheus Node Exporter
After=network-online.target

[Service]
User=node_exporter
ExecStart=/usr/local/bin/node_exporter \
  --collector.systemd \
  --collector.processes \
  --no-collector.wifi
Restart=always
RestartSec=3

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload
sudo systemctl enable --now node_exporter
curl -s http://127.0.0.1:9100/metrics | grep -E '^node_load5|^node_memory_MemTotal_bytes'

# /etc/prometheus/rules/host.yml
groups:
  - name: host-resource
    rules:
      - alert: HostDown
        expr: up{job="node"} == 0
        for: 1m
        labels:
          severity: critical      # P0
          team: ops
        annotations:
          summary: "主机失联：{{ $labels.instance }}"
          description: "node_exporter 连续 1 分钟抓取失败，请检查主机、网络与 9100 端口。"
          runbook: "https://wiki.example.com/runbook/host-down"

      - alert: DiskWillFillIn4Hours
        expr: predict_linear(node_filesystem_avail_bytes{fstype=~"ext4|xfs",mountpoint!~"/boot.*"}[6h], 4*3600) < 0
        for: 5m
        labels:
          severity: warning       # P1
          team: ops
        annotations:
          summary: "磁盘将满：{{ $labels.instance }} {{ $labels.mountpoint }}"
          description: "按当前写入速度，4 小时内将写满，立即清理或扩容。"
          value: "剩余 {{ $value | humanize }} bytes"

      - alert: HighCPUUsage
        expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 85
        for: 10m
        labels:
          severity: info          # P2
          team: ops
        annotations:
          summary: "CPU 使用率过高：{{ $labels.instance }}"
          description: "CPU 使用率持续 10 分钟超过 85%，当前 {{ $value | printf \"%.1f\" }}%。"

      - alert: SSLCertExpiringSoon
        expr: (probe_ssl_earliest_cert_expiry - time()) / 86400 < 15
        for: 30m
        labels:
          severity: info
          team: ops
        annotations:
          summary: "SSL 证书即将过期：{{ $labels.instance }}"
          description: "证书剩余有效期不足 15 天，请尽快续签。"

# 校验规则语法（promtool 随 Prometheus 一起安装）
promtool check rules /etc/prometheus/rules/host.yml

# 让 Prometheus 热加载配置（需启动时加 --web.enable-lifecycle）
curl -X POST http://127.0.0.1:9090/-/reload

# 在 UI 上确认规则已加载
curl -s http://127.0.0.1:9090/api/v1/rules | jq '.data.groups[].rules[].name'

# /etc/alertmanager/alertmanager.yml
global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.exmail.qq.com:465'
  smtp_from: 'alert@example.com'
  smtp_auth_username: 'alert@example.com'
  smtp_auth_password: '替换为应用专用密码'

templates:
  - '/etc/alertmanager/templates/*.tmpl'

route:
  receiver: 'default-im'
  group_by: ['alertname', 'instance', 'severity']
  group_wait: 30s        # 首次等待，攒一批再发
  group_interval: 5m     # 同组新告警合并间隔
  repeat_interval: 4h    # 未恢复告警的重发间隔
  routes:
    - matchers:
        - severity="critical"
      receiver: 'p0-phone'
      group_wait: 10s
      repeat_interval: 30m
      continue: false
    - matchers:
        - severity="warning"
      receiver: 'p1-im'
      repeat_interval: 1h
    - matchers:
        - severity="info"
      receiver: 'daily-mail'
      group_wait: 5m
      repeat_interval: 24h

receivers:
  - name: 'default-im'
    webhook_configs:
      - url: 'http://127.0.0.1:8060/dingtalk/ops/send'
        send_resolved: true

  - name: 'p0-phone'
    webhook_configs:
      - url: 'http://127.0.0.1:8060/dingtalk/p0/send'
        send_resolved: true
    email_configs:
      - to: 'oncall@example.com'
        send_resolved: true

  - name: 'p1-im'
    webhook_configs:
      - url: 'http://127.0.0.1:8060/dingtalk/ops/send'
        send_resolved: true

  - name: 'daily-mail'
    email_configs:
      - to: 'ops@example.com'
        send_resolved: false

inhibit_rules:
  # 主机已经失联时，抑制该主机上的一切资源类告警
  - source_matchers: ['alertname="HostDown"']
    target_matchers: ['severity="info"']
    equal: ['instance']
  # P0 触发时抑制同实例的 P1，避免刷屏
  - source_matchers: ['severity="critical"']
    target_matchers: ['severity="warning"']
    equal: ['instance']

amtool check-config /etc/alertmanager/alertmanager.yml
amtool config routes test --config.file=/etc/alertmanager/alertmanager.yml \
  severity="critical" instance="web-01:9100"

# 手动造一条告警，验证通知链路是否真的能收到
amtool alert add alertname=TestAlert severity=critical instance=web-01:9100 \
  --alertmanager.url=http://127.0.0.1:9093

# /etc/alertmanager/templates/dingtalk.tmpl
{{ define "dingtalk.message" }}
## [{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}（{{ .CommonLabels.severity }}）
{{ range .Alerts }}
- **级别**：{{ .Labels.severity }}
- **实例**：{{ .Labels.instance }}
- **摘要**：{{ .Annotations.summary }}
- **详情**：{{ .Annotations.description }}
- **开始**：{{ .StartsAt.Format "2006-01-02 15:04:05" }}
{{ end }}
> 值班人请按 Runbook 处理，处理完在群内回复"已恢复"。
{{ end }}

# 静默 web-01 上所有告警 2 小时（用于计划内维护）
amtool silence add --alertmanager.url=http://127.0.0.1:9093 \
  --author="zhangsan" --comment="发版维护窗口" \
  --duration=2h instance="web-01:9100"

amtool silence query --alertmanager.url=http://127.0.0.1:9093