服务器日志分析入门

# 开启 journald 持久化
mkdir -p /var/log/journal
systemd-tmpfiles --create --prefix /var/log/journal
systemctl restart systemd-journald
ls -la /var/log/journal/        # 出现机器 ID 目录即成功

# 限制 journal 占用（避免撑爆磁盘）
cat > /etc/systemd/journald.conf.d/99-limit.conf <<'EOF'
[Journal]
SystemMaxUse=4G
SystemKeepFree=2G
MaxRetentionSec=180day
Compress=yes
EOF
systemctl restart systemd-journald
journalctl --disk-usage        # 查看当前占用

# 1. 看服务状态与最后几行日志
systemctl status nginx --no-pager -l

# 2. 看该服务本次启动的全部日志，按优先级过滤错误
journalctl -u nginx -b -p warning --no-pager

# 3. 查看服务启动失败的具体原因（含退出码）
journalctl -u nginx -b -o cat | grep -iE "fail|error|denied|address already in use" | head -20

# 4. 排查反复重启的服务：列出每次启动的时间，计算重启间隔
systemctl show nginx -p NRestarts
journalctl -u nginx --since "24 hours ago" | grep -c "Started"

# 5. 导出日志供分析（发给同事或留存证据）
journalctl -u nginx --since "2026-03-12 14:00" --until "2026-03-12 15:00" > /tmp/nginx_1400.log

log_format combined '$remote_addr - $remote_user [$time_local] '
                    '"$request" $status $body_bytes_sent '
                    '"$http_referer" "$http_user_agent"';

203.0.113.45 - - [12/Mar/2026:14:23:11 +0800] "GET /api/orders?page=1 HTTP/1.1" 200 1532 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/122.0"

# /etc/nginx/nginx.conf  http 段
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
                '$status $body_bytes_sent "$http_referer" '
                '"$http_user_agent" "$http_x_forwarded_for" '
                'rt=$request_time urt=$upstream_response_time '
                'ua=$upstream_addr';
access_log /var/log/nginx/access.log main;

# TOP 20 访问来源 IP（第 1 列）
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20

# 带百分比的完整版
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20 \
  | awk '{printf "%-18s %8d\n", $2, $1}'

# 统计某个时间段内的 TOP IP（用 grep 先过滤时间）
grep "12/Mar/2026:14:" /var/log/nginx/access.log | awk '{print $1}' | sort | uniq -c | sort -rn | head -10

# 若配置了 X-Forwarded-For，取真实 IP（按你的日志格式调整列号）
awk -F'"' '{print $(NF-3)}' /var/log/nginx/access.log | awk -F, '{print $1}' \
  | sort | uniq -c | sort -rn | head -20

# 状态码分布（$9 是状态码列，使用默认 combined 格式时）
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

# 只看错误码（4xx/5xx）并按状态码+URL 聚合
awk '$9 ~ /^5[0-9][0-9]$/ {print $9, $7}' /var/log/nginx/access.log \
  | sort | uniq -c | sort -rn | head -20

# 5xx 占比（判断是否触发告警阈值）
awk '{total++; if ($9 ~ /^5/) err++} END {printf "总请求 %d，5xx %d，占比 %.2f%%\n", total, err, err/total*100}' \
  /var/log/nginx/access.log

# 按分钟统计 5xx 数量（定位故障起始时间点）
awk '$9 ~ /^5/ {split($4, a, ":"); print a[2]":"a[3]}' /var/log/nginx/access.log \
  | sort | uniq -c | sort -rn | head -20

# 404 TOP URL（常能发现扫描器与失效链接）
awk '$9==404 {print $7}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20

# 提取耗时超过 1 秒的请求，按耗时倒序
awk -F'rt=' '{split($2, a, " "); if (a[1]+0 > 1) print a[1], $0}' /var/log/nginx/access.log \
  | sort -rn | head -20

# 慢请求 TOP URL（按 URL 聚合平均耗时）
awk -F'rt=' '{split($2, a, " "); rt=a[1]+0; split($1, b, " "); url=b[7]; \
              sum[url]+=rt; cnt[url]++} \
     END {for (u in sum) printf "%.3f %6d %s\n", sum[u]/cnt[u], cnt[u], u}' \
  /var/log/nginx/access.log | sort -rn | head -20

# 统计 P95/P99 耗时（近似：排序后取分位）
awk -F'rt=' '{split($2, a, " "); print a[1]+0}' /var/log/nginx/access.log \
  | sort -n > /tmp/rt.txt
awk '{a[NR]=$1} END {printf "样本 %d, P50=%.3fs, P95=%.3fs, P99=%.3fs\n", \
     NR, a[int(NR*0.50)], a[int(NR*0.95)], a[int(NR*0.99)]}' /tmp/rt.txt

# 后端慢（upstream_response_time 远大于 request_time 差值说明卡在 Nginx 自身）
awk -F'urt=' '{split($2, a, " "); if (a[1]+0 > 2) print $0}' /var/log/nginx/access.log | head -10

# QPS：按秒统计请求数（取峰值）
awk '{print $4}' /var/log/nginx/access.log | tr -d '[' | cut -d: -f2-4 \
  | sort | uniq -c | sort -rn | head -10

# UV（独立 IP 数）
awk '{print $1}' /var/log/nginx/access.log | sort -u | wc -l

# 流量总量（字节）
awk '{s+=$10} END {printf "总出口流量 %.2f GB\n", s/1024/1024/1024}' /var/log/nginx/access.log

# 扫描器识别：高频 404 且 UA 含扫描特征
awk '$9==404 {print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head
grep -iE "nmap|masscan|nikto|sqlmap|wpscan|dirbuster|curl/|python-requests" \
  /var/log/nginx/access.log | awk '{print $1}' | sort | uniq -c | sort -rn | head -10

# 实时观察（每秒请求数）
tail -f /var/log/nginx/access.log | awk '{c++; print c}'    # 或用 pv

# 1. 确认慢日志是否开启
mysql -uroot -p -e "SHOW VARIABLES LIKE 'slow_query_log%'; SHOW VARIABLES LIKE 'long_query_time';"

# 2. 临时开启（重启失效，生产建议写进 my.cnf）
mysql -uroot -p -e "SET GLOBAL slow_query_log=ON; SET GLOBAL long_query_time=1;"

# 3. 用 mysqldumpslow 聚合分析慢日志
mysqldumpslow -s t -t 10 /var/log/mysql/slow.log        # 按总耗时 Top10
mysqldumpslow -s c -t 10 /var/log/mysql/slow.log        # 按出现次数 Top10
mysqldumpslow -s at -t 10 /var/log/mysql/slow.log       # 按平均耗时 Top10

# 4. 用 pt-query-digest 做更专业分析（Percona Toolkit）
pt-query-digest /var/log/mysql/slow.log > /tmp/slow_report.txt
less /tmp/slow_report.txt

# 5. MySQL 错误日志中找崩溃与连接问题
grep -iE "abort|crash|shutdown|too many connections|out of memory" /var/log/mysql/error.log | tail -20

# /etc/promtail/config.yml
server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /var/lib/promtail/positions.yaml

clients:
  - url: http://10.0.0.100:3100/loki/api/v1/push

scrape_configs:
  - job_name: system
    static_configs:
      - targets: [localhost]
        labels:
          __path__: /var/log/{messages,syslog,secure,auth.log}
          host: ${HOSTNAME}
          job: system
  - job_name: nginx
    static_configs:
      - targets: [localhost]
        labels:
          __path__: /var/log/nginx/access.log
          host: ${HOSTNAME}
          job: nginx

# 统计 Nginx 5xx 数量
sum(count_over_time({job="nginx"} |~ " 5[0-9]{2} " [5m]))

# 查看某主机的错误日志
{host="web01", job="system"} |= "error" != "ignore_this"

# 统计 TOP IP（配合 LogQL 的解析）
topk(10, sum by (ip) (count_over_time({job="nginx"} | regexp `^(?P\S+)` [1h])))

# /etc/logrotate.d/nginx 示例
/var/log/nginx/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    create 0640 nginx adm
    sharedscripts
    postrotate
        [ -f /var/run/nginx.pid ] && kill -USR1 $(cat /var/run/nginx.pid)
    endscript
}

# 手动触发与调试
logrotate -d /etc/logrotate.d/nginx     # 调试模式，不实际执行
logrotate -f /etc/logrotate.d/nginx     # 强制执行

# /etc/rsyslog.d/90-remote.conf
*.* @@10.0.0.100:514        # @@ = TCP，@ = UDP
# 若需可靠传输，配置队列防止日志服务器不可达时丢日志
$ActionQueueFileName fwdRule
$ActionQueueMaxDiskSpace 1g
$ActionQueueSaveOnShutdown on
$ActionQueueType LinkedList
$ActionResumeRetryCount -1