从零搭建工业级监控告警体系：告警规则设计、分级策略与告警风暴治理

┌─────────────────────────────┐
│   业务层：核心业务指标        │  设备在线率、数据采集成功率、工单处理时效
├─────────────────────────────┤
│   应用层：微服务运行指标      │  接口 RT、错误率、吞吐量、JVM 指标
├─────────────────────────────┤
│   中间件层：基础组件指标      │  MySQL QPS、Redis 命中率、Kafka 消费延迟
├─────────────────────────────┤
│   基础设施层：服务器资源指标   │  CPU、内存、磁盘、网络
└─────────────────────────────┘

@Component
public class DeviceMetrics {
​
    private final MeterRegistry registry;
    private final AtomicInteger onlineDeviceCount;
    private final Counter dataCollectSuccessCounter;
    private final Counter dataCollectFailCounter;
    private final Timer dataProcessTimer;
​
    public DeviceMetrics(MeterRegistry registry) {
        this.registry = registry;
​
        // 设备在线数（Gauge 类型，反映当前状态）
        this.onlineDeviceCount = registry.gauge("device.online.count",
            new AtomicInteger(0));
​
        // 数据采集成功/失败计数器（Counter 类型，只增不减）
        this.dataCollectSuccessCounter = registry.counter("device.collect",
            "result", "success");
        this.dataCollectFailCounter = registry.counter("device.collect",
            "result", "fail");
​
        // 数据处理耗时（Timer 类型，记录分布）
        this.dataProcessTimer = registry.timer("device.process.duration");
    }
​
    public void recordCollectSuccess() {
        dataCollectSuccessCounter.increment();
    }
​
    public void recordCollectFail(String reason) {
        dataCollectFailCounter.increment();
        registry.counter("device.collect.fail.reason",
            "reason", reason).increment();
    }
​
    public void recordProcessDuration(long millis) {
        dataProcessTimer.record(millis, TimeUnit.MILLISECONDS);
    }
​
    public void updateOnlineCount(int count) {
        onlineDeviceCount.set(count);
    }
}

# Prometheus 告警规则
groups:
  - name: application_alerts
    rules:
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(http_server_requests_seconds_count{status=~"5.."}[5m])) by (application, uri)
            /
            sum(rate(http_server_requests_seconds_count[5m])) by (application, uri)
          ) > 0.05
        for: 3m
        labels:
          severity: warning
          team: backend
        annotations:
          summary: "{{ $labels.application }} 接口 {{ $labels.uri }} 错误率过高"
          description: "过去5分钟错误率 {{ $value | humanizePercentage }}，超过5%阈值，已持续3分钟"

# 设备在线率下降
- alert: DeviceOnlineRateDrop
  expr: device_online_count / device_total_count < 0.85
  for: 5m
  labels:
    severity: critical
​
# Kafka 消费延迟（设备数据处理积压）
- alert: KafkaConsumerLagHigh
  expr: kafka_consumergroup_lag_sum{group="device-data-consumer"} > 10000
  for: 5m
  labels:
    severity: critical

# 接口 P99 响应时间过高
- alert: HighP99Latency
  expr: histogram_quantile(0.99, rate(http_server_requests_seconds_bucket[5m])) > 2
  for: 3m
  labels:
    severity: warning
​
# JVM 堆内存使用率过高
- alert: JvmHeapHigh
  expr: jvm_memory_used_bytes{area="heap"} / jvm_memory_max_bytes{area="heap"} > 0.85
  for: 5m
  labels:
    severity: warning
​
# Full GC 频繁
- alert: FrequentFullGC
  expr: increase(jvm_gc_pause_seconds_count{action="end of major GC"}[10m]) > 3
  for: 0m
  labels:
    severity: warning

# MySQL 慢查询激增
- alert: MysqlSlowQueries
  expr: increase(mysql_global_status_slow_queries[5m]) > 20
  for: 3m
  labels:
    severity: warning
​
# Redis 内存使用率过高
- alert: RedisMemoryHigh
  expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.8
  for: 5m
  labels:
    severity: warning
​
# MySQL 主从延迟
- alert: MysqlReplicationLag
  expr: mysql_slave_status_seconds_behind_master > 30
  for: 2m
  labels:
    severity: critical

# CPU 使用率持续过高
- alert: HighCPU
  expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
  for: 10m
  labels:
    severity: warning
​
# 磁盘空间不足
- alert: DiskSpaceLow
  expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes) < 0.15
  for: 5m
  labels:
    severity: critical

# alertmanager.yml
route:
  receiver: 'default-webhook'
  group_by: ['alertname', 'application']
  group_wait: 30s        # 同组告警等待30秒聚合后再发送
  group_interval: 5m     # 同组告警的发送间隔
  repeat_interval: 4h    # 已发送告警的重复提醒间隔
​
  routes:
    # P0: 电话 + 短信 + 钉钉
    - match:
        severity: critical
      receiver: 'critical-all-channels'
      group_wait: 10s      # 紧急告警缩短聚合等待
      repeat_interval: 30m # 未处理的话每30分钟重复提醒
​
    # P1: 钉钉群 + 邮件
    - match:
        severity: warning
      receiver: 'warning-dingtalk-email'
      repeat_interval: 2h
​
    # P2: 仅邮件
    - match:
        severity: info
      receiver: 'info-email'
      repeat_interval: 12h
​
receivers:
  - name: 'critical-all-channels'
    webhook_configs:
      - url: 'http://alert-gateway:8080/api/alert/critical'
        # 内部网关，负责同时触发电话、短信、钉钉
​
  - name: 'warning-dingtalk-email'
    webhook_configs:
      - url: 'https://oapi.dingtalk.com/robot/send?access_token=xxx'
    email_configs:
      - to: 'backend-oncall@company.com'
​
  - name: 'info-email'
    email_configs:
      - to: 'backend-team@company.com'

group_by: ['alertname', 'application']
group_wait: 30s

inhibit_rules:
  # 如果已有数据库不可用的 critical 告警，
  # 则抑制由此引发的应用层 warning 告警
  - source_match:
      alertname: MysqlDown
      severity: critical
    target_match:
      severity: warning
    equal: ['cluster']   # 同一集群内的告警才抑制

  # 如果节点宕机了，抑制该节点上所有应用的告警
  - source_match:
      alertname: NodeDown
      severity: critical
    target_match:
      severity: warning
    equal: ['instance']

# 创建静默规则：在 2025-01-20 02:00-04:00 屏蔽 MySQL 相关的所有告警
amtool silence add \
  alertname=~"Mysql.*" \
  --start="2025-01-20T02:00:00+08:00" \
  --end="2025-01-20T04:00:00+08:00" \
  --comment="MySQL主从切换维护窗口"