博文推荐｜通过 Prometheus Alertmanager 模块发送 Doris 异常信息至钉钉报警群

➜ groupadd prometheus
➜ useradd -g prometheus -M -s /sbin/nologin prometheus

➜ wget https://github.com/prometheus/alertmanager/releases/download/v0.23.0/alertmanager-0.23.0.linux-amd64.tar.gz

➜ tar xf alertmanager-0.23.0.linux-amd64.tar.gz -C /soft
➜ cd /soft
➜ mv alertmanager-0.23.0.linux-amd64 alertmanager
➜ cd alertmanager
➜ mkdir data #系统文件持久化路径
➜ chown -R prometheus.prometheus /soft/alertmanager

route:
  group_by: ['alertname']
  group_wait: 1s
  group_interval: 1m
  repeat_interval: 4h
  receiver: 'webhook2'
receivers:
- name: 'webhook2'
  webhook_configs:
    - &dingtalk_config
       send_resolved: true
       url: http://localhost:8060/dingtalk/webhook2/send
#An inhibition rule mutes an alert (target) matching a set of matchers when an alert (source) exists that matches another set of matchers. Both target and source alerts must have the same label values for the label names in the equal list.
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'dev', 'instance']

➜ cd /soft/alertmanager
➜ ./amtool check-config ./alertmanager.yml

➜ vim /usr/lib/systemd/system/alertmanager.service

[Unit]
Description=alertmanager
Documentation=https://prometheus.io/
After=network.target


[Service]
User=prometheus
Group=prometheus
ExecStart=/soft/alertmanager/alertmanager --config.file=/soft/alertmanager/alertmanager.yml --storage.path=/soft/alertmanager/data
Restart=on-failure


[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable alertmanager.service
systemctl start alertmanager.service
systemctl status alertmanager.service #查看服务状态
systemctl restart alertmanager.service #重启服务使用

vim prometheus.yml 

alerting:
  alertmanagers:
  - static_configs:
    - targets: ['localhost:9093']  #根据实际alertmanager服务地址配置。


rule_files:
   - "rule/*.yml"   #自定义规则存储目录，可以通过*.yml来通配该目录下所有规则。

➜ ./promtool check config  prometheus.yml
➜ systemctl reload prometheus.service

➜ wget https://github.com/timonwong/prometheus-webhook-dingtalk/releases/download/v1.4.0/prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz

➜ tar -xf prometheus-webhook-dingtalk-1.4.0.linux-amd64.tar.gz  -C  /soft
➜ mv prometheus-webhook-dingtalk-1.4.0.linux-amd64 prometheus-webhook-dingtalk

➜ cp  config.example.yml  config.yml
➜ vim config.yml

## Request timeout
# timeout: 5s


## Customizable templates path   自定义模板位置
templates:
  - /soft/alertmanager/alarm_template/webhook.tmpl
## You can also override default template using `default_message`
## The following example to use the 'legacy' template from v0.3.0
# default_message:
#   title: '{{ template "legacy.title" . }}'
#   text: '{{ template "legacy.content" . }}'


## Targets, previously was known as "profiles"
targets:
  webhook1:   #加签的钉钉，需要同时配置加签机器人的密钥信息
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    # secret for signature
    secret: SEC000000000000000000000
  webhook2:  #不加签钉钉
    url: https://oapi.dingtalk.com/robot/send?access_token=cf9c2fd69723661108b7fd7******
  webhook_legacy:   
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    # Customize template content
    message:
      # Use legacy template
      title: '{{ template "legacy.title" . }}'
      text: '{{ template "legacy.content" . }}'
  webhook_mention_all:  #@所有人钉钉
    url: https://oapi.dingtalk.com/robot/send?access_token=xxxxxxxxxxxx
    mention:
      all: true
  webhook_mention_users:  #@指定用户钉钉
    url: https://oapi.dingtalk.com/robot/send?access_token=cf9c2fd69723661108b7fd7****
    mention:
      mobiles: ['152****30', '134****74']

vim /usr/lib/systemd/system/webhook-dingtalk.service

[Unit]
Description=prometheus-webhook-dingtalk
Documentation=https://github.com/timonwong/prometheus-webhook-dingtalk
After=network.target


[Service]
User=prometheus
Group=prometheus
ExecStart=/soft/prometheus-webhook-dingtalk/prometheus-webhook-dingtalk  --config.file=/soft/prometheus-webhook-dingtalk/config.yml
Restart=on-failure


[Install]
WantedBy=multi-user.target

systemctl daemon-reload
systemctl enable webhook-dingtalk.service
systemctl start webhook-dingtalk.service
systemctl status webhook-dingtalk.service
systemctl restart webhook-dingtalk.service

➜ vim doris_instance.yml

groups:
  - name: doris_instance_down
    rules:
      - alert: Doris Backends Down
        expr: up {group="be", job="pro-doris"} == 0
        for: 20s
        labels:
          user: doris
          severity: error
        annotations:
          summary: "doris Instance {{ $labels.instance }} down"
          description: "doris {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 20s."
      - alert: Doris Frontends Down
        expr: up {group="fe", job="pro-doris"} == 0
        for: 20s
        labels:
          user: doris
          severity: error
        annotations:
          summary: "doris Instance {{ $labels.instance }} down"
          description: "doris {{ $labels.instance }} of job {{ $labels.job }} has been down for more than 20s."

➜ cd /soft/prometheus
➜ ./promtool check config  prometheus.yml

➜ systemctl reload prometheus.service

—— End ——