Prometheus Alertmanager 경고 구성 요소

Prometheus Alertmanager

개요

Alertmanager 소개 및 메커니즘

그룹화

억제(Inhibition)

침묵(Silences)

Alermanager 구성

프로파일

global:
  # ResolveTimeout is the time after which an alert is declared resolved
  # if it has not been updated.
  # 
  [ resolve_timeout:  | default = 5m ]

  # The default SMTP From header field.
  [ smtp_from:  ]
  # The default SMTP smarthost used for sending emails.
  [ smtp_smarthost:  ]
  # SMTP authentication information.
  [ smtp_auth_username:  ]
  [ smtp_auth_password:  ]
  [ smtp_auth_secret:  ]
  # The default SMTP TLS requirement.
  [ smtp_require_tls:  | default = true ]

  # The API URL to use for Slack notifications.
  [ slack_api_url:  ]

  [ pagerduty_url:  | default = "https://events.pagerduty.com/generic/2010-04-15/create_event.json" ]
  [ opsgenie_api_host:  | default = "https://api.opsgenie.com/" ]

# Files from which custom notification template definitions are read.
# The last component may use a wildcard matcher, e.g. 'templates/*.tmpl'.
templates:
  [ -  ... ]

# The root node of the routing tree.
route: 

# A list of notification receivers.
receivers:
  -  ...

# A list of inhibition rules.
inhibit_rules:
  [ -  ... ]

라우팅(route)

# 
[ receiver:  ]

# 
[ group_by: '[' , ... ']' ]

# Whether an alert should continue matching subsequent sibling nodes.
[ continue:  | default = false ]

# A set of equality matchers an alert has to fulfill to match the node.
# ， 
match:
  [ : , ... ]

# A set of regex-matchers an alert has to fulfill to match the node.
match_re:
# ， 
  [ : , ... ]

# How long to initially wait to send a notification for a group
# of alerts. Allows to wait for an inhibiting alert to arrive or collect
# more initial alerts for the same group. (Usually ~0s to few minutes.)
[ group_wait:  ]

# How long to wait before sending notification about new alerts that are
# in are added to a group of alerts for which an initial notification
# has already been sent. (Usually ~5min or more.)
[ group_interval:  ]

# How long to wait before sending a notification again if it has already
# been sent successfully for an alert. (Usually ~3h or more).
[ repeat_interval:  ]

# Zero or more child routes.
routes:
  [ -  ... ]

# The root route with all parameters, which are inherited by the child
# routes if they are not overwritten.
route:
  receiver: 'default-receiver'
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  group_by: [cluster, alertname]
  # All alerts that do not match the following child routes
  # will remain at the root node and be dispatched to 'default-receiver'.
  routes:
  # All alerts with service=mysql or service=cassandra
  # are dispatched to the database pager.
  - receiver: 'database-pager'
    group_wait: 10s
    match_re:
      service: mysql|cassandra
  # All alerts with the team=frontend label match this sub-route.
  # They are grouped by product and environment rather than cluster
  # and alertname.
  - receiver: 'frontend-pager'
    group_by: [product, environment]
    match:
      team: frontend

억제 규칙 inhibit_rule

# Matchers that have to be fulfilled in the alerts to be muted.
## 
target_match:
  [ : , ... ]
target_match_re:
  [ : , ... ]

# Matchers for which one or more alerts have to exist for the
# inhibition to take effect.
# 。
source_match:
  [ : , ... ]
source_match_re:
  [ : , ... ]

# Labels that must have an equal value in the source and target
# alert for the inhibition to take effect.
# 
[ equal: '[' , ... ']' ]

수신기(receiver)

# The unique name of the receiver.
name: 

# Configurations for several notification integrations.
email_configs:
  [ - , ... ]
pagerduty_configs:
  [ - , ... ]
slack_config:
  [ - , ... ]
opsgenie_configs:
  [ - , ... ]
webhook_configs:
  [ - , ... ]

# Whether or not to notify about resolved alerts.
# 
[ send_resolved:  | default = false ]

# The email address to send notifications to.
to: 
# The sender address.
[ from:  | default = global.smtp_from ]
# The SMTP host through which emails are sent.
[ smarthost:  | default = global.smtp_smarthost ]

# The HTML body of the email notification.
[ html:  | default = '{{ template "email.default.html" . }}' ] 

# Further headers email header key/value pairs. Overrides any headers
# previously set by the notification implementation.
[ headers: { : , ... } ]

# Whether or not to notify about resolved alerts.
[ send_resolved:  | default = true ]

# The Slack webhook URL.
[ api_url:  | default = global.slack_api_url ]

# The channel or user to send notifications to.
channel: 

# API request data as defined by the Slack webhook API.
[ color:  | default = '{{ if eq .Status "firing" }}danger{{ else }}good{{ end }}' ]
[ username:  | default = '{{ template "slack.default.username" . }}'
[ title:  | default = '{{ template "slack.default.title" . }}' ]
[ title_link:  | default = '{{ template "slack.default.titlelink" . }}' ]
[ pretext:  | default = '{{ template "slack.default.pretext" . }}' ]
[ text:  | default = '{{ template "slack.default.text" . }}' ]
[ fallback:  | default = '{{ template "slack.default.fallback" . }}' ]

 # Whether or not to notify about resolved alerts.
[ send_resolved:  | default = true ]

 # The endpoint to send HTTP POST requests to.
url: 

{
  "version": "3",
  "groupKey":      // key identifying the group of alerts (e.g. to deduplicate)
  "status": "",
  "receiver": ,
  "groupLabels": 

from flask import Flask
from flask import request
import json

app = Flask(__name__)

@app.route('/',methods=['POST'])
def send():
    if request.method == 'POST':
        post_data = request.get_data()
        alert_data(post_data)
    return
def alert_data(data):
    from urllib2 import Request,urlopen
    url = 'https://oapi.dingtalk.com/robot/send?access_token=xxxx'
    send_data = '{"msgtype": "text","text": {"content": %s}}' %(data)
    request = Request(url, send_data)
    request.add_header('Content-Type','application/json')
    return urlopen(request).read()
if __name__ == '__main__':
    app.run(host='0.0.0.0')

경보 규칙

경고 규칙 정의

ALERT 
  IF 
  [ FOR  ]
  [ LABELS  ]
  [ ANNOTATIONS  ]

# To insert a firing element's label values:
{{ $labels. }}
# To insert the numeric expression value of the firing element:
{{ $value }}

# Alert for any instance that is unreachable for >5 minutes.
ALERT InstanceDown
  IF up == 0
  FOR 5m
  LABELS { severity = "page" }
  ANNOTATIONS {
    summary = "Instance {{ $labels.instance }} down",
    description = "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes.",
  }

# Alert for any instance that have a median request latency >1s.
ALERT APIHighRequestLatency
  IF api_http_request_latencies_second{quantile="0.5"} > 1
  FOR 1m
  ANNOTATIONS {
    summary = "High request latency on {{ $labels.instance }}",
    description = "{{ $labels.instance }} has a median request latency above 1s (current value: {{ $value }}s)",
  }

런타임 경고 확인

경보 알림 보내기