forked from DevOps/deploy.stack
add(n9e): add all in one
This commit is contained in:
cnphpbb
216
n9e/compose-mysql/etc-nightingale/script/notify.bak.py
Normal file
216
n9e/compose-mysql/etc-nightingale/script/notify.bak.py
Normal file
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import sys
|
||||
import json
|
||||
import urllib2
|
||||
import smtplib
|
||||
from email.mime.text import MIMEText
|
||||
|
||||
reload(sys)
|
||||
sys.setdefaultencoding('utf8')
|
||||
|
||||
notify_channel_funcs = {
|
||||
"email":"email",
|
||||
"sms":"sms",
|
||||
"voice":"voice",
|
||||
"dingtalk":"dingtalk",
|
||||
"wecom":"wecom",
|
||||
"feishu":"feishu"
|
||||
}
|
||||
|
||||
mail_host = "smtp.163.com"
|
||||
mail_port = 994
|
||||
mail_user = "ulricqin"
|
||||
mail_pass = "password"
|
||||
mail_from = "ulricqin@163.com"
|
||||
|
||||
class Sender(object):
|
||||
@classmethod
|
||||
def send_email(cls, payload):
|
||||
if mail_user == "ulricqin" and mail_pass == "password":
|
||||
print("invalid smtp configuration")
|
||||
return
|
||||
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
|
||||
emails = {}
|
||||
for u in users:
|
||||
if u.get("email"):
|
||||
emails[u.get("email")] = 1
|
||||
|
||||
if not emails:
|
||||
return
|
||||
|
||||
recipients = emails.keys()
|
||||
mail_body = payload.get('tpls').get("email.tpl", "email.tpl not found")
|
||||
message = MIMEText(mail_body, 'html', 'utf-8')
|
||||
message['From'] = mail_from
|
||||
message['To'] = ", ".join(recipients)
|
||||
message["Subject"] = payload.get('tpls').get("subject.tpl", "subject.tpl not found")
|
||||
|
||||
try:
|
||||
smtp = smtplib.SMTP_SSL(mail_host, mail_port)
|
||||
smtp.login(mail_user, mail_pass)
|
||||
smtp.sendmail(mail_from, recipients, message.as_string())
|
||||
smtp.close()
|
||||
except smtplib.SMTPException, error:
|
||||
print(error)
|
||||
|
||||
@classmethod
|
||||
def send_wecom(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
|
||||
tokens = {}
|
||||
|
||||
for u in users:
|
||||
contacts = u.get("contacts")
|
||||
if contacts.get("wecom_robot_token", ""):
|
||||
tokens[contacts.get("wecom_robot_token", "")] = 1
|
||||
|
||||
opener = urllib2.build_opener(urllib2.HTTPHandler())
|
||||
method = "POST"
|
||||
|
||||
for t in tokens:
|
||||
url = "https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key={}".format(t)
|
||||
body = {
|
||||
"msgtype": "markdown",
|
||||
"markdown": {
|
||||
"content": payload.get('tpls').get("wecom.tpl", "wecom.tpl not found")
|
||||
}
|
||||
}
|
||||
request = urllib2.Request(url, data=json.dumps(body))
|
||||
request.add_header("Content-Type",'application/json;charset=utf-8')
|
||||
request.get_method = lambda: method
|
||||
try:
|
||||
connection = opener.open(request)
|
||||
print(connection.read())
|
||||
except urllib2.HTTPError, error:
|
||||
print(error)
|
||||
|
||||
@classmethod
|
||||
def send_dingtalk(cls, payload):
|
||||
event = payload.get('event')
|
||||
users = event.get("notify_users_obj")
|
||||
|
||||
rule_name = event.get("rule_name")
|
||||
event_state = "Triggered"
|
||||
if event.get("is_recovered"):
|
||||
event_state = "Recovered"
|
||||
|
||||
tokens = {}
|
||||
phones = {}
|
||||
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
|
||||
contacts = u.get("contacts")
|
||||
if contacts.get("dingtalk_robot_token", ""):
|
||||
tokens[contacts.get("dingtalk_robot_token", "")] = 1
|
||||
|
||||
opener = urllib2.build_opener(urllib2.HTTPHandler())
|
||||
method = "POST"
|
||||
|
||||
for t in tokens:
|
||||
url = "https://oapi.dingtalk.com/robot/send?access_token={}".format(t)
|
||||
body = {
|
||||
"msgtype": "markdown",
|
||||
"markdown": {
|
||||
"title": "{} - {}".format(event_state, rule_name),
|
||||
"text": payload.get('tpls').get("dingtalk.tpl", "dingtalk.tpl not found") + ' '.join(["@"+i for i in phones.keys()])
|
||||
},
|
||||
"at": {
|
||||
"atMobiles": phones.keys(),
|
||||
"isAtAll": False
|
||||
}
|
||||
}
|
||||
request = urllib2.Request(url, data=json.dumps(body))
|
||||
request.add_header("Content-Type",'application/json;charset=utf-8')
|
||||
request.get_method = lambda: method
|
||||
try:
|
||||
connection = opener.open(request)
|
||||
print(connection.read())
|
||||
except urllib2.HTTPError, error:
|
||||
print(error)
|
||||
|
||||
@classmethod
|
||||
def send_feishu(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
|
||||
tokens = {}
|
||||
phones = {}
|
||||
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
|
||||
contacts = u.get("contacts")
|
||||
if contacts.get("feishu_robot_token", ""):
|
||||
tokens[contacts.get("feishu_robot_token", "")] = 1
|
||||
|
||||
opener = urllib2.build_opener(urllib2.HTTPHandler())
|
||||
method = "POST"
|
||||
|
||||
for t in tokens:
|
||||
url = "https://open.feishu.cn/open-apis/bot/v2/hook/{}".format(t)
|
||||
body = {
|
||||
"msg_type": "text",
|
||||
"content": {
|
||||
"text": payload.get('tpls').get("feishu.tpl", "feishu.tpl not found")
|
||||
},
|
||||
"at": {
|
||||
"atMobiles": phones.keys(),
|
||||
"isAtAll": False
|
||||
}
|
||||
}
|
||||
request = urllib2.Request(url, data=json.dumps(body))
|
||||
request.add_header("Content-Type",'application/json;charset=utf-8')
|
||||
request.get_method = lambda: method
|
||||
try:
|
||||
connection = opener.open(request)
|
||||
print(connection.read())
|
||||
except urllib2.HTTPError, error:
|
||||
print(error)
|
||||
|
||||
@classmethod
|
||||
def send_sms(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
phones = {}
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
if phones:
|
||||
print("send_sms not implemented, phones: {}".format(phones.keys()))
|
||||
|
||||
@classmethod
|
||||
def send_voice(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
phones = {}
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
if phones:
|
||||
print("send_voice not implemented, phones: {}".format(phones.keys()))
|
||||
|
||||
def main():
|
||||
payload = json.load(sys.stdin)
|
||||
with open(".payload", 'w') as f:
|
||||
f.write(json.dumps(payload, indent=4))
|
||||
for ch in payload.get('event').get('notify_channels'):
|
||||
send_func_name = "send_{}".format(notify_channel_funcs.get(ch.strip()))
|
||||
if not hasattr(Sender, send_func_name):
|
||||
print("function: {} not found", send_func_name)
|
||||
continue
|
||||
send_func = getattr(Sender, send_func_name)
|
||||
send_func(payload)
|
||||
|
||||
def hello():
|
||||
print("hello nightingale")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
main()
|
||||
elif sys.argv[1] == "hello":
|
||||
hello()
|
||||
else:
|
||||
print("I am confused")
|
||||
73
n9e/compose-mysql/etc-nightingale/script/notify.py
Executable file
73
n9e/compose-mysql/etc-nightingale/script/notify.py
Executable file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: UTF-8 -*-
|
||||
import sys
|
||||
import json
|
||||
|
||||
class Sender(object):
|
||||
@classmethod
|
||||
def send_email(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_wecom(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_dingtalk(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_feishu(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_mm(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_sms(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
phones = {}
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
if phones:
|
||||
print("send_sms not implemented, phones: {}".format(phones.keys()))
|
||||
|
||||
@classmethod
|
||||
def send_voice(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
phones = {}
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
if phones:
|
||||
print("send_voice not implemented, phones: {}".format(phones.keys()))
|
||||
|
||||
def main():
|
||||
payload = json.load(sys.stdin)
|
||||
with open(".payload", 'w') as f:
|
||||
f.write(json.dumps(payload, indent=4))
|
||||
for ch in payload.get('event').get('notify_channels'):
|
||||
send_func_name = "send_{}".format(ch.strip())
|
||||
if not hasattr(Sender, send_func_name):
|
||||
print("function: {} not found", send_func_name)
|
||||
continue
|
||||
send_func = getattr(Sender, send_func_name)
|
||||
send_func(payload)
|
||||
|
||||
def hello():
|
||||
print("hello nightingale")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
main()
|
||||
elif sys.argv[1] == "hello":
|
||||
hello()
|
||||
else:
|
||||
print("I am confused")
|
||||
92
n9e/compose-mysql/etc-nightingale/script/notify_feishu.py
Normal file
92
n9e/compose-mysql/etc-nightingale/script/notify_feishu.py
Normal file
@@ -0,0 +1,92 @@
|
||||
#!/usr/bin/env python
|
||||
# -*- coding: UTF-8 -*-
|
||||
import sys
|
||||
import json
|
||||
import requests
|
||||
|
||||
class Sender(object):
|
||||
@classmethod
|
||||
def send_email(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_wecom(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_dingtalk(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_ifeishu(cls, payload):
|
||||
users = payload.get('event').get("notify_users_obj")
|
||||
tokens = {}
|
||||
phones = {}
|
||||
|
||||
for u in users:
|
||||
if u.get("phone"):
|
||||
phones[u.get("phone")] = 1
|
||||
|
||||
contacts = u.get("contacts")
|
||||
if contacts.get("feishu_robot_token", ""):
|
||||
tokens[contacts.get("feishu_robot_token", "")] = 1
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json;charset=utf-8",
|
||||
"Host": "open.feishu.cn"
|
||||
}
|
||||
|
||||
for t in tokens:
|
||||
url = "https://open.feishu.cn/open-apis/bot/v2/hook/{}".format(t)
|
||||
body = {
|
||||
"msg_type": "text",
|
||||
"content": {
|
||||
"text": payload.get('tpls').get("feishu", "feishu not found")
|
||||
},
|
||||
"at": {
|
||||
"atMobiles": list(phones.keys()),
|
||||
"isAtAll": False
|
||||
}
|
||||
}
|
||||
|
||||
response = requests.post(url, headers=headers, data=json.dumps(body))
|
||||
print(f"notify_ifeishu: token={t} status_code={response.status_code} response_text={response.text}")
|
||||
|
||||
@classmethod
|
||||
def send_mm(cls, payload):
|
||||
# already done in go code
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_sms(cls, payload):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def send_voice(cls, payload):
|
||||
pass
|
||||
|
||||
def main():
|
||||
payload = json.load(sys.stdin)
|
||||
with open(".payload", 'w') as f:
|
||||
f.write(json.dumps(payload, indent=4))
|
||||
for ch in payload.get('event').get('notify_channels'):
|
||||
send_func_name = "send_{}".format(ch.strip())
|
||||
if not hasattr(Sender, send_func_name):
|
||||
print("function: {} not found", send_func_name)
|
||||
continue
|
||||
send_func = getattr(Sender, send_func_name)
|
||||
send_func(payload)
|
||||
|
||||
def hello():
|
||||
print("hello nightingale")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) == 1:
|
||||
main()
|
||||
elif sys.argv[1] == "hello":
|
||||
hello()
|
||||
else:
|
||||
print("I am confused")
|
||||
200
n9e/compose-mysql/etc-nightingale/script/rule_converter.py
Normal file
200
n9e/compose-mysql/etc-nightingale/script/rule_converter.py
Normal file
@@ -0,0 +1,200 @@
|
||||
import json
|
||||
import yaml
|
||||
|
||||
'''
|
||||
将promtheus/vmalert的rule转换为n9e中的rule
|
||||
支持k8s的rule configmap
|
||||
'''
|
||||
|
||||
rule_file = 'rules.yaml'
|
||||
|
||||
|
||||
def convert_interval(interval):
|
||||
if interval.endswith('s') or interval.endswith('S'):
|
||||
return int(interval[:-1])
|
||||
if interval.endswith('m') or interval.endswith('M'):
|
||||
return int(interval[:-1]) * 60
|
||||
if interval.endswith('h') or interval.endswith('H'):
|
||||
return int(interval[:-1]) * 60 * 60
|
||||
if interval.endswith('d') or interval.endswith('D'):
|
||||
return int(interval[:-1]) * 60 * 60 * 24
|
||||
return int(interval)
|
||||
|
||||
|
||||
def convert_alert(rule, interval):
|
||||
name = rule['alert']
|
||||
prom_ql = rule['expr']
|
||||
if 'for' in rule:
|
||||
prom_for_duration = convert_interval(rule['for'])
|
||||
else:
|
||||
prom_for_duration = 0
|
||||
|
||||
prom_eval_interval = convert_interval(interval)
|
||||
note = ''
|
||||
if 'annotations' in rule:
|
||||
for v in rule['annotations'].values():
|
||||
note = v
|
||||
break
|
||||
|
||||
annotations = {}
|
||||
if 'annotations' in rule:
|
||||
for k, v in rule['annotations'].items():
|
||||
annotations[k] = v
|
||||
|
||||
|
||||
append_tags = []
|
||||
severity = 2
|
||||
if 'labels' in rule:
|
||||
for k, v in rule['labels'].items():
|
||||
if k != 'severity':
|
||||
append_tags.append('{}={}'.format(k, v))
|
||||
continue
|
||||
if v == 'critical':
|
||||
severity = 1
|
||||
elif v == 'info':
|
||||
severity = 3
|
||||
# elif v == 'warning':
|
||||
# severity = 2
|
||||
|
||||
|
||||
n9e_alert_rule = {
|
||||
"name": name,
|
||||
"note": note,
|
||||
"severity": severity,
|
||||
"disabled": 0,
|
||||
"prom_for_duration": prom_for_duration,
|
||||
"prom_ql": prom_ql,
|
||||
"prom_eval_interval": prom_eval_interval,
|
||||
"enable_stime": "00:00",
|
||||
"enable_etime": "23:59",
|
||||
"enable_days_of_week": [
|
||||
"1",
|
||||
"2",
|
||||
"3",
|
||||
"4",
|
||||
"5",
|
||||
"6",
|
||||
"0"
|
||||
],
|
||||
"enable_in_bg": 0,
|
||||
"notify_recovered": 1,
|
||||
"notify_channels": [],
|
||||
"notify_repeat_step": 60,
|
||||
"recover_duration": 0,
|
||||
"callbacks": [],
|
||||
"runbook_url": "",
|
||||
"append_tags": append_tags,
|
||||
"annotations":annotations
|
||||
}
|
||||
return n9e_alert_rule
|
||||
|
||||
|
||||
def convert_record(rule, interval):
|
||||
name = rule['record']
|
||||
prom_ql = rule['expr']
|
||||
prom_eval_interval = convert_interval(interval)
|
||||
note = ''
|
||||
append_tags = []
|
||||
if 'labels' in rule:
|
||||
for k, v in rule['labels'].items():
|
||||
append_tags.append('{}={}'.format(k, v))
|
||||
|
||||
n9e_record_rule = {
|
||||
"name": name,
|
||||
"note": note,
|
||||
"disabled": 0,
|
||||
"prom_ql": prom_ql,
|
||||
"prom_eval_interval": prom_eval_interval,
|
||||
"append_tags": append_tags
|
||||
}
|
||||
return n9e_record_rule
|
||||
|
||||
|
||||
'''
|
||||
example of rule group file
|
||||
---
|
||||
groups:
|
||||
- name: example
|
||||
rules:
|
||||
- alert: HighRequestLatency
|
||||
expr: job:request_latency_seconds:mean5m{job="myjob"} > 0.5
|
||||
for: 10m
|
||||
labels:
|
||||
severity: page
|
||||
annotations:
|
||||
summary: High request latency
|
||||
'''
|
||||
def deal_group(group):
|
||||
"""
|
||||
parse single prometheus/vmalert rule group
|
||||
"""
|
||||
alert_rules = []
|
||||
record_rules = []
|
||||
|
||||
for rule_segment in group['groups']:
|
||||
if 'interval' in rule_segment:
|
||||
interval = rule_segment['interval']
|
||||
else:
|
||||
interval = '15s'
|
||||
for rule in rule_segment['rules']:
|
||||
if 'alert' in rule:
|
||||
alert_rules.append(convert_alert(rule, interval))
|
||||
else:
|
||||
record_rules.append(convert_record(rule, interval))
|
||||
|
||||
return alert_rules, record_rules
|
||||
|
||||
|
||||
'''
|
||||
example of k8s rule configmap
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: rulefiles-0
|
||||
data:
|
||||
etcdrules.yaml: |
|
||||
groups:
|
||||
- name: etcd
|
||||
rules:
|
||||
- alert: etcdInsufficientMembers
|
||||
annotations:
|
||||
message: 'etcd cluster "{{ $labels.job }}": insufficient members ({{ $value}}).'
|
||||
expr: sum(up{job=~".*etcd.*"} == bool 1) by (job) < ((count(up{job=~".*etcd.*"})
|
||||
by (job) + 1) / 2)
|
||||
for: 3m
|
||||
labels:
|
||||
severity: critical
|
||||
'''
|
||||
def deal_configmap(rule_configmap):
|
||||
"""
|
||||
parse rule configmap from k8s
|
||||
"""
|
||||
all_record_rules = []
|
||||
all_alert_rules = []
|
||||
for _, rule_group_str in rule_configmap['data'].items():
|
||||
rule_group = yaml.load(rule_group_str, Loader=yaml.FullLoader)
|
||||
alert_rules, record_rules = deal_group(rule_group)
|
||||
all_alert_rules.extend(alert_rules)
|
||||
all_record_rules.extend(record_rules)
|
||||
|
||||
return all_alert_rules, all_record_rules
|
||||
|
||||
|
||||
def main():
|
||||
with open(rule_file, 'r') as f:
|
||||
rule_config = yaml.load(f, Loader=yaml.FullLoader)
|
||||
|
||||
# 如果文件是k8s中的configmap,使用下面的方法
|
||||
# alert_rules, record_rules = deal_configmap(rule_config)
|
||||
alert_rules, record_rules = deal_group(rule_config)
|
||||
|
||||
with open("alert-rules.json", 'w') as fw:
|
||||
json.dump(alert_rules, fw, indent=2, ensure_ascii=False)
|
||||
|
||||
with open("record-rules.json", 'w') as fw:
|
||||
json.dump(record_rules, fw, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user