mirror of https://gitee.com/anolis/sysom.git
626 lines
37 KiB
Python
Executable File
626 lines
37 KiB
Python
Executable File
"""
|
||
Time 2023/12/19
|
||
Author: chenshiyan
|
||
File ossre_post.py
|
||
Description:
|
||
"""
|
||
from typing import List
|
||
from .base import DiagnosisJobResult, DiagnosisPostProcessor, PostProcessResult
|
||
import json
|
||
import time
|
||
import traceback
|
||
from sysom_utils import SysomFramework
|
||
|
||
ERR_LEVEL = ["normal","warning","error","critical"]
|
||
ERR_LEVEL_CN = ["正常项","告警项","错误项","致命项"]
|
||
|
||
TOOLS_MSG = {
|
||
"memgraph":{"key":"sysom_diagnosis:memgraph","url":"/diagnose/memory/memgraph?","cn_name":"内存大盘","params":{"instance":""}},
|
||
"oomcheck":{"key":"sysom_diagnosis:oomcheck","url":"/diagnose/memory/oomcheck?","cn_name":"OOM诊断","params":{"instance":""}},
|
||
"podmem":{"key":"sysom_diagnosis:podmem","url":"/diagnose/memory/podmem?","cn_name":"内存异常诊断","params":{"instance":""}},
|
||
"loadtask":{"key":"sysom_diagnosis:laodtask","url":"/diagnose/cpu/podmem?","cn_name":"系统负载诊断","params":{"instance":""}}
|
||
}
|
||
|
||
def make_tool_option(tool,instance):
|
||
ret = {}
|
||
try:
|
||
if tool not in TOOLS_MSG:
|
||
return ret
|
||
tool_pms = ""
|
||
for x in TOOLS_MSG[tool]["params"]:
|
||
if len(tool_pms) > 0:
|
||
tool_pms = "%s&"%tool_pms
|
||
if x == "instance":
|
||
tool_pms = "%s%s=%s"%(tool_pms,x,instance)
|
||
else:
|
||
tool_pms = "%s%s=%s"%(tool_pms,x,TOOLS_MSG[tool]["params"][x])
|
||
|
||
ret = {"key":TOOLS_MSG[tool]["key"],"label":TOOLS_MSG[tool]["cn_name"],"type": "LINK","url":"%s%s"%(TOOLS_MSG[tool]["url"],tool_pms)}
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
return ret
|
||
|
||
def make_ignore_option(instance_ip,key_tmp):
|
||
ret = {}
|
||
try:
|
||
ret = {"key":"ossre_ignore-%s"%key_tmp,"label":"屏蔽检测","type": "INVOKE_DIAGNOSIS_HOOK","params":{"type":"ignore","instance":instance_ip,"key":key_tmp}}
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
return ret
|
||
|
||
def make_release_ignore_option(instance_ip,key_tmp):
|
||
ret = {}
|
||
try:
|
||
ret = {"key":"ossre_release_ignore-%s"%key_tmp,"label":"解除屏蔽","type": "INVOKE_DIAGNOSIS_HOOK","params":{"type":"release_ignore","instance":instance_ip,"key":key_tmp}}
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
return ret
|
||
|
||
def flush_data(instance_ip,origin_data):
|
||
try:
|
||
ossre_cache = SysomFramework.gcache("ossre")
|
||
total_abnormal_count = 0
|
||
level_num = {"error":0,"warning":0,"critical":0}
|
||
for i in range(len(origin_data)):
|
||
if True:
|
||
abnormal_count = 0
|
||
inspect_result = "无异常"
|
||
status = "normal"
|
||
if "children" in origin_data[i]:
|
||
if len(origin_data[i]["children"]) > 0:
|
||
for k in range(len(origin_data[i]["children"])):
|
||
abnormal_count_sub1 = 0
|
||
inspect_result_sub1 = ""
|
||
status_sub1 = "normal"
|
||
all_sub1_ignore = 0
|
||
if ossre_cache.load("%s-%s"%(instance_ip,origin_data[i]["children"][k]["key"])) == 0:
|
||
all_sub1_ignore = 1
|
||
release_ignore_ret = make_release_ignore_option(instance_ip,origin_data[i]["children"][k]["key"])
|
||
for x in range(len(origin_data[i]["children"][k]["options"])):
|
||
if origin_data[i]["children"][k]["options"][x]["label"] == "屏蔽检测":
|
||
del origin_data[i]["children"][k]["options"][x]
|
||
break
|
||
origin_data[i]["children"][k]["options"].append(release_ignore_ret)
|
||
|
||
if "children" in origin_data[i]["children"][k]:
|
||
if len(origin_data[i]["children"][k]["children"]) > 0:
|
||
for x in range(len(origin_data[i]["children"][k]["children"])):
|
||
if ossre_cache.load("%s-%s"%(instance_ip,origin_data[i]["children"][k]["children"][x]["key"])) == 0 or all_sub1_ignore == 1:
|
||
origin_data[i]["children"][k]["children"][x]["status"] = "normal"
|
||
origin_data[i]["children"][k]["children"][x]["abnormal_count"] = 0
|
||
release_ignore_ret = make_release_ignore_option(instance_ip,origin_data[i]["children"][k]["children"][x]["key"])
|
||
origin_data[i]["children"][k]["children"][x]["options"].append(release_ignore_ret)
|
||
for y in range(len(origin_data[i]["children"][k]["children"][x]["options"])):
|
||
if origin_data[i]["children"][k]["children"][x]["options"][y]["label"] == "屏蔽检测":
|
||
del origin_data[i]["children"][k]["children"][x]["options"][y]
|
||
break
|
||
if all_sub1_ignore == 1:
|
||
for y in range(len(origin_data[i]["children"][k]["children"][x]["options"])):
|
||
if origin_data[i]["children"][k]["children"][x]["options"][y]["label"] == "解除屏蔽":
|
||
del origin_data[i]["children"][k]["children"][x]["options"][y]
|
||
break
|
||
else:
|
||
if ERR_LEVEL.index(origin_data[i]["children"][k]["children"][x]["status"]) > ERR_LEVEL.index(status_sub1):
|
||
status_sub1 = origin_data[i]["children"][k]["children"][x]["status"]
|
||
if ERR_LEVEL.index(origin_data[i]["children"][k]["children"][x]["status"]) > ERR_LEVEL.index(status):
|
||
status = origin_data[i]["children"][k]["children"][x]["status"]
|
||
abnormal_count_sub1 += origin_data[i]["children"][k]["children"][x]["abnormal_count"]
|
||
level_num[origin_data[i]["children"][k]["children"][x]["status"]] += 1
|
||
if len(inspect_result_sub1) == 0:
|
||
inspect_result_sub1 = origin_data[i]["children"][k]["children"][x]["inspect_result"]
|
||
else:
|
||
inspect_result_sub1 = "%s\n%s"%(inspect_result_sub1,origin_data[i]["children"][k]["children"][x]["inspect_result"])
|
||
abnormal_count += abnormal_count_sub1
|
||
if abnormal_count > 0:
|
||
inspect_result = "存在异常"
|
||
origin_data[i]["children"][k]["status"] = status_sub1
|
||
origin_data[i]["children"][k]["abnormal_count"] = abnormal_count_sub1
|
||
origin_data[i]["children"][k]["inspect_result"] = inspect_result_sub1
|
||
origin_data[i]["status"] = status
|
||
origin_data[i]["abnormal_count"] = abnormal_count
|
||
origin_data[i]["inspect_result"] = inspect_result
|
||
total_abnormal_count += abnormal_count
|
||
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
return total_abnormal_count,level_num,origin_data
|
||
|
||
def flush_checkItems(origin_checkItems,errnum,levelnum):
|
||
try:
|
||
ori_errnum = 0
|
||
ori_warnnum = 0
|
||
ori_critnum = 0
|
||
ori_nornum = 0
|
||
for i in range(len(origin_checkItems)):
|
||
if origin_checkItems[i]["key"] == "告警项":
|
||
ori_warnnum = origin_checkItems[i]["value"]
|
||
origin_checkItems[i]["value"] = levelnum["warning"]
|
||
if origin_checkItems[i]["key"] == "错误项":
|
||
ori_errnum = origin_checkItems[i]["value"]
|
||
origin_checkItems[i]["value"] = levelnum["error"]
|
||
if origin_checkItems[i]["key"] == "严重异常项":
|
||
ori_critnum = origin_checkItems[i]["value"]
|
||
origin_checkItems[i]["value"] = levelnum["critical"]
|
||
|
||
for i in range(len(origin_checkItems)):
|
||
if origin_checkItems[i]["key"] == "正常项":
|
||
ori_nornum = origin_checkItems[i]["value"]
|
||
origin_checkItems[i]["value"] = ori_nornum + ori_errnum + ori_warnnum + ori_critnum - levelnum["warning"] - levelnum["error"] - levelnum["critical"]
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
return ori_nornum + ori_errnum + ori_warnnum + ori_critnum, origin_checkItems
|
||
|
||
def flush_hostInfo(origin_hostInfo,errnum,levelnum,total_count):
|
||
try:
|
||
line_list = origin_hostInfo.replace("\n\n","\n").splitlines()
|
||
del line_list[-1]
|
||
#line_list.append("共检测%s项，疑似有%s项风险"%(total_count,errnum))
|
||
line_list.append("共检测%s项，告警项%s个，错误项%s个，严重异常项%s个"%(total_count,levelnum["warning"],levelnum["error"],levelnum["critical"]))
|
||
origin_hostInfo = "\n\n".join(line_list)
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
return origin_hostInfo
|
||
|
||
def sysak_to_sysom_info(sysakdict):
|
||
retdict = {}
|
||
retdict["success"] = True
|
||
retdict["num_total"] = 0
|
||
retdict["num_error"] = 0
|
||
retdict["level_num"] = {"error":0,"warning":0,"critical":0}
|
||
retdict["critical_level_num"] = 0
|
||
retdict["indicator"] = {"sched":{},"mem":{},"net":{},"io":{},"misc":{},"errnum":0,"status":"normal","tool":[]}
|
||
retdict["config"] = {"sched":{},"mem":{},"net":{},"io":{},"misc":{},"errnum":0,"status":"normal","tool":[]}
|
||
retdict["issue"] = {"critical":{},"error":{},"warning":{},"errnum":0,"status":"normal","tool":[]}
|
||
retdict["log"] = {"critical":{},"error":{},"warning":{},"errnum":0,"status":"normal","tool":[]}
|
||
retdict["hw"] = {"critical":{},"error":{},"warning":{},"errnum":0,"status":"normal","tool":[]}
|
||
retdict["sysinfo"] = {"kernel_version":"","cpu_info":"","mem_info":""}
|
||
try:
|
||
if sysakdict["success"] != "true":
|
||
retdict["success"] = False
|
||
else:
|
||
retdict["sysinfo"]["kernel_version"] = sysakdict["fields"]["SYSINFO"]["kernel_version"]
|
||
retdict["sysinfo"]["cpu_info"] = sysakdict["fields"]["SYSINFO"]["cpuinfo"]
|
||
retdict["sysinfo"]["mem_info"] = sysakdict["fields"]["SYSINFO"]["meminfo"]
|
||
for i in sysakdict["items"]["config"]:
|
||
for j in sysakdict["items"]["config"][i]:
|
||
retdict["num_total"] += 1
|
||
if sysakdict["items"]["config"][i][j]["status"] == "warning":
|
||
retdict["num_error"] += 1
|
||
level_tmp = "warning"
|
||
if sysakdict["items"]["config"][i][j]["level"] in ERR_LEVEL:
|
||
level_tmp = sysakdict["items"]["config"][i][j]["level"]
|
||
retdict["level_num"][level_tmp] += 1
|
||
if len(retdict["config"][i]) == 0:
|
||
retdict["config"][i] = {"errnum":0,"status":"normal","tool":[],"summary":"","sub_items":{}}
|
||
if sysakdict["items"]["config"][i][j]["level"] in ERR_LEVEL:
|
||
if ERR_LEVEL.index(sysakdict["items"]["config"][i][j]["level"]) > ERR_LEVEL.index(retdict["config"][i]["status"]):
|
||
retdict["config"][i]["status"] = sysakdict["items"]["config"][i][j]["level"]
|
||
if ERR_LEVEL.index(sysakdict["items"]["config"][i][j]["level"]) > ERR_LEVEL.index(retdict["config"]["status"]):
|
||
retdict["config"]["status"] = sysakdict["items"]["config"][i][j]["level"]
|
||
retdict["config"][i]["errnum"] += 1
|
||
retdict["config"]["errnum"] += 1
|
||
if len(retdict["config"][i]["summary"]) > 0:
|
||
retdict["config"][i]["summary"] = "%s\n%s"%(retdict["config"][i]["summary"],sysakdict["items"]["config"][i][j]["suggestion"])
|
||
else:
|
||
retdict["config"][i]["summary"] = sysakdict["items"]["config"][i][j]["suggestion"]
|
||
retdict["config"][i]["sub_items"][j] = {"status":"normal","summary":"-","tool":""}
|
||
retdict["config"][i]["sub_items"][j]["status"] = sysakdict["items"]["config"][i][j]["level"]
|
||
retdict["config"][i]["sub_items"][j]["summary"] = sysakdict["items"]["config"][i][j]["suggestion"]
|
||
if len(sysakdict["items"]["config"][i][j]["tool"]) > 0:
|
||
retdict["config"][i]["sub_items"][j]["tool"] = sysakdict["items"]["config"][i][j]["tool"]
|
||
if sysakdict["items"]["config"][i][j]["tool"] not in retdict["config"][i]["tool"]:
|
||
retdict["config"][i]["tool"].append(sysakdict["items"]["config"][i][j]["tool"])
|
||
if sysakdict["items"]["config"][i][j]["tool"] not in retdict["config"]["tool"]:
|
||
retdict["config"]["tool"].append(sysakdict["items"]["config"][i][j]["tool"])
|
||
|
||
for i in sysakdict["items"]["indicator"]:
|
||
for j in sysakdict["items"]["indicator"][i]:
|
||
retdict["num_total"] += 1
|
||
if sysakdict["items"]["indicator"][i][j]["status"] == "warning":
|
||
retdict["num_error"] += 1
|
||
|
||
level_tmp = "warning"
|
||
if sysakdict["items"]["indicator"][i][j]["level"] in ERR_LEVEL:
|
||
level_tmp = sysakdict["items"]["indicator"][i][j]["level"]
|
||
retdict["level_num"][level_tmp] += 1
|
||
|
||
if len(retdict["indicator"][i]) == 0:
|
||
retdict["indicator"][i] = {"errnum":0,"status":"normal","tool":[],"summary":"","sub_items":{}}
|
||
if sysakdict["items"]["indicator"][i][j]["level"] in ERR_LEVEL:
|
||
if ERR_LEVEL.index(sysakdict["items"]["indicator"][i][j]["level"]) > ERR_LEVEL.index(retdict["indicator"][i]["status"]):
|
||
retdict["indicator"][i]["status"] = sysakdict["items"]["indicator"][i][j]["level"]
|
||
if ERR_LEVEL.index(sysakdict["items"]["indicator"][i][j]["level"]) > ERR_LEVEL.index(retdict["indicator"]["status"]):
|
||
retdict["indicator"]["status"] = sysakdict["items"]["indicator"][i][j]["level"]
|
||
retdict["indicator"][i]["errnum"] += 1
|
||
retdict["indicator"]["errnum"] += 1
|
||
|
||
retdict["indicator"][i]["sub_items"][j] = {"status":"normal","summary":"-","tool":""}
|
||
retdict["indicator"][i]["sub_items"][j]["status"] = sysakdict["items"]["indicator"][i][j]["level"]
|
||
retdict["indicator"][i]["sub_items"][j]["summary"] = sysakdict["items"]["indicator"][i][j]["suggestion"]
|
||
|
||
if len(sysakdict["items"]["indicator"][i][j]["summary"]) > 0:
|
||
if len(retdict["indicator"][i]["summary"]) > 0:
|
||
retdict["indicator"][i]["summary"] = "%s\n%s"%(retdict["indicator"][i]["summary"],sysakdict["items"]["indicator"][i][j]["summary"])
|
||
else:
|
||
retdict["indicator"][i]["summary"] = sysakdict["items"]["indicator"][i][j]["summary"]
|
||
retdict["indicator"][i]["sub_items"][j]["summary"] = sysakdict["items"]["indicator"][i][j]["summary"]
|
||
else:
|
||
if len(retdict["indicator"][i]["summary"]) > 0:
|
||
retdict["indicator"][i]["summary"] = "%s\n%s"%(retdict["indicator"][i]["summary"],sysakdict["items"]["indicator"][i][j]["suggestion"])
|
||
else:
|
||
retdict["indicator"][i]["summary"] = sysakdict["items"]["indicator"][i][j]["suggestion"]
|
||
retdict["indicator"][i]["sub_items"][j]["summary"] = sysakdict["items"]["indicator"][i][j]["suggestion"]
|
||
|
||
if len(sysakdict["items"]["indicator"][i][j]["tool"]) > 0:
|
||
retdict["indicator"][i]["sub_items"][j]["tool"] = sysakdict["items"]["indicator"][i][j]["tool"]
|
||
if sysakdict["items"]["indicator"][i][j]["tool"] not in retdict["indicator"][i]["tool"]:
|
||
retdict["indicator"][i]["tool"].append(sysakdict["items"]["indicator"][i][j]["tool"])
|
||
if sysakdict["items"]["indicator"][i][j]["tool"] not in retdict["indicator"]["tool"]:
|
||
retdict["indicator"]["tool"].append(sysakdict["items"]["indicator"][i][j]["tool"])
|
||
|
||
retdict["num_total"] += len(sysakdict["items"]["hw"])
|
||
for i in sysakdict["items"]["hw"]:
|
||
level = sysakdict["items"]["hw"][i]["level"]
|
||
if sysakdict["items"]["hw"][i]["status"] == "warning":
|
||
retdict["num_error"] += 1
|
||
if level not in ERR_LEVEL:
|
||
level = "warning"
|
||
retdict["level_num"][level] += 1
|
||
|
||
if len(retdict["hw"][level]) == 0:
|
||
retdict["hw"][level] = {"errnum":0,"summary":"","sub_items":{}}
|
||
retdict["hw"][level]["sub_items"][i] = sysakdict["items"]["hw"][i]
|
||
retdict["hw"][level]["sub_items"][i]["status"] = level
|
||
retdict["hw"][level]["errnum"] += 1
|
||
if len(retdict["hw"][level]["summary"]) > 0:
|
||
retdict["hw"][level]["summary"] = "%s\n%s"%(retdict["hw"][level]["summary"], sysakdict["items"]["hw"][i]["summary"])
|
||
else:
|
||
retdict["hw"][level]["summary"] = sysakdict["items"]["hw"][i]["summary"]
|
||
|
||
retdict["num_total"] += len(sysakdict["items"]["log"])
|
||
for i in sysakdict["items"]["log"]:
|
||
level = sysakdict["items"]["log"][i]["level"]
|
||
if sysakdict["items"]["log"][i]["status"] == "warning":
|
||
retdict["num_error"] += 1
|
||
if level not in ERR_LEVEL:
|
||
level = "warning"
|
||
retdict["level_num"][level] += 1
|
||
|
||
if len(retdict["log"][level]) == 0:
|
||
retdict["log"][level] = {"errnum":0,"summary":"","sub_items":{}}
|
||
retdict["log"][level]["sub_items"][i] = sysakdict["items"]["log"][i]
|
||
retdict["log"][level]["sub_items"][i]["status"] = level
|
||
retdict["log"][level]["errnum"] += 1
|
||
if len(retdict["log"][level]["summary"]) > 0:
|
||
retdict["log"][level]["summary"] = "%s\n%s"%(retdict["log"][level]["summary"], sysakdict["items"]["log"][i]["summary"])
|
||
else:
|
||
retdict["log"][level]["summary"] = sysakdict["items"]["log"][i]["summary"]
|
||
|
||
retdict["num_total"] += len(sysakdict["items"]["issue"])
|
||
for i in sysakdict["items"]["issue"]:
|
||
level = sysakdict["items"]["issue"][i]["level"]
|
||
if sysakdict["items"]["issue"][i]["return"] == True:
|
||
retdict["num_error"] += 1
|
||
if level not in ERR_LEVEL:
|
||
level = "warning"
|
||
retdict["level_num"][level] += 1
|
||
|
||
if len(retdict["issue"][level]) == 0:
|
||
retdict["issue"][level] = {"errnum":0,"summary":"","sub_items":{}}
|
||
retdict["issue"][level]["sub_items"][i] = {}
|
||
retdict["issue"][level]["sub_items"][i]["summary"] = sysakdict["items"]["issue"][i]["solution"]
|
||
retdict["issue"][level]["sub_items"][i]["status"] = level
|
||
|
||
retdict["issue"][level]["errnum"] += 1
|
||
if len(retdict["issue"][level]["summary"]) > 0:
|
||
retdict["issue"][level]["summary"] = "%s\n%s"%(retdict["issue"][level]["summary"], sysakdict["items"]["issue"][i]["solution"])
|
||
else:
|
||
retdict["issue"][level]["summary"] = sysakdict["items"]["issue"][i]["solution"]
|
||
|
||
except:
|
||
traceback.print_exc()
|
||
retdict["success"] = False
|
||
pass
|
||
return retdict
|
||
|
||
class PostProcessor(DiagnosisPostProcessor):
|
||
def parse_diagnosis_result(self, results: List[DiagnosisJobResult]) -> PostProcessResult:
|
||
postprocess_result = PostProcessResult(
|
||
code=0,
|
||
err_msg="",
|
||
result={}
|
||
)
|
||
|
||
ossre_ret = results[0].stdout
|
||
instance_ip = results[0].job.instance
|
||
print (ossre_ret)
|
||
datas = []
|
||
piedatas = []
|
||
mddata = ""
|
||
try:
|
||
results = json.loads(ossre_ret)
|
||
retdict = sysak_to_sysom_info(results["ossre"])
|
||
num = 0
|
||
if retdict["success"] == True:
|
||
config = {"key":"config","inspect_items":"配置检查","status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
config["status"] = retdict["config"]["status"]
|
||
config["abnormal_count"] = retdict["config"]["errnum"]
|
||
config["inspect_result"] = "无异常"
|
||
if config["abnormal_count"] > 0:
|
||
config["inspect_result"] = "存在异常"
|
||
|
||
config["options"] = ""
|
||
num += 1
|
||
for i in retdict["config"]:
|
||
if i != "io" and i != "mem" and i != "net" and i != "misc" and i != "sched":
|
||
continue
|
||
|
||
key_tmp = "config-%s"%i
|
||
tmpd = {"key":key_tmp,"inspect_items":i,"status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
|
||
if len(retdict["config"][i]) == 0:
|
||
config["children"].append(tmpd)
|
||
num += 1
|
||
continue
|
||
|
||
if retdict["config"][i]["errnum"] > 0:
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
tmpd["options"].append(ignore_ret)
|
||
|
||
for j in retdict["config"][i]["sub_items"]:
|
||
options_sub = []
|
||
key_tmp = "config-%s-%s"%(i,j)
|
||
if retdict["config"][i]["sub_items"][j]["status"] != "normal":
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
options_sub.append(ignore_ret)
|
||
|
||
if len(retdict["config"][i]["sub_items"][j]["tool"]) > 0:
|
||
tp_tmp = make_tool_option(retdict["config"][i]["sub_items"][j]["tool"],instance_ip)
|
||
if len(tp_tmp) > 0:
|
||
options_sub.append(tp_tmp)
|
||
tmpd["children"].append({"key":key_tmp,"inspect_items":j,"status":retdict["config"][i]["sub_items"][j]["status"],"abnormal_count":1,"inspect_result":retdict["config"][i]["sub_items"][j]["summary"],"options":options_sub})
|
||
|
||
tmpd["status"] = retdict["config"][i]["status"]
|
||
tmpd["abnormal_count"] = retdict["config"][i]["errnum"]
|
||
tmpd["inspect_result"] = retdict["config"][i]["summary"]
|
||
for t in retdict["config"][i]["tool"]:
|
||
tp_tmp = make_tool_option(t,instance_ip)
|
||
if len(tp_tmp) > 0:
|
||
tmpd["options"].append(tp_tmp)
|
||
config["children"].append(tmpd)
|
||
num += 1
|
||
datas.append(config)
|
||
|
||
indicator = {"key":"indicator","inspect_items":"指标检查","status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
indicator["status"] = retdict["indicator"]["status"]
|
||
indicator["abnormal_count"] = retdict["indicator"]["errnum"]
|
||
indicator["inspect_result"] = "无异常"
|
||
if indicator["abnormal_count"] > 0:
|
||
indicator["inspect_result"] = "存在异常"
|
||
indicator["options"] = ""
|
||
num += 1
|
||
|
||
for i in retdict["indicator"]:
|
||
if i != "io" and i != "mem" and i != "net" and i != "misc" and i != "sched":
|
||
continue
|
||
key_tmp = "indicator-%s"%i
|
||
tmpd = {"key":key_tmp,"inspect_items":i,"status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
if len(retdict["indicator"][i]) == 0:
|
||
indicator["children"].append(tmpd)
|
||
num += 1
|
||
continue
|
||
|
||
if retdict["indicator"][i]["errnum"] > 0:
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
tmpd["options"].append(ignore_ret)
|
||
|
||
for j in retdict["indicator"][i]["sub_items"]:
|
||
options_sub = []
|
||
key_tmp = "indicator-%s-%s"%(i,j)
|
||
if retdict["indicator"][i]["sub_items"][j]["status"] != "normal":
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
options_sub.append(ignore_ret)
|
||
|
||
if len(retdict["indicator"][i]["sub_items"][j]["tool"]) > 0:
|
||
tp_tmp = make_tool_option(retdict["indicator"][i]["sub_items"][j]["tool"],instance_ip)
|
||
if len(tp_tmp) > 0:
|
||
options_sub.append(tp_tmp)
|
||
tmpd["children"].append({"key":"indicator-%s-%s"%(i,j),"inspect_items":j,"status":retdict["indicator"][i]["sub_items"][j]["status"],"abnormal_count":1,"inspect_result":retdict["indicator"][i]["sub_items"][j]["summary"],"options":options_sub})
|
||
|
||
tmpd["status"] = retdict["indicator"][i]["status"]
|
||
tmpd["abnormal_count"] = retdict["indicator"][i]["errnum"]
|
||
tmpd["inspect_result"] = retdict["indicator"][i]["summary"]
|
||
tools = ""
|
||
for t in retdict["indicator"][i]["tool"]:
|
||
tp_tmp = make_tool_option(t,instance_ip)
|
||
if len(tp_tmp) > 0:
|
||
tmpd["options"].append(tp_tmp)
|
||
indicator["children"].append(tmpd)
|
||
num += 1
|
||
datas.append(indicator)
|
||
|
||
issue = {"key":"issue","inspect_items":"已知问题检查","status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
issue_num = 0
|
||
if len(retdict["issue"]["warning"]) > 0:
|
||
issue["status"] = "warning"
|
||
issue_num += retdict["issue"]["warning"]["errnum"]
|
||
if len(retdict["issue"]["error"]) > 0:
|
||
issue["status"] = "error"
|
||
issue_num += retdict["issue"]["error"]["errnum"]
|
||
if len(retdict["issue"]["critical"]) > 0:
|
||
issue["status"] = "critical"
|
||
issue_num += retdict["issue"]["critical"]["errnum"]
|
||
|
||
issue["abnormal_count"] = issue_num
|
||
issue["inspect_result"] = "无异常"
|
||
if issue["abnormal_count"] > 0:
|
||
issue["inspect_result"] = "存在异常"
|
||
issue["options"] = []
|
||
num += 1
|
||
|
||
for i in retdict["issue"]:
|
||
if i != "critical" and i != "error" and i != "warning":
|
||
continue
|
||
key_tmp = "issue-%s"%i
|
||
tmpd = {"key":key_tmp,"inspect_items":ERR_LEVEL_CN[ERR_LEVEL.index(i)],"status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
if len(retdict["issue"][i]) > 0:
|
||
if retdict["issue"][i]["errnum"] > 0:
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
tmpd["options"].append(ignore_ret)
|
||
|
||
for j in retdict["issue"][i]["sub_items"]:
|
||
options_sub = []
|
||
key_tmp = "issue-%s-%s"%(i,j)
|
||
if retdict["issue"][i]["sub_items"][j]["status"] != "normal":
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
options_sub.append(ignore_ret)
|
||
tmpd["children"].append({"key":"issue-%s-%s"%(i,j),"inspect_items":j,"status":retdict["issue"][i]["sub_items"][j]["status"],"abnormal_count":1,"inspect_result":retdict["issue"][i]["sub_items"][j]["summary"],"options":options_sub})
|
||
tmpd["status"] = i
|
||
tmpd["abnormal_count"] = retdict["issue"][i]["errnum"]
|
||
tmpd["inspect_result"] = retdict["issue"][i]["summary"]
|
||
issue["children"].append(tmpd)
|
||
num += 1
|
||
datas.append(issue)
|
||
|
||
log = {"key":"log","inspect_items":"日志检查","status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
log_num = 0
|
||
if len(retdict["log"]["warning"]) > 0:
|
||
log["status"] = "warning"
|
||
log_num += retdict["log"]["warning"]["errnum"]
|
||
if len(retdict["log"]["error"]) > 0:
|
||
log["status"] = "error"
|
||
log_num += retdict["log"]["error"]["errnum"]
|
||
if len(retdict["log"]["critical"]) > 0:
|
||
log["status"] = "critical"
|
||
log_num += retdict["log"]["critical"]["errnum"]
|
||
|
||
log["abnormal_count"] = log_num
|
||
log["inspect_result"] = "无异常"
|
||
if log["abnormal_count"] > 0:
|
||
log["inspect_result"] = "存在异常"
|
||
log["options"] = []
|
||
num += 1
|
||
|
||
for i in retdict["log"]:
|
||
if i != "critical" and i != "error" and i != "warning":
|
||
continue
|
||
key_tmp = "log-%s"%i
|
||
tmpd = {"key":key_tmp,"inspect_items":ERR_LEVEL_CN[ERR_LEVEL.index(i)],"status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
if len(retdict["log"][i]) > 0:
|
||
if retdict["log"][i]["errnum"] > 0:
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
tmpd["options"].append(ignore_ret)
|
||
|
||
for j in retdict["log"][i]["sub_items"]:
|
||
options_sub = []
|
||
key_tmp = "log-%s-%s"%(i,j)
|
||
if retdict["log"][i]["sub_items"][j]["status"] != "normal":
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
options_sub.append(ignore_ret)
|
||
|
||
if len(retdict["log"][i]["sub_items"][j]["tool"]) > 0:
|
||
tp_tmp = make_tool_option(retdict["log"][i]["sub_items"][j]["tool"],instance_ip)
|
||
if len(tp_tmp) > 0:
|
||
options_sub.append(tp_tmp)
|
||
tmpd["children"].append({"key":"log-%s-%s"%(i,j),"inspect_items":j,"status":retdict["log"][i]["sub_items"][j]["status"],"abnormal_count":1,"inspect_result":retdict["log"][i]["sub_items"][j]["summary"],"options":options_sub})
|
||
tmpd["status"] = i
|
||
tmpd["abnormal_count"] = retdict["log"][i]["errnum"]
|
||
tmpd["inspect_result"] = retdict["log"][i]["summary"]
|
||
|
||
log["children"].append(tmpd)
|
||
num += 1
|
||
datas.append(log)
|
||
|
||
hw = {"key":"hw","inspect_items":"硬件检查","status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
hw_num = 0
|
||
if len(retdict["hw"]["warning"]) > 0:
|
||
hw["status"] = "warning"
|
||
hw_num += retdict["hw"]["warning"]["errnum"]
|
||
if len(retdict["hw"]["error"]) > 0:
|
||
hw["status"] = "error"
|
||
hw_num += retdict["hw"]["error"]["errnum"]
|
||
if len(retdict["hw"]["critical"]) > 0:
|
||
hw["status"] = "critical"
|
||
hw_num += retdict["hw"]["critical"]["errnum"]
|
||
|
||
hw["abnormal_count"] = hw_num
|
||
hw["inspect_result"] = "无异常"
|
||
if hw["abnormal_count"] > 0:
|
||
hw["inspect_result"] = "存在异常"
|
||
hw["options"] = []
|
||
num += 1
|
||
|
||
for i in retdict["hw"]:
|
||
if i != "critical" and i != "error" and i != "warning":
|
||
continue
|
||
key_tmp = "hw-%s"%i
|
||
tmpd = {"key":key_tmp,"inspect_items":ERR_LEVEL_CN[ERR_LEVEL.index(i)],"status":"normal","abnormal_count":0,"inspect_result":"","options":[],"children":[]}
|
||
|
||
if len(retdict["hw"][i]) > 0:
|
||
if retdict["hw"][i]["errnum"] > 0:
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
tmpd["options"].append(ignore_ret)
|
||
|
||
for j in retdict["hw"][i]["sub_items"]:
|
||
options_sub = []
|
||
key_tmp = "hw-%s-%s"%(i,j)
|
||
if retdict["hw"][i]["sub_items"][j]["status"] != "normal":
|
||
ignore_ret = make_ignore_option(instance_ip,key_tmp)
|
||
if len(ignore_ret) > 0:
|
||
options_sub.append(ignore_ret)
|
||
|
||
tmpd["children"].append({"key":"hw-%s-%s"%(i,j),"inspect_items":j,"status":retdict["hw"][i]["sub_items"][j]["status"],"abnormal_count":1,"inspect_result":retdict["hw"][i]["sub_items"][j]["summary"],"options":options_sub})
|
||
|
||
tmpd["status"] = i
|
||
tmpd["abnormal_count"] = retdict["hw"][i]["errnum"]
|
||
tmpd["inspect_result"] = retdict["hw"][i]["summary"]
|
||
hw["children"].append(tmpd)
|
||
num += 1
|
||
datas.append(hw)
|
||
piedatas.append({"key":"告警项","value":retdict["level_num"]["warning"]})
|
||
piedatas.append({"key":"错误项","value":retdict["level_num"]["error"]})
|
||
piedatas.append({"key":"严重异常项","value":retdict["level_num"]["critical"]})
|
||
piedatas.append({"key":"正常项","value":retdict["num_total"]-retdict["num_error"]})
|
||
mddata = "%s### 内核版本\n\n%s\n\n"%(mddata,retdict["sysinfo"]["kernel_version"])
|
||
mddata = "%s### CPU信息\n\nNuma Node：%s CPU数量：%s CPU Model：%s\n\n"%(mddata,retdict["sysinfo"]["cpu_info"]["numa"],retdict["sysinfo"]["cpu_info"]["cpunum"],retdict["sysinfo"]["cpu_info"]["model name"])
|
||
mddata = "%s### 检测结果\n\n共检测%s项，告警项%s个，错误项%s个，严重异常项%s个\n\n"%(mddata,retdict["num_total"],retdict["level_num"]["warning"],retdict["level_num"]["error"],retdict["level_num"]["critical"])
|
||
else:
|
||
datas.append({"key":0,"ossre":"Fail","result":"sysak_to_sysom_info fail!"})
|
||
|
||
OssreResult = {"data": datas,"checkItems":piedatas,"hostInfo":mddata}
|
||
ossre_cache = SysomFramework.gcache("ossre")
|
||
ossre_cache.store("%s-OssreResult"%instance_ip,OssreResult)
|
||
|
||
total_abnormal_count,level_num,new_data = flush_data(instance_ip,datas)
|
||
total_count,new_checkItems = flush_checkItems(piedatas,total_abnormal_count,level_num)
|
||
new_hostInfo = flush_hostInfo(mddata,total_abnormal_count,level_num,total_count)
|
||
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
try:
|
||
postprocess_result.result = {
|
||
"OssreResult": {"data": new_data},"checkItems":{"data":new_checkItems},"hostInfo":{"data":new_hostInfo}
|
||
}
|
||
except:
|
||
traceback.print_exc()
|
||
pass
|
||
print (postprocess_result.result)
|
||
return postprocess_result
|
||
|