Fix very long job failure text in Slack report (#17630)
* Fix very long job failure text in Slack report Co-authored-by: ydshieh <ydshieh@users.noreply.github.com>
This commit is contained in:
parent
2351729f7d
commit
c70dacde94
|
@ -407,8 +407,22 @@ class Message:
|
||||||
)
|
)
|
||||||
|
|
||||||
def get_reply_blocks(self, job_name, job_result, failures, device, text):
|
def get_reply_blocks(self, job_name, job_result, failures, device, text):
|
||||||
if len(failures) > 2500:
|
"""
|
||||||
failures = "\n".join(failures.split("\n")[:20]) + "\n\n[Truncated]"
|
failures: A list with elements of the form {"line": full test name, "trace": error trace}
|
||||||
|
"""
|
||||||
|
# `text` must be less than 3001 characters in Slack SDK
|
||||||
|
# keep some room for adding "[Truncated]" when necessary
|
||||||
|
MAX_ERROR_TEXT = 3000 - len("[Truncated]")
|
||||||
|
|
||||||
|
failure_text = ""
|
||||||
|
for idx, error in enumerate(failures):
|
||||||
|
new_text = failure_text + f'*{error["line"]}*\n_{error["trace"]}_\n\n'
|
||||||
|
if len(new_text) > MAX_ERROR_TEXT:
|
||||||
|
# `failure_text` here has length <= 3000
|
||||||
|
failure_text = failure_text + "[Truncated]"
|
||||||
|
break
|
||||||
|
# `failure_text` here has length <= MAX_ERROR_TEXT
|
||||||
|
failure_text = new_text
|
||||||
|
|
||||||
title = job_name
|
title = job_name
|
||||||
if device is not None:
|
if device is not None:
|
||||||
|
@ -426,7 +440,7 @@ class Message:
|
||||||
return [
|
return [
|
||||||
{"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
|
{"type": "header", "text": {"type": "plain_text", "text": title.upper(), "emoji": True}},
|
||||||
content,
|
content,
|
||||||
{"type": "section", "text": {"type": "mrkdwn", "text": failures}},
|
{"type": "section", "text": {"type": "mrkdwn", "text": failure_text}},
|
||||||
]
|
]
|
||||||
|
|
||||||
def post_reply(self):
|
def post_reply(self):
|
||||||
|
@ -638,11 +652,11 @@ if __name__ == "__main__":
|
||||||
line = line.split()[0].replace("\n", "")
|
line = line.split()[0].replace("\n", "")
|
||||||
|
|
||||||
if artifact_path["gpu"] not in model_results[model]["failures"]:
|
if artifact_path["gpu"] not in model_results[model]["failures"]:
|
||||||
model_results[model]["failures"][artifact_path["gpu"]] = ""
|
model_results[model]["failures"][artifact_path["gpu"]] = []
|
||||||
|
|
||||||
model_results[model]["failures"][
|
model_results[model]["failures"][artifact_path["gpu"]].append(
|
||||||
artifact_path["gpu"]
|
{"line": line, "trace": stacktraces.pop(0)}
|
||||||
] += f"*{line}*\n_{stacktraces.pop(0)}_\n\n"
|
)
|
||||||
|
|
||||||
if re.search("test_modeling_tf_", line):
|
if re.search("test_modeling_tf_", line):
|
||||||
model_results[model]["failed"]["TensorFlow"][artifact_path["gpu"]] += 1
|
model_results[model]["failed"]["TensorFlow"][artifact_path["gpu"]] += 1
|
||||||
|
@ -727,11 +741,11 @@ if __name__ == "__main__":
|
||||||
line = line.split()[0].replace("\n", "")
|
line = line.split()[0].replace("\n", "")
|
||||||
|
|
||||||
if artifact_path["gpu"] not in additional_results[key]["failures"]:
|
if artifact_path["gpu"] not in additional_results[key]["failures"]:
|
||||||
additional_results[key]["failures"][artifact_path["gpu"]] = ""
|
additional_results[key]["failures"][artifact_path["gpu"]] = []
|
||||||
|
|
||||||
additional_results[key]["failures"][
|
additional_results[key]["failures"][artifact_path["gpu"]].append(
|
||||||
artifact_path["gpu"]
|
{"line": line, "trace": stacktraces.pop(0)}
|
||||||
] += f"*{line}*\n_{stacktraces.pop(0)}_\n\n"
|
)
|
||||||
|
|
||||||
# To find the PR number in a commit title, for example, `Add AwesomeFormer model (#99999)`
|
# To find the PR number in a commit title, for example, `Add AwesomeFormer model (#99999)`
|
||||||
pr_number_re = re.compile(r"\(#(\d+)\)$")
|
pr_number_re = re.compile(r"\(#(\d+)\)$")
|
||||||
|
|
Loading…
Reference in New Issue