replica
/
firesim
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Aggregate bitstream failures after all completions

This commit is contained in:
Abraham Gonzalez 2022-07-13 04:36:01 +00:00
parent fc0a80d4e3
commit 8e23e1f634
2 changed files with 36 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
deploy/buildtools/bitbuilder.py
View File

@ -8,7 +8,7 @@ import random
import string
import logging
import os
from fabric.api import prefix, local, run, env, lcd, parallel # type: ignore
from fabric.api import prefix, local, run, env, lcd, parallel, settings # type: ignore
from fabric.contrib.console import confirm # type: ignore
from fabric.contrib.project import rsync_project # type: ignore
@ -66,12 +66,15 @@ class BitBuilder(metaclass=abc.ABCMeta):
raise NotImplementedError
@abc.abstractmethod
def build_bitstream(self, bypass: bool = False) -> None:
def build_bitstream(self, bypass: bool = False) -> bool:
"""Run bitstream build and terminate the build host at the end.
Must run after `replace_rtl` and `build_driver` are run.
Args:
bypass: If true, immediately return and terminate build host. Used for testing purposes.
Returns:
Boolean indicating if the build passed or failed.
"""
raise NotImplementedError
@ -169,15 +172,18 @@ class F1BitBuilder(BitBuilder):
return f"{dest_awsfpga_dir}/{fpga_build_postfix}"
def build_bitstream(self, bypass: bool = False) -> None:
def build_bitstream(self, bypass: bool = False) -> bool:
"""Run Vivado, convert tar -> AGFI/AFI, and then terminate the instance at the end.
Args:
bypass: If true, immediately return and terminate build host. Used for testing purposes.
Returns:
Boolean indicating if the build passed or failed.
"""
if bypass:
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
return
return True
# The default error-handling procedure. Send an email and teardown instance
def on_build_failure():
@ -216,7 +222,8 @@ class F1BitBuilder(BitBuilder):
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
with settings(warn_only=True):
vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
# put build results in the result-build area
@ -230,14 +237,16 @@ class F1BitBuilder(BitBuilder):
if vivado_result != 0:
on_build_failure()
return
return False
if not self.aws_create_afi():
on_build_failure()
return
return False
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
return True
def aws_create_afi(self) -> Optional[bool]:
"""Convert the tarball created by Vivado build into an Amazon Global FPGA Image (AGFI).
@ -420,15 +429,18 @@ class VitisBitBuilder(BitBuilder):
return f"{dest_vitis_dir}/{fpga_build_postfix}"
def build_bitstream(self, bypass: bool = False) -> None:
def build_bitstream(self, bypass: bool = False) -> bool:
""" Run Vitis to generate an xclbin. Then terminate the instance at the end.
Args:
bypass: If true, immediately return and terminate build host. Used for testing purposes.
Returns:
Boolean indicating if the build passed or failed.
"""
if bypass:
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
return
return True
# The default error-handling procedure. Send an email and teardown instance
def on_build_failure():
@ -471,7 +483,8 @@ class VitisBitBuilder(BitBuilder):
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
with settings(warn_only=True):
vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
# put build results in the result-build area
@ -485,7 +498,7 @@ class VitisBitBuilder(BitBuilder):
if vitis_result != 0:
on_build_failure()
return
return False
hwdb_entry_name = self.build_config.name
xclbin_path = cl_dir + "/bitstream/build_dir.xilinx_u250_gen3x16_xdma_3_1_202020_1/firesim.xclbin"
@ -512,7 +525,6 @@ class VitisBitBuilder(BitBuilder):
outputfile.write(hwdb_entry)
if self.build_config.post_build_hook:
localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
@ -520,3 +532,5 @@ class VitisBitBuilder(BitBuilder):
rootLogger.info(f"Build complete! Vitis bitstream ready. See {os.path.join(hwdb_entry_file_location,hwdb_entry_name)}.")
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
return True

12
deploy/firesim
View File

@ -280,18 +280,24 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None:
build_config_file.wait_on_build_host_initializations()
@parallel
def parallel_build_helper(config_file: BuildConfigFile, bypass: bool = False) -> None:
def parallel_build_helper(config_file: BuildConfigFile, bypass: bool = False) -> bool:
"""Wrap parallel portion of ``build_bitstream``
Args:
config_file: BuildConfigFile
bypass: If true, immediately return and terminate build host. Used for testing purposes.
Returns:
Boolean indicating if the build passed or not.
"""
build_config_file = config_file.get_build_by_ip(env.host_string)
build_config_file.bitbuilder.build_bitstream(bypass)
return build_config_file.bitbuilder.build_bitstream(bypass)
# run builds, then terminate instances
execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set)
results = execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set)
if False in results.values():
rootLogger.critical("ERROR: A bitstream build failed.")
sys.exit(1)
@register_task
def builddriver(runtime_conf: RuntimeConfig) -> None: