diff --git a/deploy/buildtools/bitbuilder.py b/deploy/buildtools/bitbuilder.py index d9adb275..0f2d4a61 100644 --- a/deploy/buildtools/bitbuilder.py +++ b/deploy/buildtools/bitbuilder.py @@ -8,7 +8,7 @@ import random import string import logging import os -from fabric.api import prefix, local, run, env, lcd, parallel # type: ignore +from fabric.api import prefix, local, run, env, lcd, parallel, settings # type: ignore from fabric.contrib.console import confirm # type: ignore from fabric.contrib.project import rsync_project # type: ignore @@ -66,12 +66,15 @@ class BitBuilder(metaclass=abc.ABCMeta): raise NotImplementedError @abc.abstractmethod - def build_bitstream(self, bypass: bool = False) -> None: + def build_bitstream(self, bypass: bool = False) -> bool: """Run bitstream build and terminate the build host at the end. Must run after `replace_rtl` and `build_driver` are run. Args: bypass: If true, immediately return and terminate build host. Used for testing purposes. + + Returns: + Boolean indicating if the build passed or failed. """ raise NotImplementedError @@ -169,15 +172,18 @@ class F1BitBuilder(BitBuilder): return f"{dest_awsfpga_dir}/{fpga_build_postfix}" - def build_bitstream(self, bypass: bool = False) -> None: + def build_bitstream(self, bypass: bool = False) -> bool: """Run Vivado, convert tar -> AGFI/AFI, and then terminate the instance at the end. Args: bypass: If true, immediately return and terminate build host. Used for testing purposes. + + Returns: + Boolean indicating if the build passed or failed. """ if bypass: self.build_config.build_config_file.build_farm.release_build_host(self.build_config) - return + return True # The default error-handling procedure. Send an email and teardown instance def on_build_failure(): @@ -216,7 +222,8 @@ class F1BitBuilder(BitBuilder): rootLogger.debug(rsync_cap) rootLogger.debug(rsync_cap.stderr) - vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code + with settings(warn_only=True): + vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code # put build results in the result-build area @@ -230,14 +237,16 @@ class F1BitBuilder(BitBuilder): if vivado_result != 0: on_build_failure() - return + return False if not self.aws_create_afi(): on_build_failure() - return + return False self.build_config.build_config_file.build_farm.release_build_host(self.build_config) + return True + def aws_create_afi(self) -> Optional[bool]: """Convert the tarball created by Vivado build into an Amazon Global FPGA Image (AGFI). @@ -400,7 +409,7 @@ class VitisBitBuilder(BitBuilder): # do the rsync, but ignore any checkpoints that might exist on this machine # (in case builds were run locally) # extra_opts -l preserves symlinks - + run('mkdir -p {}'.format(dest_vitis_dir)) rsync_cap = rsync_project( local_dir=local_vitis_dir, @@ -420,15 +429,18 @@ class VitisBitBuilder(BitBuilder): return f"{dest_vitis_dir}/{fpga_build_postfix}" - def build_bitstream(self, bypass: bool = False) -> None: + def build_bitstream(self, bypass: bool = False) -> bool: """ Run Vitis to generate an xclbin. Then terminate the instance at the end. Args: bypass: If true, immediately return and terminate build host. Used for testing purposes. + + Returns: + Boolean indicating if the build passed or failed. """ if bypass: self.build_config.build_config_file.build_farm.release_build_host(self.build_config) - return + return True # The default error-handling procedure. Send an email and teardown instance def on_build_failure(): @@ -471,7 +483,8 @@ class VitisBitBuilder(BitBuilder): rootLogger.debug(rsync_cap) rootLogger.debug(rsync_cap.stderr) - vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code + with settings(warn_only=True): + vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code # put build results in the result-build area @@ -485,7 +498,7 @@ class VitisBitBuilder(BitBuilder): if vitis_result != 0: on_build_failure() - return + return False hwdb_entry_name = self.build_config.name xclbin_path = cl_dir + "/bitstream/build_dir.xilinx_u250_gen3x16_xdma_3_1_202020_1/firesim.xclbin" @@ -512,7 +525,6 @@ class VitisBitBuilder(BitBuilder): outputfile.write(hwdb_entry) if self.build_config.post_build_hook: - localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True) rootLogger.debug("[localhost] " + str(localcap)) rootLogger.debug("[localhost] " + str(localcap.stderr)) @@ -520,3 +532,5 @@ class VitisBitBuilder(BitBuilder): rootLogger.info(f"Build complete! Vitis bitstream ready. See {os.path.join(hwdb_entry_file_location,hwdb_entry_name)}.") self.build_config.build_config_file.build_farm.release_build_host(self.build_config) + + return True diff --git a/deploy/firesim b/deploy/firesim index 29d4e6a8..3d3a10eb 100755 --- a/deploy/firesim +++ b/deploy/firesim @@ -283,18 +283,24 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None: build_config_file.wait_on_build_host_initializations() @parallel - def parallel_build_helper(config_file: BuildConfigFile, bypass: bool = False) -> None: + def parallel_build_helper(config_file: BuildConfigFile, bypass: bool = False) -> bool: """Wrap parallel portion of ``build_bitstream`` Args: config_file: BuildConfigFile bypass: If true, immediately return and terminate build host. Used for testing purposes. + + Returns: + Boolean indicating if the build passed or not. """ build_config_file = config_file.get_build_by_ip(env.host_string) - build_config_file.bitbuilder.build_bitstream(bypass) + return build_config_file.bitbuilder.build_bitstream(bypass) # run builds, then terminate instances - execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set) + results = execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set) + if False in results.values(): + rootLogger.critical("ERROR: A bitstream build failed.") + sys.exit(1) @register_task def builddriver(runtime_conf: RuntimeConfig) -> None: