Merge pull request #1130 from firesim/fix-buildafi-no-email-on-fail
Add warn_only to vivado builds + Postpone error until all builds complete
This commit is contained in:
commit
9394a18489
|
@ -8,7 +8,7 @@ import random
|
|||
import string
|
||||
import logging
|
||||
import os
|
||||
from fabric.api import prefix, local, run, env, lcd, parallel # type: ignore
|
||||
from fabric.api import prefix, local, run, env, lcd, parallel, settings # type: ignore
|
||||
from fabric.contrib.console import confirm # type: ignore
|
||||
from fabric.contrib.project import rsync_project # type: ignore
|
||||
|
||||
|
@ -66,12 +66,15 @@ class BitBuilder(metaclass=abc.ABCMeta):
|
|||
raise NotImplementedError
|
||||
|
||||
@abc.abstractmethod
|
||||
def build_bitstream(self, bypass: bool = False) -> None:
|
||||
def build_bitstream(self, bypass: bool = False) -> bool:
|
||||
"""Run bitstream build and terminate the build host at the end.
|
||||
Must run after `replace_rtl` and `build_driver` are run.
|
||||
|
||||
Args:
|
||||
bypass: If true, immediately return and terminate build host. Used for testing purposes.
|
||||
|
||||
Returns:
|
||||
Boolean indicating if the build passed or failed.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
@ -169,15 +172,18 @@ class F1BitBuilder(BitBuilder):
|
|||
|
||||
return f"{dest_awsfpga_dir}/{fpga_build_postfix}"
|
||||
|
||||
def build_bitstream(self, bypass: bool = False) -> None:
|
||||
def build_bitstream(self, bypass: bool = False) -> bool:
|
||||
"""Run Vivado, convert tar -> AGFI/AFI, and then terminate the instance at the end.
|
||||
|
||||
Args:
|
||||
bypass: If true, immediately return and terminate build host. Used for testing purposes.
|
||||
|
||||
Returns:
|
||||
Boolean indicating if the build passed or failed.
|
||||
"""
|
||||
if bypass:
|
||||
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
|
||||
return
|
||||
return True
|
||||
|
||||
# The default error-handling procedure. Send an email and teardown instance
|
||||
def on_build_failure():
|
||||
|
@ -216,7 +222,8 @@ class F1BitBuilder(BitBuilder):
|
|||
rootLogger.debug(rsync_cap)
|
||||
rootLogger.debug(rsync_cap.stderr)
|
||||
|
||||
vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
|
||||
with settings(warn_only=True):
|
||||
vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
|
||||
|
||||
# put build results in the result-build area
|
||||
|
||||
|
@ -230,14 +237,16 @@ class F1BitBuilder(BitBuilder):
|
|||
|
||||
if vivado_result != 0:
|
||||
on_build_failure()
|
||||
return
|
||||
return False
|
||||
|
||||
if not self.aws_create_afi():
|
||||
on_build_failure()
|
||||
return
|
||||
return False
|
||||
|
||||
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
|
||||
|
||||
return True
|
||||
|
||||
def aws_create_afi(self) -> Optional[bool]:
|
||||
"""Convert the tarball created by Vivado build into an Amazon Global FPGA Image (AGFI).
|
||||
|
||||
|
@ -400,7 +409,7 @@ class VitisBitBuilder(BitBuilder):
|
|||
# do the rsync, but ignore any checkpoints that might exist on this machine
|
||||
# (in case builds were run locally)
|
||||
# extra_opts -l preserves symlinks
|
||||
|
||||
|
||||
run('mkdir -p {}'.format(dest_vitis_dir))
|
||||
rsync_cap = rsync_project(
|
||||
local_dir=local_vitis_dir,
|
||||
|
@ -420,15 +429,18 @@ class VitisBitBuilder(BitBuilder):
|
|||
|
||||
return f"{dest_vitis_dir}/{fpga_build_postfix}"
|
||||
|
||||
def build_bitstream(self, bypass: bool = False) -> None:
|
||||
def build_bitstream(self, bypass: bool = False) -> bool:
|
||||
""" Run Vitis to generate an xclbin. Then terminate the instance at the end.
|
||||
|
||||
Args:
|
||||
bypass: If true, immediately return and terminate build host. Used for testing purposes.
|
||||
|
||||
Returns:
|
||||
Boolean indicating if the build passed or failed.
|
||||
"""
|
||||
if bypass:
|
||||
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
|
||||
return
|
||||
return True
|
||||
|
||||
# The default error-handling procedure. Send an email and teardown instance
|
||||
def on_build_failure():
|
||||
|
@ -471,7 +483,8 @@ class VitisBitBuilder(BitBuilder):
|
|||
rootLogger.debug(rsync_cap)
|
||||
rootLogger.debug(rsync_cap.stderr)
|
||||
|
||||
vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
|
||||
with settings(warn_only=True):
|
||||
vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
|
||||
|
||||
# put build results in the result-build area
|
||||
|
||||
|
@ -485,7 +498,7 @@ class VitisBitBuilder(BitBuilder):
|
|||
|
||||
if vitis_result != 0:
|
||||
on_build_failure()
|
||||
return
|
||||
return False
|
||||
|
||||
hwdb_entry_name = self.build_config.name
|
||||
xclbin_path = cl_dir + "/bitstream/build_dir.xilinx_u250_gen3x16_xdma_3_1_202020_1/firesim.xclbin"
|
||||
|
@ -512,7 +525,6 @@ class VitisBitBuilder(BitBuilder):
|
|||
outputfile.write(hwdb_entry)
|
||||
|
||||
if self.build_config.post_build_hook:
|
||||
|
||||
localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True)
|
||||
rootLogger.debug("[localhost] " + str(localcap))
|
||||
rootLogger.debug("[localhost] " + str(localcap.stderr))
|
||||
|
@ -520,3 +532,5 @@ class VitisBitBuilder(BitBuilder):
|
|||
rootLogger.info(f"Build complete! Vitis bitstream ready. See {os.path.join(hwdb_entry_file_location,hwdb_entry_name)}.")
|
||||
|
||||
self.build_config.build_config_file.build_farm.release_build_host(self.build_config)
|
||||
|
||||
return True
|
||||
|
|
|
@ -283,18 +283,24 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None:
|
|||
build_config_file.wait_on_build_host_initializations()
|
||||
|
||||
@parallel
|
||||
def parallel_build_helper(config_file: BuildConfigFile, bypass: bool = False) -> None:
|
||||
def parallel_build_helper(config_file: BuildConfigFile, bypass: bool = False) -> bool:
|
||||
"""Wrap parallel portion of ``build_bitstream``
|
||||
|
||||
Args:
|
||||
config_file: BuildConfigFile
|
||||
bypass: If true, immediately return and terminate build host. Used for testing purposes.
|
||||
|
||||
Returns:
|
||||
Boolean indicating if the build passed or not.
|
||||
"""
|
||||
build_config_file = config_file.get_build_by_ip(env.host_string)
|
||||
build_config_file.bitbuilder.build_bitstream(bypass)
|
||||
return build_config_file.bitbuilder.build_bitstream(bypass)
|
||||
|
||||
# run builds, then terminate instances
|
||||
execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set)
|
||||
results = execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set)
|
||||
if False in results.values():
|
||||
rootLogger.critical("ERROR: A bitstream build failed.")
|
||||
sys.exit(1)
|
||||
|
||||
@register_task
|
||||
def builddriver(runtime_conf: RuntimeConfig) -> None:
|
||||
|
|
Loading…
Reference in New Issue