vcu118 support throughout firesim
This commit is contained in:
Sagar Karandikar
parent
ac5541c52d
commit
d6de4ab565
|
|
@ -19,3 +19,6 @@
|
|||
[submodule "utils/FlameGraph"]
|
||||
path = utils/fireperf/FlameGraph
|
||||
url = https://github.com/brendangregg/FlameGraph.git
|
||||
[submodule "platforms/xilinx_vcu118/garnet-firesim"]
|
||||
path = platforms/xilinx_vcu118/garnet-firesim
|
||||
url = https://github.com/firesim/garnet-firesim
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
# Build-time bitbuilder design configuration for the FireSim Simulation Manager
|
||||
# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params.
|
||||
|
||||
###########
|
||||
# Schema:
|
||||
###########
|
||||
# # Class name of the bitbuilder type.
|
||||
# # This can be determined from `deploy/buildtools/bitbuilder.py`).
|
||||
# bitbuilder_type: <TYPE NAME>
|
||||
# args:
|
||||
# # Bitbuilder arguments that are passed to the `BitBuilder`
|
||||
# # object. Determined by looking at `_parse_args` function of class.
|
||||
# <K/V pairs of args>
|
||||
|
||||
bit_builder_type: XilinxVCU118BitBuilder
|
||||
args: null
|
||||
|
|
@ -705,3 +705,53 @@ class XilinxAlveoU250BitBuilder(XilinxAlveoBitBuilder):
|
|||
def __init__(self, build_config: BuildConfig, args: Dict[str, Any]) -> None:
|
||||
super().__init__(build_config, args)
|
||||
self.BOARD_NAME = "au250"
|
||||
|
||||
class XilinxVCU118BitBuilder(XilinxAlveoBitBuilder):
|
||||
"""Bit builder class that builds a Xilinx VCU118 bitstream from the build config."""
|
||||
BOARD_NAME: Optional[str]
|
||||
|
||||
def __init__(self, build_config: BuildConfig, args: Dict[str, Any]) -> None:
|
||||
super().__init__(build_config, args)
|
||||
self.BOARD_NAME = "xilinx_vcu118"
|
||||
|
||||
def cl_dir_setup(self, chisel_quintuplet: str, dest_build_dir: str) -> str:
|
||||
"""Setup CL_DIR on build host.
|
||||
|
||||
Args:
|
||||
chisel_quintuplet: Build config chisel quintuplet used to uniquely identify build dir.
|
||||
dest_build_dir: Destination base directory to use.
|
||||
|
||||
Returns:
|
||||
Path to CL_DIR directory (that is setup) or `None` if invalid.
|
||||
"""
|
||||
fpga_build_postfix = f"cl_{chisel_quintuplet}"
|
||||
|
||||
# local paths
|
||||
local_alveo_dir = f"{get_deploy_dir()}/../platforms/{self.PLATFORM}/garnet-firesim"
|
||||
|
||||
dest_alveo_dir = f"{dest_build_dir}/platforms/{self.PLATFORM}/garnet-firesim"
|
||||
|
||||
# copy alveo files to the build instance.
|
||||
# do the rsync, but ignore any checkpoints that might exist on this machine
|
||||
# (in case builds were run locally)
|
||||
# extra_opts -L resolves symlinks
|
||||
|
||||
run(f'mkdir -p {dest_alveo_dir}')
|
||||
rsync_cap = rsync_project(
|
||||
local_dir=local_alveo_dir,
|
||||
remote_dir=dest_alveo_dir,
|
||||
ssh_opts="-o StrictHostKeyChecking=no",
|
||||
exclude="cl_*",
|
||||
extra_opts="-L", capture=True)
|
||||
rootLogger.debug(rsync_cap)
|
||||
rootLogger.debug(rsync_cap.stderr)
|
||||
rsync_cap = rsync_project(
|
||||
local_dir=f"{local_alveo_dir}/{fpga_build_postfix}/",
|
||||
remote_dir=f'{dest_alveo_dir}/{fpga_build_postfix}',
|
||||
ssh_opts="-o StrictHostKeyChecking=no",
|
||||
extra_opts="-L", capture=True)
|
||||
rootLogger.debug(rsync_cap)
|
||||
rootLogger.debug(rsync_cap.stderr)
|
||||
|
||||
return f"{dest_alveo_dir}/{fpga_build_postfix}"
|
||||
|
||||
|
|
|
|||
|
|
@ -184,7 +184,7 @@ def managerinit(args: argparse.Namespace):
|
|||
"managerinit replace start",
|
||||
"managerinit replace end",
|
||||
bf_recipe_lines)
|
||||
elif args.platform == 'vitis' or args.platform == 'xilinx_alveo_u250' or args.platform == 'xilinx_alveo_u280':
|
||||
elif args.platform == 'vitis' or args.platform == 'xilinx_alveo_u250' or args.platform == 'xilinx_alveo_u280' or args.platform == 'xilinx_vcu118':
|
||||
runfarm_default_file = "run-farm-recipes/externally_provisioned.yaml"
|
||||
with open(runfarm_default_file, "r") as f:
|
||||
rf_recipe_lines = f.readlines()
|
||||
|
|
|
|||
|
|
@ -847,3 +847,120 @@ class XilinxAlveoU280InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
|||
super().__init__(parent_node)
|
||||
self.PLATFORM_NAME = "xilinx_alveo_u280"
|
||||
self.BOARD_NAME = "au280"
|
||||
|
||||
class XilinxVCU118InstanceDeployManager(InstanceDeployManager):
|
||||
""" This class manages a Xilinx VCU118-enabled instance using the
|
||||
garnet shell. """
|
||||
|
||||
@classmethod
|
||||
def sim_command_requires_sudo(cls) -> bool:
|
||||
""" This sim does requires sudo. """
|
||||
return True
|
||||
|
||||
def __init__(self, parent_node: Inst) -> None:
|
||||
super().__init__(parent_node)
|
||||
|
||||
def unload_xdma(self) -> None:
|
||||
""" unload the xdma and xvsec kernel modules. """
|
||||
if self.instance_assigned_simulations():
|
||||
self.instance_logger("Unloading XDMA Driver Kernel Module.")
|
||||
|
||||
with warn_only():
|
||||
remote_kmsg("removing_xdma_xvsec_start")
|
||||
run('sudo rmmod xdma')
|
||||
run('sudo rmmod xvsec')
|
||||
remote_kmsg("removing_xdma_xvsec_end")
|
||||
|
||||
def load_xdma(self) -> None:
|
||||
""" load the xdma and xvsec kernel modules. """
|
||||
if self.instance_assigned_simulations():
|
||||
# unload first
|
||||
self.unload_xdma()
|
||||
# load xdma
|
||||
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
||||
# must be installed to this path on sim. machine
|
||||
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
||||
run(f"sudo insmod /lib/modules/$(uname -r)/updates/kernel/drivers/xvsec/xvsec.ko", shell=True)
|
||||
|
||||
def flash_fpgas(self) -> None:
|
||||
if self.instance_assigned_simulations():
|
||||
self.instance_logger("""Flash all FPGA Slots.""")
|
||||
|
||||
for slotno, firesimservernode in enumerate(self.parent_node.sim_slots):
|
||||
serv = self.parent_node.sim_slots[slotno]
|
||||
hwcfg = serv.get_resolved_server_hardware_config()
|
||||
|
||||
bit_tar = hwcfg.get_bit_tar_filename()
|
||||
remote_sim_dir = self.get_remote_sim_dir_for_slot(slotno)
|
||||
bit_tar_unpack_dir = f"{remote_sim_dir}/{self.PLATFORM_NAME}"
|
||||
bit = f"{remote_sim_dir}/{self.PLATFORM_NAME}/firesim.bit"
|
||||
|
||||
# at this point the tar file is in the sim slot
|
||||
run(f"rm -rf {bit_tar_unpack_dir}")
|
||||
run(f"tar xvf {remote_sim_dir}/{bit_tar} -C {remote_sim_dir}")
|
||||
|
||||
self.instance_logger(f"""Determine BDF for {slotno}""")
|
||||
collect = run('lspci | grep -i serial.*xilinx')
|
||||
|
||||
# TODO: is hardcoded cap 0x1 correct?
|
||||
# TODO: is "Partial Reconfig Clear File" useful (see xvsecctl help)?
|
||||
bdfs = [ { "busno": "0x" + i[:2], "devno": "0x" + i[3:5], "capno": "0x1" } for i in collect.splitlines() if len(i.strip()) >= 0 ]
|
||||
bdf = bdfs[slotno]
|
||||
|
||||
busno = bdf['busno']
|
||||
devno = bdf['devno']
|
||||
capno = bdf['capno']
|
||||
self.instance_logger(f"""Flashing FPGA Slot: {slotno} (bus:{busno}, dev:{devno}, cap:{capno}) with bit: {bit}""")
|
||||
run(f"""sudo xvsecctl -b {busno} -F {devno} -c {capno} -p {bit}""")
|
||||
|
||||
def infrasetup_instance(self, uridir: str) -> None:
|
||||
""" Handle infrastructure setup for this platform. """
|
||||
metasim_enabled = self.parent_node.metasimulation_enabled
|
||||
|
||||
if self.instance_assigned_simulations():
|
||||
# This is a sim-host node.
|
||||
|
||||
# copy sim infrastructure
|
||||
for slotno in range(len(self.parent_node.sim_slots)):
|
||||
self.copy_sim_slot_infrastructure(slotno, uridir)
|
||||
self.extract_driver_tarball(slotno)
|
||||
|
||||
if not metasim_enabled:
|
||||
# load xdma driver
|
||||
self.load_xdma()
|
||||
# flash fpgas
|
||||
self.flash_fpgas()
|
||||
|
||||
if self.instance_assigned_switches():
|
||||
# all nodes could have a switch
|
||||
for slotno in range(len(self.parent_node.switch_slots)):
|
||||
self.copy_switch_slot_infrastructure(slotno)
|
||||
|
||||
def terminate_instance(self) -> None:
|
||||
""" XilinxVCU118InstanceDeployManager machines cannot be terminated. """
|
||||
return
|
||||
|
||||
def start_sim_slot(self, slotno: int) -> None:
|
||||
""" start a simulation. (same as the default except that you have a mapping from slotno to a specific BDF)"""
|
||||
if self.instance_assigned_simulations():
|
||||
self.instance_logger(f"""Starting {self.sim_type_message} simulation for slot: {slotno}.""")
|
||||
remote_home_dir = self.parent_node.sim_dir
|
||||
remote_sim_dir = """{}/sim_slot_{}/""".format(remote_home_dir, slotno)
|
||||
assert slotno < len(self.parent_node.sim_slots), f"{slotno} can not index into sim_slots {len(self.parent_node.sim_slots)} on {self.parent_node.host}"
|
||||
server = self.parent_node.sim_slots[slotno]
|
||||
|
||||
self.instance_logger(f"""Determine BDF for {slotno}""")
|
||||
collect = run('lspci | grep -i serial.*xilinx')
|
||||
bdfs = [ i[:2] for i in collect.splitlines() if len(i.strip()) >= 0 ]
|
||||
bdf = bdfs[slotno]
|
||||
|
||||
# make the local job results dir for this sim slot
|
||||
server.mkdir_and_prep_local_job_results_dir()
|
||||
sim_start_script_local_path = server.write_sim_start_script(bdf, (self.sim_command_requires_sudo() and has_sudo()))
|
||||
put(sim_start_script_local_path, remote_sim_dir)
|
||||
|
||||
with cd(remote_sim_dir):
|
||||
run("chmod +x sim-run.sh")
|
||||
run("./sim-run.sh")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,9 @@ builds_to_run:
|
|||
# - alveou250_firesim_rocket_singlecore_no_nic
|
||||
# - alveou280_firesim_rocket_singlecore_no_nic
|
||||
|
||||
# Config for xilinx vcu118
|
||||
# - xilinx_vcu118_firesim_rocket_singlecore_no_nic
|
||||
|
||||
agfis_to_share:
|
||||
- firesim_rocket_quadcore_nic_l2_llc4mb_ddr3
|
||||
- firesim_rocket_quadcore_no_nic_l2_llc4mb_ddr3
|
||||
|
|
|
|||
|
|
@ -316,3 +316,18 @@ alveou280_firesim_rocket_singlecore_no_nic:
|
|||
post_build_hook: null
|
||||
metasim_customruntimeconfig: null
|
||||
bit_builder_recipe: bit-builder-recipes/xilinx_alveo_u280.yaml
|
||||
|
||||
# Additional Xilinx VCU118-only Config
|
||||
xilinx_vcu118_firesim_rocket_singlecore_no_nic:
|
||||
PLATFORM: xilinx_vcu118
|
||||
TARGET_PROJECT: firesim
|
||||
DESIGN: FireSim
|
||||
TARGET_CONFIG: FireSimRocketConfig
|
||||
PLATFORM_CONFIG: BaseXilinxVCU118Config
|
||||
deploy_quintuplet: null
|
||||
platform_config_args:
|
||||
fpga_frequency: 15
|
||||
build_strategy: TIMING
|
||||
post_build_hook: null
|
||||
metasim_customruntimeconfig: null
|
||||
bit_builder_recipe: bit-builder-recipes/xilinx_vcu118.yaml
|
||||
|
|
|
|||
|
|
@ -0,0 +1,80 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This script is called by FireSim's bitbuilder to create a bit file
|
||||
|
||||
# exit script if any command fails
|
||||
set -e
|
||||
set -o pipefail
|
||||
|
||||
usage() {
|
||||
echo "usage: ${0} [OPTIONS]"
|
||||
echo ""
|
||||
echo "Options"
|
||||
echo " --cl_dir : Custom logic directory to build Vivado bitstream from"
|
||||
echo " --frequency : Frequency in MHz of the desired FPGA host clock."
|
||||
echo " --strategy : A string to a precanned set of build directives.
|
||||
See aws-fpga documentation for more info/.
|
||||
For this platform TIMING and AREA supported."
|
||||
echo " --board : FPGA board {au250,au280}."
|
||||
echo " --help : Display this message"
|
||||
exit "$1"
|
||||
}
|
||||
|
||||
CL_DIR=""
|
||||
FREQUENCY=""
|
||||
STRATEGY=""
|
||||
BOARD=""
|
||||
|
||||
# getopts does not support long options, and is inflexible
|
||||
while [ "$1" != "" ];
|
||||
do
|
||||
case $1 in
|
||||
--help)
|
||||
usage 1 ;;
|
||||
--cl_dir )
|
||||
shift
|
||||
CL_DIR=$1 ;;
|
||||
--strategy )
|
||||
shift
|
||||
STRATEGY=$1 ;;
|
||||
--frequency )
|
||||
shift
|
||||
FREQUENCY=$1 ;;
|
||||
--board )
|
||||
shift
|
||||
BOARD=$1 ;;
|
||||
* )
|
||||
echo "invalid option $1"
|
||||
usage 1 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if [ -z "$CL_DIR" ] ; then
|
||||
echo "no cl directory specified"
|
||||
usage 1
|
||||
fi
|
||||
|
||||
if [ -z "$FREQUENCY" ] ; then
|
||||
echo "No --frequency specified"
|
||||
usage 1
|
||||
fi
|
||||
|
||||
if [ -z "$STRATEGY" ] ; then
|
||||
echo "No --strategy specified"
|
||||
usage 1
|
||||
fi
|
||||
|
||||
if [ -z "$BOARD" ] ; then
|
||||
echo "No --board specified"
|
||||
usage 1
|
||||
fi
|
||||
|
||||
# run build
|
||||
cd $CL_DIR/build
|
||||
vivado -mode batch -source $CL_DIR/../tcl/build.tcl -tclargs $FREQUENCY $STRATEGY $BOARD
|
||||
|
||||
# TODO: remove later. this is for temporary compatibility with u250 flow
|
||||
# in manager
|
||||
mkdir -p ../vivado_proj
|
||||
cp cl_firesim_pblock_partition_partial.bit ../vivado_proj/firesim.bit
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 716db1fe48c70c394ed2f46e15afa6d104e496cb
|
||||
|
|
@ -82,6 +82,12 @@ class BaseXilinxAlveoConfig extends Config(
|
|||
new midas.XilinxAlveoConfig
|
||||
)
|
||||
|
||||
class BaseXilinxVCU118Config extends Config(
|
||||
new WithWiringTransform ++
|
||||
new WithAsyncResetReplacement ++
|
||||
new midas.XilinxVCU118Config
|
||||
)
|
||||
|
||||
class BaseVitisConfig extends Config(
|
||||
new WithWiringTransform ++
|
||||
new WithAsyncResetReplacement ++
|
||||
|
|
|
|||
|
|
@ -66,6 +66,23 @@ $(xilinx_alveo_u280): $(header) $(DRIVER_CC) $(DRIVER_H) $(midas_cc) $(midas_h)
|
|||
DRIVER="$(DRIVER_CC)" \
|
||||
TOP_DIR=$(chipyard_dir)
|
||||
|
||||
$(xilinx_vcu118): export CXXFLAGS := $(CXXFLAGS) $(common_cxx_flags) $(DRIVER_CXXOPTS) \
|
||||
-idirafter ${CONDA_PREFIX}/include -idirafter /usr/include
|
||||
$(xilinx_vcu118): export LDFLAGS := $(LDFLAGS) $(common_ld_flags) -Wl,-rpath='$$$$ORIGIN' \
|
||||
-L${CONDA_PREFIX}/lib -Wl,-rpath-link=/usr/lib/x86_64-linux-gnu -L/usr/lib/x86_64-linux-gnu
|
||||
|
||||
# Compile Driver
|
||||
$(xilinx_vcu118): $(header) $(DRIVER_CC) $(DRIVER_H) $(midas_cc) $(midas_h)
|
||||
mkdir -p $(OUTPUT_DIR)/build
|
||||
cp $(header) $(OUTPUT_DIR)/build/
|
||||
$(MAKE) -C $(simif_dir) driver MAIN=$(PLATFORM) PLATFORM=$(PLATFORM) \
|
||||
DRIVER_NAME=$(DESIGN) \
|
||||
GEN_FILE_BASENAME=$(BASE_FILE_NAME) \
|
||||
GEN_DIR=$(OUTPUT_DIR)/build \
|
||||
OUT_DIR=$(OUTPUT_DIR) \
|
||||
DRIVER="$(DRIVER_CC)" \
|
||||
TOP_DIR=$(chipyard_dir)
|
||||
|
||||
$(vitis): export CXXFLAGS := $(CXXFLAGS) $(common_cxx_flags) $(DRIVER_CXXOPTS) \
|
||||
-idirafter ${CONDA_PREFIX}/include -idirafter /usr/include -idirafter $(XILINX_XRT)/include
|
||||
# -ldl needed for Ubuntu 20.04 systems (is backwards compatible with U18.04 systems)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,8 @@ else ifeq ($(PLATFORM), xilinx_alveo_u250)
|
|||
board_dir := $(platforms_dir)/xilinx_alveo_u250
|
||||
else ifeq ($(PLATFORM), xilinx_alveo_u280)
|
||||
board_dir := $(platforms_dir)/xilinx_alveo_u280
|
||||
else ifeq ($(PLATFORM), xilinx_vcu118)
|
||||
board_dir := $(platforms_dir)/xilinx_vcu118/garnet-firesim
|
||||
else ifeq ($(PLATFORM), f1)
|
||||
board_dir := $(platforms_dir)/f1/aws-fpga/hdk/cl/developer_designs
|
||||
else
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
simif_xilinx_alveo_u250.cc
|
||||
|
|
@ -122,6 +122,12 @@ class XilinxAlveoConfig extends Config(new Config((site, here, up) => {
|
|||
case PostLinkCircuitPath => None
|
||||
}) ++ new F1Config ++ new SimConfig)
|
||||
|
||||
class XilinxVCU118Config extends Config(new Config((site, here, up) => {
|
||||
case HostMemNumChannels => 1
|
||||
case PreLinkCircuitPath => None
|
||||
case PostLinkCircuitPath => None
|
||||
}) ++ new F1Config ++ new SimConfig)
|
||||
|
||||
class VitisConfig extends Config(new Config((site, here, up) => {
|
||||
case Platform => (p: Parameters) => new VitisShim()(p)
|
||||
case CPUManagedAXI4Key => None
|
||||
|
|
|
|||
Loading…
Reference in New Issue