Merge pull request #1502 from firesim/ntnu-extended
Misc. U250/U280 FPGA Fixes
This commit is contained in:
commit
2db8ab547a
|
@ -21,6 +21,7 @@ env:
|
||||||
FIRESIM_PEM: ${{ secrets.FIRESIM_PEM }}
|
FIRESIM_PEM: ${{ secrets.FIRESIM_PEM }}
|
||||||
FIRESIM_PEM_PUBLIC: ${{ secrets.FIRESIM_PEM_PUBLIC }}
|
FIRESIM_PEM_PUBLIC: ${{ secrets.FIRESIM_PEM_PUBLIC }}
|
||||||
MANAGER_FIRESIM_LOCATION: "~/firesim"
|
MANAGER_FIRESIM_LOCATION: "~/firesim"
|
||||||
|
REMOTE_WORK_DIR: unused
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
cull-old-ci-instances:
|
cull-old-ci-instances:
|
||||||
|
|
|
@ -303,6 +303,11 @@ def builddriver(runtime_conf: RuntimeConfig) -> None:
|
||||||
an entire metasim)."""
|
an entire metasim)."""
|
||||||
runtime_conf.build_driver()
|
runtime_conf.build_driver()
|
||||||
|
|
||||||
|
@register_task
|
||||||
|
def enumeratefpgas(runtime_conf: RuntimeConfig) -> None:
|
||||||
|
""" For all run hosts, create the /opt/firesim-db.json file """
|
||||||
|
runtime_conf.enumerate_fpgas()
|
||||||
|
|
||||||
@register_task
|
@register_task
|
||||||
# XXX this needs to be renamed or rethought, perhaps this is a backend-specific task?
|
# XXX this needs to be renamed or rethought, perhaps this is a backend-specific task?
|
||||||
def tar2afi(build_config_file: BuildConfigFile) -> None:
|
def tar2afi(build_config_file: BuildConfigFile) -> None:
|
||||||
|
|
|
@ -447,6 +447,27 @@ class FireSimTopologyWithPasses:
|
||||||
|
|
||||||
execute(infrasetup_node_wrapper, self.run_farm, uridir, hosts=all_run_farm_ips)
|
execute(infrasetup_node_wrapper, self.run_farm, uridir, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
|
def enumerate_fpgas_passes(self, use_mock_instances_for_testing: bool) -> None:
|
||||||
|
""" extra passes needed to do enumerate_fpgas """
|
||||||
|
self.run_farm.post_launch_binding(use_mock_instances_for_testing)
|
||||||
|
|
||||||
|
@parallel
|
||||||
|
def enumerate_fpgas_node_wrapper(run_farm: RunFarm, dir: str) -> None:
|
||||||
|
my_node = run_farm.lookup_by_host(env.host_string)
|
||||||
|
assert my_node is not None
|
||||||
|
assert my_node.instance_deploy_manager is not None
|
||||||
|
my_node.instance_deploy_manager.enumerate_fpgas(dir)
|
||||||
|
|
||||||
|
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
|
||||||
|
execute(instance_liveness, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
|
# Steps occur within the context of a tempdir.
|
||||||
|
# This allows URI's to survive until after deploy, and cleanup upon error
|
||||||
|
with TemporaryDirectory() as uridir:
|
||||||
|
self.pass_fetch_URI_resolve_runtime_cfg(uridir)
|
||||||
|
self.pass_build_required_drivers()
|
||||||
|
execute(enumerate_fpgas_node_wrapper, self.run_farm, uridir, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
def build_driver_passes(self) -> None:
|
def build_driver_passes(self) -> None:
|
||||||
""" Only run passes to build drivers. """
|
""" Only run passes to build drivers. """
|
||||||
|
|
||||||
|
@ -496,8 +517,6 @@ class FireSimTopologyWithPasses:
|
||||||
""" Passes that kill the simulator. """
|
""" Passes that kill the simulator. """
|
||||||
self.run_farm.post_launch_binding(use_mock_instances_for_testing)
|
self.run_farm.post_launch_binding(use_mock_instances_for_testing)
|
||||||
|
|
||||||
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
|
|
||||||
|
|
||||||
@parallel
|
@parallel
|
||||||
def kill_switch_wrapper(run_farm: RunFarm) -> None:
|
def kill_switch_wrapper(run_farm: RunFarm) -> None:
|
||||||
my_node = run_farm.lookup_by_host(env.host_string)
|
my_node = run_farm.lookup_by_host(env.host_string)
|
||||||
|
@ -510,6 +529,13 @@ class FireSimTopologyWithPasses:
|
||||||
assert my_node.instance_deploy_manager is not None
|
assert my_node.instance_deploy_manager is not None
|
||||||
my_node.instance_deploy_manager.kill_simulations_instance(disconnect_all_nbds=disconnect_all_nbds)
|
my_node.instance_deploy_manager.kill_simulations_instance(disconnect_all_nbds=disconnect_all_nbds)
|
||||||
|
|
||||||
|
# Steps occur within the context of a tempdir.
|
||||||
|
# This allows URI's to survive until after deploy, and cleanup upon error
|
||||||
|
with TemporaryDirectory() as uridir:
|
||||||
|
self.pass_fetch_URI_resolve_runtime_cfg(uridir)
|
||||||
|
|
||||||
|
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
|
||||||
|
|
||||||
execute(kill_switch_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
execute(kill_switch_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
||||||
execute(kill_simulation_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
execute(kill_simulation_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
|
|
|
@ -84,6 +84,11 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
"""Run platform specific implementation of how to enumerate FPGAs for FireSim."""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
"""Run platform specific implementation of how to terminate host
|
"""Run platform specific implementation of how to terminate host
|
||||||
|
@ -658,6 +663,10 @@ class EC2InstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" FPGAs are enumerated already with F1 """
|
||||||
|
return
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
self.instance_logger("Terminating instance", debug=True)
|
self.instance_logger("Terminating instance", debug=True)
|
||||||
self.parent_node.terminate_self()
|
self.parent_node.terminate_self()
|
||||||
|
@ -717,6 +726,10 @@ class VitisInstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" FPGAs are enumerated already with Vitis """
|
||||||
|
return
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
""" VitisInstanceDeployManager machines cannot be terminated. """
|
""" VitisInstanceDeployManager machines cannot be terminated. """
|
||||||
return
|
return
|
||||||
|
@ -724,7 +737,7 @@ class VitisInstanceDeployManager(InstanceDeployManager):
|
||||||
class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
""" This class manages a Xilinx Alveo-enabled instance """
|
""" This class manages a Xilinx Alveo-enabled instance """
|
||||||
PLATFORM_NAME: Optional[str]
|
PLATFORM_NAME: Optional[str]
|
||||||
BOARD_NAME: Optional[str]
|
JSON_DB: str = "/opt/firesim-db.json"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def sim_command_requires_sudo(cls) -> bool:
|
def sim_command_requires_sudo(cls) -> bool:
|
||||||
|
@ -734,33 +747,32 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = None
|
self.PLATFORM_NAME = None
|
||||||
self.BOARD_NAME = None
|
|
||||||
|
|
||||||
def unload_xdma(self) -> None:
|
|
||||||
if self.instance_assigned_simulations():
|
|
||||||
self.instance_logger("Unloading XDMA Driver Kernel Module.")
|
|
||||||
|
|
||||||
with warn_only():
|
|
||||||
remote_kmsg("removing_xdma_start")
|
|
||||||
run('sudo rmmod xdma')
|
|
||||||
remote_kmsg("removing_xdma_end")
|
|
||||||
|
|
||||||
def load_xdma(self) -> None:
|
def load_xdma(self) -> None:
|
||||||
""" load the xdma kernel module. """
|
""" load the xdma kernel module. """
|
||||||
if self.instance_assigned_simulations():
|
if self.instance_assigned_simulations():
|
||||||
# unload first
|
# load xdma if unloaded
|
||||||
self.unload_xdma()
|
if run('lsmod | grep -wq xdma', warn_only=True).return_code != 0:
|
||||||
# load xdma
|
|
||||||
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
||||||
# must be installed to this path on sim. machine
|
# must be installed to this path on sim. machine
|
||||||
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
||||||
|
else:
|
||||||
|
self.instance_logger("XDMA Driver Kernel Module already loaded.")
|
||||||
|
|
||||||
|
def slot_to_bdf(self, slotno: int) -> str:
|
||||||
|
# get fpga information from db
|
||||||
|
self.instance_logger(f"""Determine BDF for {slotno}""")
|
||||||
|
collect = run(f'cat {self.JSON_DB}')
|
||||||
|
db = json.loads(collect)
|
||||||
|
assert slotno < len(db), f"Less FPGAs available than slots ({slotno} >= {len(db)})"
|
||||||
|
return db[slotno]['bdf']
|
||||||
|
|
||||||
def flash_fpgas(self) -> None:
|
def flash_fpgas(self) -> None:
|
||||||
if self.instance_assigned_simulations():
|
if self.instance_assigned_simulations():
|
||||||
self.instance_logger("""Flash all FPGA Slots.""")
|
self.instance_logger("""Flash all FPGA Slots.""")
|
||||||
|
|
||||||
for slotno, firesimservernode in enumerate(self.parent_node.sim_slots):
|
for slotno, firesimservernode in enumerate(self.parent_node.sim_slots):
|
||||||
serv = self.parent_node.sim_slots[slotno]
|
serv = firesimservernode
|
||||||
hwcfg = serv.get_resolved_server_hardware_config()
|
hwcfg = serv.get_resolved_server_hardware_config()
|
||||||
|
|
||||||
bitstream_tar = hwcfg.get_bitstream_tar_filename()
|
bitstream_tar = hwcfg.get_bitstream_tar_filename()
|
||||||
|
@ -782,13 +794,10 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
rootLogger.debug(rsync_cap)
|
rootLogger.debug(rsync_cap)
|
||||||
rootLogger.debug(rsync_cap.stderr)
|
rootLogger.debug(rsync_cap.stderr)
|
||||||
|
|
||||||
self.instance_logger(f"""Determine BDF for {slotno}""")
|
bdf = self.slot_to_bdf(slotno)
|
||||||
collect = run('lspci | grep -i serial.*xilinx')
|
|
||||||
bdfs = [ "0000:" + i[:7] for i in collect.splitlines() if len(i.strip()) >= 0 ]
|
|
||||||
bdf = bdfs[slotno]
|
|
||||||
|
|
||||||
self.instance_logger(f"""Flashing FPGA Slot: {slotno} with bit: {bit}""")
|
self.instance_logger(f"""Flashing FPGA Slot: {slotno} ({bdf}) with bitstream: {bit}""")
|
||||||
run(f"""EXTENDED_DEVICE_BDF1={bdf} {remote_sim_dir}/scripts/program_fpga.sh {bit} {self.BOARD_NAME}""")
|
run(f"""{remote_sim_dir}/scripts/fpga-util.py --bitstream {bit} --bdf {bdf}""")
|
||||||
|
|
||||||
def infrasetup_instance(self, uridir: str) -> None:
|
def infrasetup_instance(self, uridir: str) -> None:
|
||||||
""" Handle infrastructure setup for this platform. """
|
""" Handle infrastructure setup for this platform. """
|
||||||
|
@ -813,6 +822,70 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def create_fpga_database(self, uridir: str) -> None:
|
||||||
|
self.instance_logger(f"""Creating FPGA database""")
|
||||||
|
|
||||||
|
remote_home_dir = self.parent_node.get_sim_dir()
|
||||||
|
remote_sim_dir = f"{remote_home_dir}/enumerate_fpgas_staging"
|
||||||
|
remote_sim_rsync_dir = f"{remote_sim_dir}/rsyncdir/"
|
||||||
|
run(f"mkdir -p {remote_sim_rsync_dir}")
|
||||||
|
|
||||||
|
# only use the collateral from 1 driver (no need to copy all things)
|
||||||
|
assert len(self.parent_node.sim_slots) > 0
|
||||||
|
serv = self.parent_node.sim_slots[0]
|
||||||
|
|
||||||
|
files_to_copy = serv.get_required_files_local_paths()
|
||||||
|
|
||||||
|
# Append required URI paths to the end of this list
|
||||||
|
hwcfg = serv.get_resolved_server_hardware_config()
|
||||||
|
files_to_copy.extend(hwcfg.get_local_uri_paths(uridir))
|
||||||
|
|
||||||
|
for local_path, remote_path in files_to_copy:
|
||||||
|
# -z --inplace
|
||||||
|
rsync_cap = rsync_project(local_dir=local_path, remote_dir=pjoin(remote_sim_rsync_dir, remote_path),
|
||||||
|
ssh_opts="-o StrictHostKeyChecking=no", extra_opts="-L", capture=True)
|
||||||
|
rootLogger.debug(rsync_cap)
|
||||||
|
rootLogger.debug(rsync_cap.stderr)
|
||||||
|
|
||||||
|
run(f"cp -r {remote_sim_rsync_dir}/* {remote_sim_dir}/", shell=True)
|
||||||
|
|
||||||
|
rsync_cap = rsync_project(
|
||||||
|
local_dir=f'../platforms/{self.PLATFORM_NAME}/scripts',
|
||||||
|
remote_dir=remote_sim_dir + "/",
|
||||||
|
ssh_opts="-o StrictHostKeyChecking=no",
|
||||||
|
extra_opts="-L -p",
|
||||||
|
capture=True)
|
||||||
|
rootLogger.debug(rsync_cap)
|
||||||
|
rootLogger.debug(rsync_cap.stderr)
|
||||||
|
|
||||||
|
bitstream_tar = hwcfg.get_bitstream_tar_filename()
|
||||||
|
bitstream_tar_unpack_dir = f"{remote_sim_dir}/{self.PLATFORM_NAME}"
|
||||||
|
bitstream = f"{remote_sim_dir}/{self.PLATFORM_NAME}/firesim.bit"
|
||||||
|
|
||||||
|
with cd(remote_sim_dir):
|
||||||
|
run(f"tar -xf {hwcfg.get_driver_tar_filename()}")
|
||||||
|
|
||||||
|
# at this point the tar file is in the sim slot
|
||||||
|
run(f"rm -rf {bitstream_tar_unpack_dir}")
|
||||||
|
run(f"tar xvf {remote_sim_dir}/{bitstream_tar} -C {remote_sim_dir}")
|
||||||
|
|
||||||
|
driver = f"{remote_sim_dir}/FireSim-{self.PLATFORM_NAME}"
|
||||||
|
|
||||||
|
with cd(remote_sim_dir):
|
||||||
|
run(f"""./scripts/generate-fpga-db.py --bitstream {bitstream} --driver {driver} --out-db-json {json}""")
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" Handle fpga setup for this platform. """
|
||||||
|
|
||||||
|
if self.instance_assigned_simulations():
|
||||||
|
# This is a sim-host node.
|
||||||
|
|
||||||
|
# load xdma driver
|
||||||
|
self.load_xdma()
|
||||||
|
|
||||||
|
# run the passes
|
||||||
|
self.create_fpga_database(uridir)
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
""" XilinxAlveoInstanceDeployManager machines cannot be terminated. """
|
""" XilinxAlveoInstanceDeployManager machines cannot be terminated. """
|
||||||
return
|
return
|
||||||
|
@ -826,10 +899,7 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
assert slotno < len(self.parent_node.sim_slots), f"{slotno} can not index into sim_slots {len(self.parent_node.sim_slots)} on {self.parent_node.host}"
|
assert slotno < len(self.parent_node.sim_slots), f"{slotno} can not index into sim_slots {len(self.parent_node.sim_slots)} on {self.parent_node.host}"
|
||||||
server = self.parent_node.sim_slots[slotno]
|
server = self.parent_node.sim_slots[slotno]
|
||||||
|
|
||||||
self.instance_logger(f"""Determine BDF for {slotno}""")
|
bdf = self.slot_to_bdf(slotno)
|
||||||
collect = run('lspci | grep -i serial.*xilinx')
|
|
||||||
bdfs = [ i[:2] for i in collect.splitlines() if len(i.strip()) >= 0 ]
|
|
||||||
bdf = bdfs[slotno]
|
|
||||||
|
|
||||||
# make the local job results dir for this sim slot
|
# make the local job results dir for this sim slot
|
||||||
server.mkdir_and_prep_local_job_results_dir()
|
server.mkdir_and_prep_local_job_results_dir()
|
||||||
|
@ -844,19 +914,16 @@ class XilinxAlveoU250InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = "xilinx_alveo_u250"
|
self.PLATFORM_NAME = "xilinx_alveo_u250"
|
||||||
self.BOARD_NAME = "au250"
|
|
||||||
|
|
||||||
class XilinxAlveoU280InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
class XilinxAlveoU280InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = "xilinx_alveo_u280"
|
self.PLATFORM_NAME = "xilinx_alveo_u280"
|
||||||
self.BOARD_NAME = "au280"
|
|
||||||
|
|
||||||
class XilinxVCU118InstanceDeployManager(InstanceDeployManager):
|
class XilinxVCU118InstanceDeployManager(InstanceDeployManager):
|
||||||
""" This class manages a Xilinx VCU118-enabled instance using the
|
""" This class manages a Xilinx VCU118-enabled instance using the
|
||||||
garnet shell. """
|
garnet shell. """
|
||||||
PLATFORM_NAME: Optional[str]
|
PLATFORM_NAME: Optional[str]
|
||||||
BOARD_NAME: Optional[str]
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def sim_command_requires_sudo(cls) -> bool:
|
def sim_command_requires_sudo(cls) -> bool:
|
||||||
|
@ -866,29 +933,27 @@ class XilinxVCU118InstanceDeployManager(InstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = "xilinx_vcu118"
|
self.PLATFORM_NAME = "xilinx_vcu118"
|
||||||
self.BOARD_NAME = "xilinx_vcu118"
|
|
||||||
|
|
||||||
def unload_xdma(self) -> None:
|
|
||||||
""" unload the xdma and xvsec kernel modules. """
|
|
||||||
if self.instance_assigned_simulations():
|
|
||||||
self.instance_logger("Unloading XDMA Driver Kernel Module.")
|
|
||||||
|
|
||||||
with warn_only():
|
|
||||||
remote_kmsg("removing_xdma_xvsec_start")
|
|
||||||
run('sudo rmmod xdma')
|
|
||||||
run('sudo rmmod xvsec')
|
|
||||||
remote_kmsg("removing_xdma_xvsec_end")
|
|
||||||
|
|
||||||
def load_xdma(self) -> None:
|
def load_xdma(self) -> None:
|
||||||
""" load the xdma and xvsec kernel modules. """
|
""" load the xdma kernel module. """
|
||||||
if self.instance_assigned_simulations():
|
if self.instance_assigned_simulations():
|
||||||
# unload first
|
# load xdma if unloaded
|
||||||
self.unload_xdma()
|
if run('lsmod | grep -wq xdma', warn_only=True).return_code != 0:
|
||||||
# load xdma
|
|
||||||
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
||||||
# must be installed to this path on sim. machine
|
# must be installed to this path on sim. machine
|
||||||
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
||||||
|
else:
|
||||||
|
self.instance_logger("XDMA Driver Kernel Module already loaded.")
|
||||||
|
|
||||||
|
def load_xvsec(self) -> None:
|
||||||
|
""" load the xvsec kernel modules. """
|
||||||
|
if self.instance_assigned_simulations():
|
||||||
|
if run('lsmod | grep -wq xvsec', warn_only=True).return_code != 0:
|
||||||
|
self.instance_logger("Loading XVSEC Driver Kernel Module.")
|
||||||
|
# must be installed to this path on sim. machine
|
||||||
run(f"sudo insmod /lib/modules/$(uname -r)/updates/kernel/drivers/xvsec/xvsec.ko", shell=True)
|
run(f"sudo insmod /lib/modules/$(uname -r)/updates/kernel/drivers/xvsec/xvsec.ko", shell=True)
|
||||||
|
else:
|
||||||
|
self.instance_logger("XVSEC Driver Kernel Module already loaded.")
|
||||||
|
|
||||||
def flash_fpgas(self) -> None:
|
def flash_fpgas(self) -> None:
|
||||||
if self.instance_assigned_simulations():
|
if self.instance_assigned_simulations():
|
||||||
|
@ -936,6 +1001,7 @@ class XilinxVCU118InstanceDeployManager(InstanceDeployManager):
|
||||||
if not metasim_enabled:
|
if not metasim_enabled:
|
||||||
# load xdma driver
|
# load xdma driver
|
||||||
self.load_xdma()
|
self.load_xdma()
|
||||||
|
self.load_xvsec()
|
||||||
# flash fpgas
|
# flash fpgas
|
||||||
self.flash_fpgas()
|
self.flash_fpgas()
|
||||||
|
|
||||||
|
@ -944,6 +1010,10 @@ class XilinxVCU118InstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" FPGAs are enumerated already with VCU118's """
|
||||||
|
return
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
""" XilinxVCU118InstanceDeployManager machines cannot be terminated. """
|
""" XilinxVCU118InstanceDeployManager machines cannot be terminated. """
|
||||||
return
|
return
|
||||||
|
|
|
@ -918,6 +918,11 @@ class RuntimeConfig:
|
||||||
""" directly called by top-level builddriver command. """
|
""" directly called by top-level builddriver command. """
|
||||||
self.firesim_topology_with_passes.build_driver_passes()
|
self.firesim_topology_with_passes.build_driver_passes()
|
||||||
|
|
||||||
|
def enumerate_fpgas(self) -> None:
|
||||||
|
""" directly called by top-level enumeratefpgas command. """
|
||||||
|
use_mock_instances_for_testing = False
|
||||||
|
self.firesim_topology_with_passes.enumerate_fpgas_passes(use_mock_instances_for_testing)
|
||||||
|
|
||||||
def boot(self) -> None:
|
def boot(self) -> None:
|
||||||
""" directly called by top-level boot command. """
|
""" directly called by top-level boot command. """
|
||||||
use_mock_instances_for_testing = False
|
use_mock_instances_for_testing = False
|
||||||
|
|
|
@ -5,12 +5,12 @@ usage: firesim [-h] [-c RUNTIMECONFIGFILE] [-b BUILDCONFIGFILE]
|
||||||
[-m TERMINATESOMEM416] [--terminatesome TERMINATESOME] [-q]
|
[-m TERMINATESOMEM416] [--terminatesome TERMINATESOME] [-q]
|
||||||
[-t LAUNCHTIME]
|
[-t LAUNCHTIME]
|
||||||
[--platform {f1,vitis,xilinx_alveo_u250,xilinx_alveo_u280,xilinx_vcu118}]
|
[--platform {f1,vitis,xilinx_alveo_u250,xilinx_alveo_u280,xilinx_vcu118}]
|
||||||
{managerinit,infrasetup,boot,kill,runworkload,buildbitstream,builddriver,tar2afi,runcheck,launchrunfarm,terminaterunfarm,shareagfi}
|
{managerinit,infrasetup,boot,kill,runworkload,buildbitstream,builddriver,enumeratefpgas,tar2afi,runcheck,launchrunfarm,terminaterunfarm,shareagfi}
|
||||||
|
|
||||||
FireSim Simulation Manager.
|
FireSim Simulation Manager.
|
||||||
|
|
||||||
positional arguments:
|
positional arguments:
|
||||||
{managerinit,infrasetup,boot,kill,runworkload,buildbitstream,builddriver,tar2afi,runcheck,launchrunfarm,terminaterunfarm,shareagfi}
|
{managerinit,infrasetup,boot,kill,runworkload,buildbitstream,builddriver,enumeratefpgas,tar2afi,runcheck,launchrunfarm,terminaterunfarm,shareagfi}
|
||||||
Management task to run.
|
Management task to run.
|
||||||
|
|
||||||
options:
|
options:
|
||||||
|
|
|
@ -0,0 +1,97 @@
|
||||||
|
FPGA Board Setup
|
||||||
|
===================
|
||||||
|
|
||||||
|
FPGA Setup
|
||||||
|
----------
|
||||||
|
|
||||||
|
.. warning:: Currently, FireSim only supports a single type of FPGA (i.e only |fpga_name| FPGAs) installed on a machine.
|
||||||
|
This includes not mixing the use of Xilinx Vitis/XRT-enabled FPGAs on the system.
|
||||||
|
|
||||||
|
.. Warning:: Power-users can skip this setup and just create the database file listed below by hand if you want to target specific fpgas.
|
||||||
|
|
||||||
|
We need to flash the |fpga_name| FPGA(s) with a dummy XDMA-enabled design and determine the PCI-e ID (or BDF) associated with the serial number of the FPGA.
|
||||||
|
First, we need to flash the FPGA's with the dummy XDMA-enabled design so that the PCI-e subsystem can be initially configured.
|
||||||
|
Afterwards, we will generate the mapping from FPGA serial number to BDF.
|
||||||
|
We provide a a set of scripts to do this.
|
||||||
|
|
||||||
|
First lets obtain the sample bitstream, let's find the URL to download the file to the machine with the FPGA.
|
||||||
|
Below find the HWDB entry called |hwdb_entry_name|.
|
||||||
|
|
||||||
|
.. literalinclude:: /../deploy/sample-backup-configs/sample_config_hwdb.yaml
|
||||||
|
:language: yaml
|
||||||
|
:start-after: DOCREF START: Xilinx Alveo HWDB Entries
|
||||||
|
:end-before: DOCREF END: Xilinx Alveo HWDB Entries
|
||||||
|
|
||||||
|
Look for the ``bitstream_tar: <URL>`` line within |hwdb_entry_name| and keep note of the URL.
|
||||||
|
We will replace the ``BITSTREAM_TAR`` bash variable below with that URL.
|
||||||
|
|
||||||
|
Next, lets flash all FPGAs in the system with the dummy bitstream.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
:substitutions:
|
||||||
|
|
||||||
|
# enter the firesim directory checked out
|
||||||
|
cd firesim
|
||||||
|
|
||||||
|
cd platforms/|platform_name|/scripts
|
||||||
|
|
||||||
|
vivado -mode tcl -source get_serial_dev_for_fpgas.tcl
|
||||||
|
# get the UID/serial number's from this script
|
||||||
|
|
||||||
|
BITSTREAM_TAR=<# replace me!>
|
||||||
|
tar xvf $BITSTREAM_TAR
|
||||||
|
./fpga-util.py --serial $SERIAL_NO |platform_name|/*.bit
|
||||||
|
|
||||||
|
Next, **warm reboot** the computer.
|
||||||
|
This will reconfigure your PCI-E settings such that FireSim can detect the XDMA-enabled bitstream.
|
||||||
|
After the machine is rebooted, you may need to re-insert the XDMA kernel module.
|
||||||
|
Then verify that you can see the XDMA module with:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
lsmod | grep -i xdma
|
||||||
|
|
||||||
|
Also, verify that the FPGA programming worked by looking at the ``lspci`` output.
|
||||||
|
For example, we should see ``Serial controller`` for BDF's that were flashed.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
lspci | grep -i xilinx
|
||||||
|
|
||||||
|
# example output
|
||||||
|
04:00.0 Serial controller: Xilinx Corporation Device 903f (rev ff)
|
||||||
|
83:00.0 Serial controller: Xilinx Corporation Device 903f (rev ff)
|
||||||
|
|
||||||
|
.. Warning:: Anytime the host computer is rebooted you may need to re-run parts of the setup process (i.e. re-insert XDMA kernel module).
|
||||||
|
Before continuing to FireSim simulations after a host computer reboot, ensure that ``cat /proc/devices | grep xdma`` command is successful.
|
||||||
|
Also ensure that you see ``Serial controller`` for the BDF of the FPGA you would like to use in ``lspci | grep -i xilinx`` (otherwise, re-run this setup).
|
||||||
|
|
||||||
|
Next, let's generate the mapping from FPGA serial numbers to the BDF.
|
||||||
|
Re-enter the FireSim repository and run the following commands to re-setup the repo after reboot.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
:substitutions:
|
||||||
|
|
||||||
|
cd firesim
|
||||||
|
|
||||||
|
# rerunning this since the machine rebooted
|
||||||
|
source sourceme-f1-manager.sh --skip-ssh-setup
|
||||||
|
|
||||||
|
Next, open up the ``deploy/config_runtime.yaml`` file and replace the following keys to be the following:
|
||||||
|
|
||||||
|
* ``default_platform`` should be |deploy_manager_code|
|
||||||
|
|
||||||
|
* ``default_simulation_dir`` should point to a temporary simulation directory of your choice
|
||||||
|
|
||||||
|
* ``default_hw_config`` should be |hwdb_entry_name|
|
||||||
|
|
||||||
|
Then, run the following command to generate a mapping from a PCI-E BDF to FPGA UID/serial number.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
:substitutions:
|
||||||
|
|
||||||
|
firesim enumeratefpgas
|
||||||
|
|
||||||
|
This will generate a database file in ``/opt/firesim-db.json`` that has this mapping.
|
||||||
|
|
||||||
|
Now you're ready to continue with other FireSim setup!
|
|
@ -1,4 +1,4 @@
|
||||||
FPGA and Tool Setup
|
FPGA Software Setup
|
||||||
===================
|
===================
|
||||||
|
|
||||||
Requirements and Installations
|
Requirements and Installations
|
||||||
|
@ -21,13 +21,11 @@ We require the following programs/packages installed from the Xilinx website in
|
||||||
Importantly, using this FPGA with FireSim requires that you have ``sudo`` **passwordless** access to the machine with the FPGA.
|
Importantly, using this FPGA with FireSim requires that you have ``sudo`` **passwordless** access to the machine with the FPGA.
|
||||||
This is needed to flash the FPGA bitstream onto the FPGA.
|
This is needed to flash the FPGA bitstream onto the FPGA.
|
||||||
|
|
||||||
FPGA Setup
|
XDMA Setup
|
||||||
----------
|
----------
|
||||||
|
|
||||||
.. warning:: Currently, FireSim only supports a single |fpga_name| installed on a machine. Future support will address this.
|
To communicate with the FPGA over PCI-e, we need to install the Xilinx XDMA kernel module.
|
||||||
|
First, lets install the XDMA kernel module into a FireSim-known location:
|
||||||
After installing the |fpga_name| using the Xilinx instructions and installing the specific version of Vivado, we need to flash the |fpga_name| with a dummy XDMA-enabled design to finish setup.
|
|
||||||
First, lets install the XDMA kernel module in a FireSim known location:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
|
@ -46,81 +44,13 @@ Next, lets add the kernel module:
|
||||||
# by the `make install` previously run
|
# by the `make install` previously run
|
||||||
sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1
|
sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1
|
||||||
|
|
||||||
Next, let's determine the BDF's (unique ID) of the/any FPGA you want to use with FireSim.
|
By default, FireSim will refer to this location to check if the XDMA driver is loaded.
|
||||||
|
Verify that you can see the XDMA module with:
|
||||||
.. code-block:: bash
|
|
||||||
:substitutions:
|
|
||||||
|
|
||||||
# determine BDF of FPGA that you want to use / re-flash
|
|
||||||
lspci | grep -i xilinx
|
|
||||||
|
|
||||||
# example output of a 2 |fpga_name| FPGA system:
|
|
||||||
# 04:00.0 Processing accelerators: Xilinx Corporation Device 5004
|
|
||||||
# 04:00.1 Processing accelerators: Xilinx Corporation Device 5005
|
|
||||||
# 83:00.0 Processing accelerators: Xilinx Corporation Device 5004
|
|
||||||
# 83:00.1 Processing accelerators: Xilinx Corporation Device 5005
|
|
||||||
|
|
||||||
# BDF would be 04:00.0 if you want to flash the '04' FPGA
|
|
||||||
# the extended BDF would be 0000: + the BDF from before (i.e. 0000:04:00.0)
|
|
||||||
# note: that you BDF to use is the one ending in .0
|
|
||||||
|
|
||||||
Keep note of the **extended BDF** of the FPGA you would like to setup.
|
|
||||||
Next, let's flash each |fpga_name| that you would like to use with the dummy bitstream.
|
|
||||||
To obtain the sample bitstream, let's find the URL to download the file to the machine with the FPGA.
|
|
||||||
Below find the HWDB entry called |hwdb_entry_name|.
|
|
||||||
|
|
||||||
.. literalinclude:: /../deploy/sample-backup-configs/sample_config_hwdb.yaml
|
|
||||||
:language: yaml
|
|
||||||
:start-after: DOCREF START: Xilinx Alveo HWDB Entries
|
|
||||||
:end-before: DOCREF END: Xilinx Alveo HWDB Entries
|
|
||||||
|
|
||||||
Look for the ``bitstream_tar: <URL>`` line within |hwdb_entry_name| and keep note of the URL.
|
|
||||||
Next, we will do the following for each FPGA that will be used with FireSim.
|
|
||||||
|
|
||||||
#. Create a temporary flashing area that we will delete after flashing the FPGA.
|
|
||||||
#. Download the bitstream file.
|
|
||||||
#. Download a temporary FireSim repository to have access to the flashing scripts.
|
|
||||||
#. Flash the FPGA (with the extended BDF obtained) and the bitstream file.
|
|
||||||
#. Delete the temporary flashing area.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
:substitutions:
|
|
||||||
|
|
||||||
mkdir /tmp/tempdownload
|
|
||||||
cd /tmp/tempdownload
|
|
||||||
wget <BIT_TAR URL SAVED FROM PREVIOUSLY>
|
|
||||||
tar xvf firesim.tar.gz
|
|
||||||
cd |platform_name|
|
|
||||||
|
|
||||||
git clone --branch |overall_version| https://github.com/firesim/firesim
|
|
||||||
EXTENDED_DEVICE_BDF1=<YOUR BDF HERE> ./firesim/platforms/|platform_name|/scripts/program_fpga.sh ./firesim.bit |board_name|
|
|
||||||
|
|
||||||
rm -rf /tmp/tempdownload
|
|
||||||
|
|
||||||
Next, **warm reboot** the computer.
|
|
||||||
This will reconfigure your PCI-E settings such that FireSim can detect the XDMA-enabled bitstream.
|
|
||||||
After the machine is rebooted, you may need to re-insert the XDMA kernel module.
|
|
||||||
Then verify that you can see the XDMA module with:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
cat /proc/devices | grep xdma
|
lsmod | grep -wq xdma
|
||||||
|
|
||||||
Also, verify that the FPGA programming worked by seeing if the ``lspci`` output has changed.
|
.. warning:: After the machine is rebooted, you may need to re-insert the XDMA kernel module.
|
||||||
For example, we should see ``Serial controller`` for BDF's that were flashed.
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
lspci | grep -i xilinx
|
|
||||||
|
|
||||||
# example output if only the 0000:04:00.0 FPGA was programmed
|
|
||||||
04:00.0 Serial controller: Xilinx Corporation Device 903f (rev ff)
|
|
||||||
83:00.0 Processing accelerators: Xilinx Corporation Device 5004
|
|
||||||
83:00.1 Processing accelerators: Xilinx Corporation Device 5005
|
|
||||||
|
|
||||||
.. Warning:: Anytime the host computer is rebooted you may need to re-run parts of the setup process (i.e. re-insert XDMA kernel module).
|
|
||||||
Before continuing to FireSim simulations after a host computer reboot, ensure that the previously mentioned ``cat /proc/devices | grep xdma`` command is successful.
|
|
||||||
Also ensure that you see ``Serial controller`` for the BDF of the FPGA you would like to use (otherwise, re-run this setup).
|
|
||||||
|
|
||||||
Now you're ready to continue with other FireSim setup!
|
Now you're ready to continue with other FireSim setup!
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
.. |fpga_name| replace:: Xilinx Alveo U250
|
||||||
|
.. |hwdb_entry_name| replace:: ``alveo_u250_firesim_rocket_singlecore_no_nic``
|
||||||
|
.. |platform_name| replace:: xilinx_alveo_u250
|
||||||
|
.. |board_name| replace:: au250
|
||||||
|
.. |deploy_manager_code| replace:: ``XilinxAlveoU250InstanceDeployManager``
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Template-Part2.rst
|
|
@ -0,0 +1,7 @@
|
||||||
|
.. |fpga_name| replace:: Xilinx Alveo U280
|
||||||
|
.. |hwdb_entry_name| replace:: ``alveo_u280_firesim_rocket_singlecore_no_nic``
|
||||||
|
.. |platform_name| replace:: xilinx_alveo_u280
|
||||||
|
.. |board_name| replace:: au280
|
||||||
|
.. |deploy_manager_code| replace:: ``XilinxAlveoU280InstanceDeployManager``
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Template-Part2.rst
|
|
@ -158,4 +158,4 @@ To run it, do the following:
|
||||||
It will create initial configuration files, which we will edit in later
|
It will create initial configuration files, which we will edit in later
|
||||||
sections.
|
sections.
|
||||||
|
|
||||||
Now you're ready to launch FireSim simulations! Hit Next to learn how to run single-node simulations.
|
Hit Next to continue with the guide.
|
||||||
|
|
|
@ -14,23 +14,3 @@ After you complete these tutorials, you can look at the "Advanced Docs"
|
||||||
in the sidebar to the left.
|
in the sidebar to the left.
|
||||||
|
|
||||||
Here's a high-level outline of what we'll be doing in our tutorials:
|
Here's a high-level outline of what we'll be doing in our tutorials:
|
||||||
|
|
||||||
#. **FPGA Setup**: Installing the FPGA board and relevant software.
|
|
||||||
|
|
||||||
#. **On-Premises Machine Setup**
|
|
||||||
|
|
||||||
#. Setting up a "Manager Machine" from which you will coordinate building
|
|
||||||
and deploying simulations locally.
|
|
||||||
|
|
||||||
#. **Single-node simulation tutorial**: This tutorial guides you through the
|
|
||||||
process of running one simulation locally consisting of a single
|
|
||||||
|fpga_name|, using our pre-built public FireSim |bit_type| bitstream.
|
|
||||||
|
|
||||||
#. **Building your own hardware designs tutorial (Chisel to FPGA Image)**:
|
|
||||||
This tutorial guides you through the full process of taking Rocket Chip RTL
|
|
||||||
and any custom RTL plugged into Rocket Chip and producing a FireSim bitstream
|
|
||||||
to plug into your simulations. This automatically runs Chisel elaboration,
|
|
||||||
FAME-1 Transformation, and the |build_type| FPGA flow.
|
|
||||||
|
|
||||||
Generally speaking, you only need to follow Step 4 if you're modifying Chisel
|
|
||||||
RTL or changing non-runtime configurable hardware parameters.
|
|
||||||
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
#. **FPGA Software Setup**: Installing the relevant FPGA software.
|
||||||
|
|
||||||
|
#. **On-Premises Machine Setup**
|
||||||
|
|
||||||
|
#. Setting up a "Manager Machine" from which you will coordinate building
|
||||||
|
and deploying simulations locally.
|
||||||
|
|
||||||
|
#. **FPGA Board Setup**: Finish initial programming/setting up FPGA boards.
|
||||||
|
|
||||||
|
#. **Single-node simulation tutorial**: This tutorial guides you through the
|
||||||
|
process of running one simulation locally consisting of a single
|
||||||
|
|fpga_name|, using our pre-built public FireSim |bit_type| bitstream.
|
||||||
|
|
||||||
|
#. **Building your own hardware designs tutorial (Chisel to FPGA Image)**:
|
||||||
|
This tutorial guides you through the full process of taking Rocket Chip RTL
|
||||||
|
and any custom RTL plugged into Rocket Chip and producing a FireSim bitstream
|
||||||
|
to plug into your simulations. This automatically runs Chisel elaboration,
|
||||||
|
FAME-1 Transformation, and the |build_type| FPGA flow.
|
||||||
|
|
||||||
|
Generally speaking, you only need to follow Step 5 if you're modifying Chisel
|
||||||
|
RTL or changing non-runtime configurable hardware parameters.
|
|
@ -4,10 +4,13 @@
|
||||||
|
|
||||||
.. include:: Intro-Template.rst
|
.. include:: Intro-Template.rst
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Outline-Template.rst
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
FPGA-Setup/Xilinx-Alveo-U250
|
FPGA-Setup/Xilinx-Alveo-U250
|
||||||
Initial-Setup/Setting-Up-Xilinx-Alveo-U250
|
Initial-Setup/Setting-Up-Xilinx-Alveo-U250
|
||||||
|
FPGA-Setup/Xilinx-Alveo-U250-Part2
|
||||||
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U250
|
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U250
|
||||||
Building-a-FireSim-Bitstream/Xilinx-Alveo-U250
|
Building-a-FireSim-Bitstream/Xilinx-Alveo-U250
|
||||||
|
|
|
@ -4,10 +4,13 @@
|
||||||
|
|
||||||
.. include:: Intro-Template.rst
|
.. include:: Intro-Template.rst
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Outline-Template.rst
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
FPGA-Setup/Xilinx-Alveo-U280
|
FPGA-Setup/Xilinx-Alveo-U280
|
||||||
Initial-Setup/Setting-Up-Xilinx-Alveo-U280
|
Initial-Setup/Setting-Up-Xilinx-Alveo-U280
|
||||||
|
FPGA-Setup/Xilinx-Alveo-U280-Part2
|
||||||
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U280
|
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U280
|
||||||
Building-a-FireSim-Bitstream/Xilinx-Alveo-U280
|
Building-a-FireSim-Bitstream/Xilinx-Alveo-U280
|
||||||
|
|
|
@ -4,6 +4,26 @@
|
||||||
|
|
||||||
.. include:: Intro-Template.rst
|
.. include:: Intro-Template.rst
|
||||||
|
|
||||||
|
#. **FPGA Setup**: Installing the FPGA board and relevant software.
|
||||||
|
|
||||||
|
#. **On-Premises Machine Setup**
|
||||||
|
|
||||||
|
#. Setting up a "Manager Machine" from which you will coordinate building
|
||||||
|
and deploying simulations locally.
|
||||||
|
|
||||||
|
#. **Single-node simulation tutorial**: This tutorial guides you through the
|
||||||
|
process of running one simulation locally consisting of a single
|
||||||
|
|fpga_name|, using our pre-built public FireSim |bit_type| bitstream.
|
||||||
|
|
||||||
|
#. **Building your own hardware designs tutorial (Chisel to FPGA Image)**:
|
||||||
|
This tutorial guides you through the full process of taking Rocket Chip RTL
|
||||||
|
and any custom RTL plugged into Rocket Chip and producing a FireSim bitstream
|
||||||
|
to plug into your simulations. This automatically runs Chisel elaboration,
|
||||||
|
FAME-1 Transformation, and the |build_type| FPGA flow.
|
||||||
|
|
||||||
|
Generally speaking, you only need to follow Step 4 if you're modifying Chisel
|
||||||
|
RTL or changing non-runtime configurable hardware parameters.
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
.*.swp
|
.*.swp
|
||||||
.*.swo
|
.*.swo
|
||||||
|
__pycache__
|
||||||
|
|
||||||
cl_*
|
cl_*
|
||||||
!cl_firesim
|
!cl_firesim
|
||||||
|
|
|
@ -21,4 +21,3 @@
|
||||||
set part xcu250-figd2104-2l-e
|
set part xcu250-figd2104-2l-e
|
||||||
set board_part xilinx.com:au250:part0:1.3
|
set board_part xilinx.com:au250:part0:1.3
|
||||||
set zynq_family 0
|
set zynq_family 0
|
||||||
set hw_device xcu250_0
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
../cl_firesim/scripts/au250.tcl
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
BITSTREAM=$SCRATCH_HOME/firesim-private/deploy/results-build/2023-05-12--07-11-08-alveou250_firesim_rocket_singlecore_no_nic_w_pres/cl_xilinx_alveo_u250-firesim-FireSim-FireSimRocketConfig-BaseXilinxAlveoConfig/xilinx_alveo_u250/firesim.bit
|
||||||
|
|
||||||
|
./fpga-util.py --bdf 04:00.0 --disconnect-bdf
|
||||||
|
./fpga-util.py --bdf 83:00.0 --disconnect-bdf
|
||||||
|
|
||||||
|
if lspci | grep -i xilinx; then
|
||||||
|
echo "Something went wrong"
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
echo "Missing them"
|
||||||
|
fi
|
||||||
|
|
||||||
|
./fpga-util.py --serial Xilinx/21320733400EA --bitstream $BITSTREAM
|
||||||
|
./fpga-util.py --serial Xilinx/213207334001A --bitstream $BITSTREAM
|
||||||
|
|
||||||
|
if lspci | grep -i xilinx; then
|
||||||
|
echo "Something went wrong"
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
echo "Missing them"
|
||||||
|
fi
|
||||||
|
|
||||||
|
./fpga-util.py --bdf 04:00.0 --reconnect-bdf
|
||||||
|
./fpga-util.py --bdf 83:00.0 --reconnect-bdf
|
||||||
|
|
||||||
|
if lspci | grep -i xilinx; then
|
||||||
|
echo "Found them"
|
||||||
|
else
|
||||||
|
echo "Something went wrong"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
## from scratch
|
||||||
|
#
|
||||||
|
#./fpga-util.py --bdf 04:00.0 --disconnect-bdf
|
||||||
|
#./fpga-util.py --bdf 04:00.1 --disconnect-bdf
|
||||||
|
#./fpga-util.py --bdf 83:00.0 --disconnect-bdf
|
||||||
|
#./fpga-util.py --bdf 83:00.1 --disconnect-bdf
|
||||||
|
#
|
||||||
|
#./fpga-util.py --serial Xilinx/21320733400EA --bitstream $BITSTREAM
|
||||||
|
#./fpga-util.py --serial Xilinx/213207334001A --bitstream $BITSTREAM
|
||||||
|
#
|
||||||
|
##./fpga-util.py --bdf 04:00.0 --reconnect-bdf
|
||||||
|
##./fpga-util.py --bdf 04:00.1 --reconnect-bdf
|
||||||
|
##./fpga-util.py --bdf 83:00.0 --reconnect-bdf
|
||||||
|
##./fpga-util.py --bdf 83:00.1 --reconnect-bdf
|
|
@ -0,0 +1,4 @@
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
# firesim specific location of where to read/write database file
|
||||||
|
dbPath = Path("/opt/firesim-db.json")
|
|
@ -0,0 +1,194 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import shutil
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import pcielib
|
||||||
|
import util
|
||||||
|
import firesim
|
||||||
|
|
||||||
|
from typing import Dict, Any, List
|
||||||
|
|
||||||
|
scriptPath = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
def program_fpga(vivado: Path, serial: str, bitstream: str) -> None:
|
||||||
|
progTcl = scriptPath / 'program_fpga.tcl'
|
||||||
|
assert progTcl.exists(), f"Unable to find {progTcl}"
|
||||||
|
rc, stdout, stderr = util.call_vivado(
|
||||||
|
vivado,
|
||||||
|
[
|
||||||
|
'-source', str(progTcl),
|
||||||
|
'-tclargs',
|
||||||
|
'-serial', serial,
|
||||||
|
'-bitstream_path', bitstream,
|
||||||
|
]
|
||||||
|
)
|
||||||
|
if rc != 0:
|
||||||
|
sys.exit(f":ERROR: Unable to flash FPGA {serial} with {bitstream}.\nstdout:\n{stdout}\nstderr:\n{stderr}")
|
||||||
|
|
||||||
|
# mapping functions
|
||||||
|
|
||||||
|
def get_fpga_db() -> Dict[Any, Any]:
|
||||||
|
if firesim.dbPath.exists():
|
||||||
|
with open(firesim.dbPath, 'r') as f:
|
||||||
|
return json.load(f)
|
||||||
|
else:
|
||||||
|
print(f":ERROR: Unable to open {firesim.dbPath}. Does it exist? Did you run 'firesim enumeratefpgas'?", file=sys.stderr)
|
||||||
|
sys.exit(f":ERROR: Unable to create FPGA database from {firesim.dbPath}")
|
||||||
|
|
||||||
|
def get_serial_from_bus_id(id: str) -> str:
|
||||||
|
deviceBDF = pcielib.get_bdf_from_extended_bdf(pcielib.get_singular_device_extended_bdf(id))
|
||||||
|
for e in get_fpga_db():
|
||||||
|
if deviceBDF == e['bdf']:
|
||||||
|
return e['uid']
|
||||||
|
sys.exit(":ERROR: Unable to get serial number from bus id")
|
||||||
|
|
||||||
|
def get_serials() -> List[str]:
|
||||||
|
serials = []
|
||||||
|
for e in get_fpga_db():
|
||||||
|
serials.append(e['uid'])
|
||||||
|
return serials
|
||||||
|
|
||||||
|
def get_extended_bdfs() -> List[str]:
|
||||||
|
bdfs = []
|
||||||
|
for e in get_fpga_db():
|
||||||
|
bdfs.append(pcielib.get_extended_bdf_from_bdf(e['bdfs']))
|
||||||
|
return bdfs
|
||||||
|
|
||||||
|
# main
|
||||||
|
|
||||||
|
def main(args: List[str]) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Program/manipulate a Xilinx XDMA-enabled FPGA device")
|
||||||
|
megroup = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
megroup.add_argument("--bus_id", help="Bus number of FPGA (i.e. ****:<THIS>:**.*)")
|
||||||
|
megroup.add_argument("--bdf", help="BDF of FPGA (i.e. ****:<THIS>)")
|
||||||
|
megroup.add_argument("--extended-bdf", help="Extended BDF of FPGA (i.e. all of this - ****:**:**.*)")
|
||||||
|
megroup.add_argument("--serial", help="Serial number of FPGA (i.e. what 'get_hw_target' shows in Vivado)")
|
||||||
|
megroup.add_argument("--all-serials", help="Use all serial numbers (no PCI-E manipulation)", action="store_true")
|
||||||
|
megroup.add_argument("--all-bdfs", help="Use all BDFs (PCI-E manipulation)", action="store_true")
|
||||||
|
parser.add_argument("--vivado-bin", help="Explicit path to 'vivado'", type=Path)
|
||||||
|
parser.add_argument("--hw-server-bin", help="Explicit path to 'hw_server'", type=Path)
|
||||||
|
megroup2 = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
megroup2.add_argument("--bitstream", help="The bitstream to flash onto FPGA(s)", type=Path)
|
||||||
|
megroup2.add_argument("--disconnect-bdf", help="Disconnect BDF(s)", action="store_true")
|
||||||
|
megroup2.add_argument("--reconnect-bdf", help="Reconnect BDF(s)", action="store_true")
|
||||||
|
parsed_args = parser.parse_args(args)
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
parsed_args.hw_server_bin = shutil.which('hw_server')
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
parsed_args.vivado_bin = shutil.which('vivado')
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Hardware Server!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Vivado!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
parsed_args.vivado_bin = Path(parsed_args.vivado_bin).absolute()
|
||||||
|
parsed_args.hw_server_bin = Path(parsed_args.hw_server_bin).absolute()
|
||||||
|
|
||||||
|
eUserId = os.geteuid()
|
||||||
|
sudoUserId = os.getenv('SUDO_UID')
|
||||||
|
isAdmin = (eUserId == 0) and (sudoUserId is None)
|
||||||
|
userId = eUserId if sudoUserId is None else int(sudoUserId)
|
||||||
|
|
||||||
|
# if not sudoer, spawn w/ sudo
|
||||||
|
if eUserId != 0:
|
||||||
|
execvArgs = ['/usr/bin/sudo', str(Path(__file__).absolute())] + sys.argv[1:]
|
||||||
|
execvArgs += ['--vivado-bin', str(parsed_args.vivado_bin), '--hw-server-bin', str(parsed_args.hw_server_bin)]
|
||||||
|
print(f":INFO: Running: {execvArgs}")
|
||||||
|
os.execv(execvArgs[0], execvArgs)
|
||||||
|
|
||||||
|
def is_bdf_arg(parsed_args) -> bool:
|
||||||
|
return parsed_args.bus_id or parsed_args.bdf or parsed_args.extended_bdf or parsed_args.all_bdfs
|
||||||
|
|
||||||
|
def get_bus_ids_from_args(parsed_args) -> List[str]:
|
||||||
|
bus_ids = []
|
||||||
|
if parsed_args.bus_id:
|
||||||
|
bus_ids.append(parsed_args.bus_id)
|
||||||
|
if parsed_args.bdf:
|
||||||
|
bus_ids.append(pcielib.get_bus_id_from_extended_bdf(pcielib.get_extended_bdf_from_bdf(parsed_args.bdf)))
|
||||||
|
if parsed_args.extended_bdf:
|
||||||
|
bus_ids.append(pcielib.get_bus_id_from_extended_bdf(parsed_args.extended_bdf))
|
||||||
|
if parsed_args.all_bdfs:
|
||||||
|
bus_ids.extend([pcielib.get_bus_id_from_extended_bdf(bdf) for bdf in get_extended_bdfs()])
|
||||||
|
return bus_ids
|
||||||
|
|
||||||
|
def disconnect_bus_id(bus_id: str) -> None:
|
||||||
|
pcielib.clear_serr_bits(bus_id)
|
||||||
|
pcielib.clear_fatal_error_reporting_bits(bus_id)
|
||||||
|
pcielib.remove(bus_id)
|
||||||
|
assert not pcielib.any_device_exists(bus_id), f"{bus_id} still visible. Check for proper removal."
|
||||||
|
|
||||||
|
def reconnect_bus_id(bus_id: str) -> None:
|
||||||
|
pcielib.rescan(bus_id)
|
||||||
|
pcielib.enable_memmapped_transfers(bus_id)
|
||||||
|
assert pcielib.any_device_exists(bus_id), f"{bus_id} not visible. Check for proper rescan."
|
||||||
|
|
||||||
|
# program based on bitstream
|
||||||
|
if parsed_args.bitstream is not None:
|
||||||
|
if not parsed_args.bitstream.is_file() or not parsed_args.bitstream.exists():
|
||||||
|
sys.exit(f":ERROR: Invalid bitstream: {parsed_args.bitstream}")
|
||||||
|
else:
|
||||||
|
parsed_args.bitstream = parsed_args.bitstream.absolute()
|
||||||
|
|
||||||
|
if is_bdf_arg(parsed_args):
|
||||||
|
bus_ids = get_bus_ids_from_args(parsed_args)
|
||||||
|
|
||||||
|
# must be called before the remove otherwise it will not find a serial number
|
||||||
|
serialNums = []
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
serialNums.append(get_serial_from_bus_id(bus_id))
|
||||||
|
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
disconnect_bus_id(bus_id)
|
||||||
|
|
||||||
|
# program fpga(s) separately if doing multiple bdfs
|
||||||
|
for i, bus_id in enumerate(bus_ids):
|
||||||
|
serialNumber = serialNums[i]
|
||||||
|
program_fpga(parsed_args.vivado_bin, serialNumber, parsed_args.bitstream)
|
||||||
|
print(f":INFO: Successfully programmed FPGA {bus_id} with {parsed_args.bitstream}")
|
||||||
|
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
reconnect_bus_id(bus_id)
|
||||||
|
|
||||||
|
if parsed_args.serial or parsed_args.all_serials:
|
||||||
|
serials = []
|
||||||
|
if parsed_args.serial:
|
||||||
|
serials.append(parsed_args.serial)
|
||||||
|
if parsed_args.all_serials:
|
||||||
|
serials.extend(get_serials())
|
||||||
|
|
||||||
|
for serial in serials:
|
||||||
|
program_fpga(parsed_args.vivado_bin, serial, parsed_args.bitstream)
|
||||||
|
print(f":INFO: Successfully programmed FPGA {serial} with {parsed_args.bitstream}")
|
||||||
|
print(":WARNING: Please warm reboot the machine")
|
||||||
|
|
||||||
|
# disconnect bdfs
|
||||||
|
if parsed_args.disconnect_bdf:
|
||||||
|
if is_bdf_arg(parsed_args):
|
||||||
|
bus_ids = get_bus_ids_from_args(parsed_args)
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
disconnect_bus_id(bus_id)
|
||||||
|
else:
|
||||||
|
sys.exit("Must provide a BDF-like argument to disconnect")
|
||||||
|
|
||||||
|
# reconnect bdfs
|
||||||
|
if parsed_args.reconnect_bdf:
|
||||||
|
if is_bdf_arg(parsed_args):
|
||||||
|
bus_ids = get_bus_ids_from_args(parsed_args)
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
reconnect_bus_id(bus_id)
|
||||||
|
else:
|
||||||
|
sys.exit("Must provide a BDF-like argument to disconnect")
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main(sys.argv[1:]))
|
|
@ -0,0 +1,263 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import pwd
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import pcielib
|
||||||
|
import util
|
||||||
|
|
||||||
|
from typing import Optional, Dict, Any, List
|
||||||
|
|
||||||
|
scriptPath = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
def get_bdfs() -> List[str]:
|
||||||
|
pLspci= subprocess.Popen(['lspci'], stdout=subprocess.PIPE)
|
||||||
|
pGrep = subprocess.Popen(['grep', '-i', 'serial.*xilinx'], stdin=pLspci.stdout, stdout=subprocess.PIPE)
|
||||||
|
if pLspci.stdout is not None:
|
||||||
|
pLspci.stdout.close()
|
||||||
|
|
||||||
|
sout, serr = pGrep.communicate()
|
||||||
|
|
||||||
|
if pGrep.returncode != 0:
|
||||||
|
sys.exit(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}")
|
||||||
|
|
||||||
|
outputLines = sout.decode('utf-8').splitlines()
|
||||||
|
bdfs = [ i[:7] for i in outputLines if len(i.strip()) >= 0]
|
||||||
|
return bdfs
|
||||||
|
|
||||||
|
def call_fpga_util(args: List[str]) -> None:
|
||||||
|
progScript = scriptPath / 'fpga-util.py'
|
||||||
|
assert progScript.exists(), f"Unable to find {progScript}"
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
[str(progScript.resolve().absolute())] + args,
|
||||||
|
stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
if pProg.returncode != 0:
|
||||||
|
sys.exit(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}")
|
||||||
|
|
||||||
|
def disconnect_bdf(bdf: str, vivado: str, hw_server: str) -> None:
|
||||||
|
print(f":INFO: Disconnecting BDF: {bdf}")
|
||||||
|
call_fpga_util([
|
||||||
|
"--bdf", bdf,
|
||||||
|
"--disconnect-bdf",
|
||||||
|
"--vivado-bin", vivado,
|
||||||
|
"--hw-server-bin", hw_server,
|
||||||
|
])
|
||||||
|
|
||||||
|
def reconnect_bdf(bdf: str, vivado: str, hw_server: str) -> None:
|
||||||
|
print(f":INFO: Reconnecting BDF: {bdf}")
|
||||||
|
call_fpga_util([
|
||||||
|
"--bdf", bdf,
|
||||||
|
"--reconnect-bdf",
|
||||||
|
"--vivado-bin", vivado,
|
||||||
|
"--hw-server-bin", hw_server,
|
||||||
|
])
|
||||||
|
|
||||||
|
def program_fpga(serial: str, bitstream: str, vivado: str, hw_server: str) -> None:
|
||||||
|
print(f":INFO: Programming {serial} with {bitstream}")
|
||||||
|
call_fpga_util([
|
||||||
|
"--serial_no", serial,
|
||||||
|
"--bitstream", bitstream,
|
||||||
|
"--vivado-bin", vivado,
|
||||||
|
"--hw-server-bin", hw_server,
|
||||||
|
])
|
||||||
|
|
||||||
|
def get_serial_numbers_and_fpga_types(vivado: Path) -> Dict[str, str]:
|
||||||
|
tclScript = scriptPath / 'get_serial_dev_for_fpgas.tcl'
|
||||||
|
assert tclScript.exists(), f"Unable to find {tclScript}"
|
||||||
|
rc, stdout, stderr = util.call_vivado(vivado, ['-source', str(tclScript)])
|
||||||
|
if rc != 0:
|
||||||
|
sys.exit(f":ERROR: It failed with:\nstdout:\n{stdout}\nstderr:\n{stderr}")
|
||||||
|
|
||||||
|
outputLines = stdout.splitlines()
|
||||||
|
relevantLines= [s for s in outputLines if ("hw_dev" in s) or ("hw_uid" in s)]
|
||||||
|
devs = []
|
||||||
|
uids = []
|
||||||
|
|
||||||
|
for line in relevantLines:
|
||||||
|
m = re.match(r"^hw_dev: (.*)$", line)
|
||||||
|
if m:
|
||||||
|
devs.append(m.group(1))
|
||||||
|
|
||||||
|
m = re.match(r"^hw_uid: (.*)$", line)
|
||||||
|
if m:
|
||||||
|
uids.append(m.group(1))
|
||||||
|
|
||||||
|
uid2dev = {}
|
||||||
|
for uid, dev in zip(uids, devs):
|
||||||
|
uid2dev[uid] = dev
|
||||||
|
|
||||||
|
return uid2dev
|
||||||
|
|
||||||
|
def call_driver(bdf: str, driver: Path, args: List[str]) -> int:
|
||||||
|
bus_id = pcielib.get_bus_id_from_extended_bdf(pcielib.get_extended_bdf_from_bdf(bdf))
|
||||||
|
|
||||||
|
driverPath = driver.resolve().absolute()
|
||||||
|
assert driverPath.exists(), f"Unable to find {driverPath}"
|
||||||
|
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
[
|
||||||
|
str(driverPath),
|
||||||
|
"+permissive",
|
||||||
|
f"+slotid={bus_id}",
|
||||||
|
] + args + [
|
||||||
|
"+permissive-off",
|
||||||
|
"+prog0=none",
|
||||||
|
],
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
sout, serr = pProg.communicate(timeout=5)
|
||||||
|
except:
|
||||||
|
# spam any amount of flush signals
|
||||||
|
pProg.send_signal(signal.SIGPIPE)
|
||||||
|
pProg.send_signal(signal.SIGUSR1)
|
||||||
|
pProg.send_signal(signal.SIGUSR2)
|
||||||
|
|
||||||
|
# spam any amount of kill signals
|
||||||
|
pProg.kill()
|
||||||
|
pProg.send_signal(signal.SIGINT)
|
||||||
|
pProg.send_signal(signal.SIGTERM)
|
||||||
|
|
||||||
|
# retrieve flushed output
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
if pProg.returncode == 124 or pProg.returncode is None:
|
||||||
|
sys.exit(":ERROR: Timed out...")
|
||||||
|
elif pProg.returncode != 0:
|
||||||
|
print(f":WARNING: Running the driver failed...", file=sys.stderr)
|
||||||
|
|
||||||
|
print(f":DEBUG: bdf: {bdf} bus_id: {bus_id}\nstdout:\n{sout.decode('utf-8')}")
|
||||||
|
return pProg.returncode
|
||||||
|
|
||||||
|
def run_driver_check_fingerprint(bdf: str, driver: Path) -> int:
|
||||||
|
print(f":INFO: Running check fingerprint driver call with {bdf}")
|
||||||
|
return call_driver(bdf, driver, ["+check-fingerprint"])
|
||||||
|
|
||||||
|
def run_driver_write_fingerprint(bdf: str, driver: Path, write_val: int) -> int:
|
||||||
|
print(f":INFO: Running write fingerprint driver call with {bdf}")
|
||||||
|
# TODO: maybe confirm write went through in the stdout/err?
|
||||||
|
return call_driver(bdf, driver, [f"+write-fingerprint={write_val}"])
|
||||||
|
|
||||||
|
def main(args: List[str]) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="Generate a FireSim json database file")
|
||||||
|
parser.add_argument("--bitstream", help="Bitstream to flash on all Xilinx XDMA-enabled FPGAs (must align with --driver)", type=Path, required=True)
|
||||||
|
parser.add_argument("--driver", help="FireSim driver to test bitstream with (must align with --bitstream)", type=Path, required=True)
|
||||||
|
parser.add_argument("--out-db-json", help="Path to output FireSim database", type=Path, required=True)
|
||||||
|
parser.add_argument("--vivado-bin", help="Explicit path to 'vivado'", type=Path)
|
||||||
|
parser.add_argument("--hw-server-bin", help="Explicit path to 'hw_server'", type=Path)
|
||||||
|
parsed_args = parser.parse_args(args)
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
parsed_args.hw_server_bin = shutil.which('hw_server')
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
parsed_args.vivado_bin = shutil.which('vivado')
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Hardware Server!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Vivado!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
parsed_args.vivado_bin = Path(parsed_args.vivado_bin).absolute()
|
||||||
|
parsed_args.hw_server_bin = Path(parsed_args.hw_server_bin).absolute()
|
||||||
|
|
||||||
|
eUserId = os.geteuid()
|
||||||
|
sudoUserId = os.getenv('SUDO_UID')
|
||||||
|
isAdmin = (eUserId == 0) and (sudoUserId is None)
|
||||||
|
userId = eUserId if sudoUserId is None else int(sudoUserId)
|
||||||
|
|
||||||
|
if eUserId != 0:
|
||||||
|
execvArgs = ['/usr/bin/sudo', str(Path(__file__).absolute())] + sys.argv[1:]
|
||||||
|
execvArgs += ['--vivado-bin', str(parsed_args.vivado_bin), '--hw-server-bin', str(parsed_args.hw_server_bin)]
|
||||||
|
print(f":INFO: Running: {execvArgs}")
|
||||||
|
os.execv(execvArgs[0], execvArgs)
|
||||||
|
|
||||||
|
print(":INFO: This script expects that all Xilinx XDMA-enabled FPGAs are programmed with the same --bitstream arg. by default (through an MCS file for bistream file)")
|
||||||
|
|
||||||
|
# 1. get all serial numbers for all fpgas on the system
|
||||||
|
|
||||||
|
sno2fpga = get_serial_numbers_and_fpga_types(parsed_args.vivado_bin)
|
||||||
|
serials = sno2fpga.keys()
|
||||||
|
bdfs = get_bdfs()
|
||||||
|
|
||||||
|
# 2. program all fpgas so that they are in a known state
|
||||||
|
|
||||||
|
# disconnect all
|
||||||
|
for bdf in bdfs:
|
||||||
|
disconnect_bdf(bdf, str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
for serial in serials:
|
||||||
|
program_fpga(serial, str(parsed_args.working_bitstream.resolve().absolute()), str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
# reconnect all
|
||||||
|
for bdf in bdfs:
|
||||||
|
reconnect_bdf(bdf, str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
serial2BDF: Dict[str, str] = {}
|
||||||
|
|
||||||
|
write_val = 0xDEADBEEF
|
||||||
|
|
||||||
|
# 3. write to all fingerprints based on bdfs
|
||||||
|
|
||||||
|
for bdf in bdfs:
|
||||||
|
run_driver_write_fingerprint(bdf, parsed_args.driver, write_val)
|
||||||
|
|
||||||
|
# 4. create mapping by checking if fingerprint was overridden
|
||||||
|
|
||||||
|
for serial in serials:
|
||||||
|
# disconnect all
|
||||||
|
for bdf in bdfs:
|
||||||
|
disconnect_bdf(bdf, str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
program_fpga(serial, str(parsed_args.working_bitstream.resolve().absolute()), str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
# reconnect all
|
||||||
|
for bdf in bdfs:
|
||||||
|
reconnect_bdf(bdf, str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
# read all fingerprints to find the good one
|
||||||
|
for bdf in bdfs:
|
||||||
|
if not (bdf in serial2BDF.values()):
|
||||||
|
rc = run_driver_check_fingerprint(bdf, parsed_args.driver)
|
||||||
|
if rc == 0:
|
||||||
|
serial2BDF[serial] = bdf
|
||||||
|
break
|
||||||
|
|
||||||
|
if not (serial in serial2BDF):
|
||||||
|
print(f":ERROR: Unable to determine BDF for {serial} FPGA. Something went wrong", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(f":INFO: Mapping: {serial2BDF}")
|
||||||
|
|
||||||
|
finalMap = []
|
||||||
|
for s, b in serial2BDF.items():
|
||||||
|
finalMap.append({
|
||||||
|
"uid" : s,
|
||||||
|
"device" : sno2fpga[s],
|
||||||
|
"bdf" : b
|
||||||
|
})
|
||||||
|
|
||||||
|
with open(parsed_args.out_db_json, 'w') as f:
|
||||||
|
json.dump(finalMap, f, indent=2)
|
||||||
|
|
||||||
|
print(f":INFO: Successfully wrote to {parsed_args.out_db_json}")
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main(sys.argv[1:]))
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Directory variables
|
||||||
|
set script_path [file normalize [info script]]
|
||||||
|
set script_dir [file dirname $script_path]
|
||||||
|
set root_dir [file dirname $script_dir]
|
||||||
|
|
||||||
|
set_param labtools.enable_cs_server false
|
||||||
|
|
||||||
|
open_hw_manager
|
||||||
|
connect_hw_server -allow_non_jtag
|
||||||
|
|
||||||
|
# by default vivado opens a default hw target
|
||||||
|
close_hw_target
|
||||||
|
|
||||||
|
foreach {hw_target} [get_hw_targets] {
|
||||||
|
open_hw_target $hw_target
|
||||||
|
set hw_dev [get_hw_device]
|
||||||
|
set hw_uid [get_property UID $hw_target]
|
||||||
|
puts "hw_dev: $hw_dev"
|
||||||
|
puts "hw_uid: $hw_uid"
|
||||||
|
close_hw_target
|
||||||
|
}
|
||||||
|
|
||||||
|
exit
|
|
@ -0,0 +1,154 @@
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import re
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from typing import Dict, List
|
||||||
|
|
||||||
|
pciDevicesPath = Path('/sys/bus/pci/devices')
|
||||||
|
|
||||||
|
def get_device_paths(bus_id: str) -> List[Path]:
|
||||||
|
result = []
|
||||||
|
for entry in pciDevicesPath.iterdir():
|
||||||
|
if re.match('^0000:' + re.escape(bus_id) + ':[a-fA-F0-9]{2}\.[0-7]$', entry.name):
|
||||||
|
result.append(entry)
|
||||||
|
return result
|
||||||
|
|
||||||
|
def get_device_extended_bdfs(bus_id: str) -> List[str]:
|
||||||
|
return [e.name for e in get_device_paths(bus_id)]
|
||||||
|
|
||||||
|
def get_singular_device_path(bus_id: str) -> Path:
|
||||||
|
devicePaths = get_device_paths(bus_id)
|
||||||
|
if len(devicePaths) == 0:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Device BDF path for {bus_id}")
|
||||||
|
if len(devicePaths) != 1:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Device BDF path for {bus_id} since too many Extended Device BDFs match: {devicePaths}")
|
||||||
|
return devicePaths[0]
|
||||||
|
|
||||||
|
def get_singular_device_extended_bdf(bus_id: str) -> str:
|
||||||
|
deviceBDFs = get_device_extended_bdfs(bus_id)
|
||||||
|
if len(deviceBDFs) == 0:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Device BDF for {bus_id}")
|
||||||
|
if len(deviceBDFs) != 1:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Device BDF for {bus_id} since too many Extended Device BDFs match: {deviceBDFs}")
|
||||||
|
return deviceBDFs[0]
|
||||||
|
|
||||||
|
# obtain bridge paths/bdfs
|
||||||
|
|
||||||
|
def get_bridge_paths(bus_id: str) -> List[Path]:
|
||||||
|
return [e.resolve().absolute().parent for e in get_device_paths(bus_id)]
|
||||||
|
|
||||||
|
def get_bridge_extended_bdfs(bus_id: str) -> List[str]:
|
||||||
|
return [e.name for e in get_bridge_paths(bus_id)]
|
||||||
|
|
||||||
|
def get_singular_bridge_path(bus_id: str) -> Path:
|
||||||
|
bridgePaths = get_bridge_paths(bus_id)
|
||||||
|
if len(bridgePaths) == 0:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Bridge BDF path for {bus_id}")
|
||||||
|
if len(bridgePaths) != 1:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Bridge BDF path for {bus_id} since too many Extended Bridge BDFs match: {bridgePaths}")
|
||||||
|
return bridgePaths[0]
|
||||||
|
|
||||||
|
def get_singular_bridge_extended_bdf(bus_id: str) -> str:
|
||||||
|
bridgeBDFs = get_bridge_extended_bdfs(bus_id)
|
||||||
|
if len(bridgeBDFs) == 0:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Bridge BDF for {bus_id}")
|
||||||
|
if len(bridgeBDFs) != 1:
|
||||||
|
sys.exit(f":ERROR: Unable to obtain Extended Bridge BDF for {bus_id} since too many Extended Bridge BDFs match: {bridgeBDFs}")
|
||||||
|
return bridgeBDFs[0]
|
||||||
|
|
||||||
|
# misc
|
||||||
|
|
||||||
|
def get_fpga_devs(bus_id) -> List[Path]:
|
||||||
|
def readUevent(path: Path) -> Dict[str, str]:
|
||||||
|
if not (path / 'uevent').exists():
|
||||||
|
return {}
|
||||||
|
return { entry[0]: entry[1] for entry in [line.strip('\n\r ').split('=') for line in open(f'{path}/uevent', 'r').readlines()] if len(entry) >= 2 }
|
||||||
|
|
||||||
|
def xdmaResolver(path: Path) -> List[Path]:
|
||||||
|
xdmaDevs = []
|
||||||
|
for f in ['resource', 'resource0', 'resource1']:
|
||||||
|
rsrcPath = (path / f)
|
||||||
|
if rsrcPath.exists():
|
||||||
|
xdmaDevs.append(rsrcPath)
|
||||||
|
xdmaPath = (path / 'xdma')
|
||||||
|
if xdmaPath.is_dir():
|
||||||
|
ueventEntries = [readUevent(xdmaPath / entry.name) for entry in xdmaPath.iterdir() if (xdmaPath / entry.name).is_dir()]
|
||||||
|
xdmaDevs.extend([Path('/dev') / uevent['DEVNAME'] for uevent in ueventEntries if 'DEVNAME' in uevent and (Path('/dev') / uevent['DEVNAME']).exists()])
|
||||||
|
return xdmaDevs
|
||||||
|
|
||||||
|
resolvers = {
|
||||||
|
'xdma' : xdmaResolver
|
||||||
|
}
|
||||||
|
|
||||||
|
returnDevs = []
|
||||||
|
fpgaDevices = get_device_extended_bdfs(bus_id)
|
||||||
|
for fpgaDev in fpgaDevices:
|
||||||
|
path = pciDevicesPath / fpgaDev
|
||||||
|
fpgaDevUevent = readUevent(path)
|
||||||
|
if 'DRIVER' not in fpgaDevUevent:
|
||||||
|
print(":WARNING: Verify that 'xdma' driver is loaded")
|
||||||
|
continue
|
||||||
|
if fpgaDevUevent['DRIVER'] not in resolvers:
|
||||||
|
continue
|
||||||
|
returnDevs.extend(resolvers[fpgaDevUevent['DRIVER']](path.resolve()))
|
||||||
|
|
||||||
|
return returnDevs
|
||||||
|
|
||||||
|
# clear SERR bit in command register
|
||||||
|
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
||||||
|
def clear_serr_bits(bus_id: str) -> None:
|
||||||
|
for bridgeBDF in get_bridge_extended_bdfs(bus_id):
|
||||||
|
run = subprocess.run(['setpci', '-s', bridgeBDF, 'COMMAND=0000:0100'])
|
||||||
|
if run.returncode != 0:
|
||||||
|
sys.exit(f":ERROR: Unable to clear SERR bit for {bridgeBDF}")
|
||||||
|
|
||||||
|
# clear fatal error reporting enable bit in the device control register
|
||||||
|
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
||||||
|
def clear_fatal_error_reporting_bits(bus_id: str) -> None:
|
||||||
|
for bridgeBDF in get_bridge_extended_bdfs(bus_id):
|
||||||
|
run = subprocess.run(['setpci', '-s', bridgeBDF, 'CAP_EXP+8.w=0000:0004'])
|
||||||
|
if run.returncode != 0:
|
||||||
|
sys.exit(f":ERROR: Unable to clear error reporting bit for {bridgeBDF}")
|
||||||
|
|
||||||
|
def write_to_linux_device_path(path: Path, data: str = '1\n') -> None:
|
||||||
|
try:
|
||||||
|
print(f":INFO: Writing to {path}: {data.strip()}")
|
||||||
|
open(path, 'w').write(data)
|
||||||
|
except:
|
||||||
|
sys.exit(f":ERROR: Cannot write to {path} value: {data}")
|
||||||
|
|
||||||
|
def remove(bus_id: str) -> None:
|
||||||
|
for devicePaths in get_device_paths(bus_id):
|
||||||
|
removePath = devicePaths.resolve().absolute() / 'remove'
|
||||||
|
if removePath.exists():
|
||||||
|
write_to_linux_device_path(removePath)
|
||||||
|
|
||||||
|
def rescan(bus_id: str) -> None:
|
||||||
|
for bridgePath in get_bridge_paths(bus_id):
|
||||||
|
rescanPath = bridgePath / 'rescan'
|
||||||
|
if rescanPath.exists():
|
||||||
|
write_to_linux_device_path(rescanPath)
|
||||||
|
write_to_linux_device_path(Path('/sys/bus/pci/rescan'))
|
||||||
|
|
||||||
|
# enable memory mapped transfers for the fpga
|
||||||
|
# https://support.xilinx.com/s/question/0D52E00006iHlNoSAK/lspci-reports-bar-0-disabled?language=en_US
|
||||||
|
def enable_memmapped_transfers(bus_id: str) -> None:
|
||||||
|
for deviceBDF in get_device_extended_bdfs(bus_id):
|
||||||
|
run = subprocess.run(['setpci', '-s', deviceBDF, 'COMMAND=0x02'])
|
||||||
|
if run.returncode != 0:
|
||||||
|
sys.exit(f":ERROR: Unable to enable memmapped transfers on {deviceBDF}")
|
||||||
|
|
||||||
|
def any_device_exists(bus_id: str) -> bool:
|
||||||
|
return len(get_device_paths(bus_id)) > 0
|
||||||
|
|
||||||
|
# converter funcs
|
||||||
|
|
||||||
|
def get_extended_bdf_from_bdf(bdf: str) -> str:
|
||||||
|
return '0000:' + bdf
|
||||||
|
|
||||||
|
def get_bus_id_from_extended_bdf(extended_bdf: str) -> str:
|
||||||
|
return extended_bdf[5:7]
|
||||||
|
|
||||||
|
def get_bdf_from_extended_bdf(extended_bdf: str) -> str:
|
||||||
|
return extended_bdf[5:]
|
|
@ -1,194 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import pwd
|
|
||||||
import re
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from typing import Optional, Dict, Any, List
|
|
||||||
|
|
||||||
pciDevicesPath = Path('/sys/bus/pci/devices')
|
|
||||||
|
|
||||||
def get_bridge_bdf(id: str) -> str:
|
|
||||||
for entry in pciDevicesPath.iterdir():
|
|
||||||
if re.match('^0000:' + re.escape(id) + ':[a-fA-F0-9]{2}\.[0-7]$', entry.name):
|
|
||||||
bridgePath = entry.resolve().absolute().parent
|
|
||||||
if bridgePath.exists():
|
|
||||||
return bridgePath.name
|
|
||||||
print(":ERROR: Unable to obtain bridge BDF")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def get_fpga_bdfs(id: str) -> List[str]:
|
|
||||||
result = []
|
|
||||||
for entry in pciDevicesPath.iterdir():
|
|
||||||
if re.match('^0000:' + re.escape(id) + ':[a-fA-F0-9]{2}\.[0-7]$', entry.name):
|
|
||||||
result.append(entry.name)
|
|
||||||
return result
|
|
||||||
|
|
||||||
def get_fpga_devs(id) -> List[Path]:
|
|
||||||
def readUevent(path: Path) -> Dict[Any, Any]:
|
|
||||||
if not (path / 'uevent').exists():
|
|
||||||
return {}
|
|
||||||
return { entry[0]: entry[1] for entry in [line.strip('\n\r ').split('=') for line in open(f'{path}/uevent', 'r').readlines()] if len(entry) >= 2 }
|
|
||||||
|
|
||||||
def xdmaResolver(path: Path) -> List[Path]:
|
|
||||||
xdmaDevs = []
|
|
||||||
for f in ['resource', 'resource0', 'resource1']:
|
|
||||||
rsrcPath = (path / f)
|
|
||||||
if rsrcPath.exists():
|
|
||||||
xdmaDevs.append(rsrcPath)
|
|
||||||
xdmaPath = (path / 'xdma')
|
|
||||||
if xdmaPath.is_dir():
|
|
||||||
ueventEntries = [readUevent(xdmaPath / entry.name) for entry in xdmaPath.iterdir() if (xdmaPath / entry.name).is_dir()]
|
|
||||||
xdmaDevs.extend([Path('/dev') / uevent['DEVNAME'] for uevent in ueventEntries if 'DEVNAME' in uevent and (Path('/dev') / uevent['DEVNAME']).exists()])
|
|
||||||
return xdmaDevs
|
|
||||||
|
|
||||||
resolvers = {
|
|
||||||
'xdma' : xdmaResolver
|
|
||||||
}
|
|
||||||
|
|
||||||
returnDevs = []
|
|
||||||
fpgaDevices = get_fpga_bdfs(id)
|
|
||||||
for fpgaDev in fpgaDevices:
|
|
||||||
path = pciDevicesPath / fpgaDev
|
|
||||||
fpgaDevUevent = readUevent(path)
|
|
||||||
if 'DRIVER' not in fpgaDevUevent:
|
|
||||||
print(":WARNING: Verify that 'xdma' driver is loaded")
|
|
||||||
continue
|
|
||||||
if fpgaDevUevent['DRIVER'] not in resolvers:
|
|
||||||
continue
|
|
||||||
returnDevs.extend(resolvers[fpgaDevUevent['DRIVER']](path.resolve()))
|
|
||||||
|
|
||||||
return returnDevs
|
|
||||||
|
|
||||||
# clear SERR bit in command register
|
|
||||||
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
|
||||||
def clear_serr(id: str) -> None:
|
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
|
||||||
run = subprocess.run(['setpci', '-s', bridgeBDF, 'COMMAND=0000:0100'])
|
|
||||||
if run.returncode != 0:
|
|
||||||
print(":ERROR: Unable to clear SERR bit")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
|
|
||||||
# clear fatal error reporting enable bit in the device control register
|
|
||||||
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
|
||||||
def clear_fatal_error_reporting(id: str) -> None:
|
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
|
||||||
run = subprocess.run(['setpci', '-s', bridgeBDF, 'CAP_EXP+8.w=0000:0004'])
|
|
||||||
if run.returncode != 0:
|
|
||||||
print(":ERROR: Unable to clear SERR bit")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def write_to_linux_device_path(path: Path, data: str = '1\n') -> None:
|
|
||||||
try:
|
|
||||||
open(path, 'w').write(data)
|
|
||||||
except:
|
|
||||||
print(f":ERROR: Cannot write to {path} value: {data}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def remove(id: str) -> None:
|
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
|
||||||
deviceBDFs = get_fpga_bdfs(id)
|
|
||||||
for deviceBDF in deviceBDFs:
|
|
||||||
removePath = pciDevicesPath / bridgeBDF / deviceBDF / 'remove'
|
|
||||||
if removePath.exists():
|
|
||||||
write_to_linux_device_path(removePath)
|
|
||||||
|
|
||||||
def rescan(id: str) -> None:
|
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
|
||||||
if bridgeBDF is not None:
|
|
||||||
rescanPath = pciDevicesPath / bridgeBDF / 'rescan'
|
|
||||||
write_to_linux_device_path(rescanPath)
|
|
||||||
else:
|
|
||||||
write_to_linux_device_path('/sys/bus/pci/rescan')
|
|
||||||
|
|
||||||
# enable memory mapped transfers for the fpga
|
|
||||||
# https://support.xilinx.com/s/question/0D52E00006iHlNoSAK/lspci-reports-bar-0-disabled?language=en_US
|
|
||||||
def enable_memmapped_transfers(id: str) -> None:
|
|
||||||
deviceBDFs = get_fpga_bdfs(id)
|
|
||||||
for deviceBDF in deviceBDFs:
|
|
||||||
run = subprocess.run(['setpci', '-s', deviceBDF, 'COMMAND=0x02'])
|
|
||||||
if run.returncode != 0:
|
|
||||||
print(f":ERROR: Unable to enable memmapped transfers on {deviceBDF}")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def program_fpga(serial: str, board: str, bitstream: str) -> None:
|
|
||||||
print(":WARNING: This only can target the 1st FPGA on a machine currently...")
|
|
||||||
|
|
||||||
pVivado = subprocess.Popen(
|
|
||||||
[
|
|
||||||
'vivado',
|
|
||||||
'-mode', 'tcl',
|
|
||||||
'-nolog', '-nojournal', '-notrace',
|
|
||||||
'-source', scriptPath / 'program_fpga.tcl',
|
|
||||||
'-tclargs',
|
|
||||||
'-board', board,
|
|
||||||
'-bitstream_path', bitstream,
|
|
||||||
],
|
|
||||||
stdin=subprocess.DEVNULL
|
|
||||||
)
|
|
||||||
|
|
||||||
pVivado.wait()
|
|
||||||
|
|
||||||
if pVivado.returncode != 0:
|
|
||||||
print(":ERROR: Unable to flash FPGA")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def get_serial_from_bdf(id: str) -> str:
|
|
||||||
deviceBDFs = get_fpga_bdfs(parsed_args.bus_id)
|
|
||||||
if len(deviceBDFs) == 0:
|
|
||||||
print(f":ERROR: Unable to obtain Extended Device BDF for {parsed_args.bus_id}")
|
|
||||||
sys.exit(1)
|
|
||||||
return "TODO"
|
|
||||||
|
|
||||||
def main(args: List[str]) -> int:
|
|
||||||
parser = argparse.ArgumentParser(description="Program a Xilinx XDMA-enabled FPGA")
|
|
||||||
megroup = parser.add_mutually_exclusive_group(required=True)
|
|
||||||
megroup.add_argument("--bus_id", help="Bus number of FPGA to flash (i.e. ****:<THIS>:**.*)")
|
|
||||||
megroup.add_argument("--serial_no", help="Serial number of FPGA to flash (i.e. what 'get_hw_target' shows in Vivado)")
|
|
||||||
parser.add_argument("--bitstream", help="Bitstream to flash onto FPGA", required=True, type=Path)
|
|
||||||
parser.add_argument("--board", help="FPGA board to flash", required=True)
|
|
||||||
parsed_args = parser.parse_args(args)
|
|
||||||
|
|
||||||
scriptPath = Path(__file__).resolve().parent
|
|
||||||
|
|
||||||
eUserId = os.geteuid()
|
|
||||||
sudoUserId = os.getenv('SUDO_UID')
|
|
||||||
isAdmin = (eUserId == 0) and (sudoUserId is None)
|
|
||||||
userId = eUserId if sudoUserId is None else int(sudoUserId)
|
|
||||||
|
|
||||||
if not isAdmin:
|
|
||||||
print(":ERROR: Requires running script with 'sudo'")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if not parsed_args.bitstream.is_file() or not parsed_args.bitstream.exists():
|
|
||||||
print(f":ERROR: Invalid bitstream: {parsed_args.bitstream}")
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
|
||||||
parsed_args.bitstream = parsed_args.bitstream.absolute()
|
|
||||||
|
|
||||||
if parsed_args.bus_id:
|
|
||||||
serialNumber = get_serial_from_bdf(id)
|
|
||||||
clear_serr(parsed_args.bus_id)
|
|
||||||
clear_fatal_error_reporting(parsed_args.bus_id)
|
|
||||||
remove(parsed_args.bus_id)
|
|
||||||
program_fpga(serialNumber, parsed_args.board, parsed_args.bitstream)
|
|
||||||
rescan(parsed_args.bus_id)
|
|
||||||
enable_memmapped_transfers(parsed_args.bus_id)
|
|
||||||
|
|
||||||
print(f"Successfully programmed FPGA {parsed_args.bus_id} with {parsed_args.bitstream}")
|
|
||||||
|
|
||||||
if parsed_args.serial_no:
|
|
||||||
program_fpga(parsed_args.serial_no, parsed_args.board, parsed_args.bitstream)
|
|
||||||
|
|
||||||
print(f"Successfully programmed FPGA {parsed_args.serial_no} with {parsed_args.bitstream}")
|
|
||||||
print(":WARNING: Please warm reboot the machine")
|
|
||||||
|
|
||||||
return 0
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
sys.exit(main(sys.argv[1:]))
|
|
|
@ -1,58 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
# Adapted from https://github.com/Xilinx/open-nic-shell
|
|
||||||
|
|
||||||
echo $#
|
|
||||||
if [[ $# -le 1 ]] || [[ -z EXTENDED_DEVICE_BDF1 ]] || [[ -z $XILINX_VIVADO ]]; then
|
|
||||||
echo "Usage: EXTENDED_DEVICE_BDF1=<EXTENDED_DEVICE_BDF1> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please export EXTENDED_DEVICE_BDF1 and [EXTENDED_DEVICE_BDF2 (if needed for 2 port boards)]"
|
|
||||||
echo "Example: EXTENDED_DEVICE_BDF1=<0000:86:00.0> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please ensure vivado is loaded into system path."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
bridge_bdf=""
|
|
||||||
bitstream_path=$1
|
|
||||||
board=$2
|
|
||||||
probes_path="${3:-}"
|
|
||||||
# ^^ Probes are used for specifying hardware debugger symbols.
|
|
||||||
|
|
||||||
# Infer bridge
|
|
||||||
if [ -e "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1" ]; then
|
|
||||||
bridge_bdf=$(basename $(dirname $(readlink "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1")))
|
|
||||||
# Both devices will be on the same bridge as they are on the same FPGA board.
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Remove
|
|
||||||
if [[ $bridge_bdf != "" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF1}/remove" > /dev/null
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]] && [[ -e "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}/remove" > /dev/null
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Could not find bridge_bdf for the device $EXTENDED_DEVICE_BDF1"
|
|
||||||
echo "If remove was called on the device already, then manually set bridge_bdf here and comment 'exit 1'."
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Program fpga
|
|
||||||
vivado -mode tcl -source $SCRIPT_DIR/program_fpga.tcl \
|
|
||||||
-tclargs -board $board \
|
|
||||||
-bitstream_path $bitstream_path \
|
|
||||||
-probes_path $probes_path
|
|
||||||
|
|
||||||
# Rescan
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/rescan" > /dev/null
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF1 COMMAND=0x02
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]]; then
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF2 COMMAND=0x02
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "program_fpga.sh completed"
|
|
||||||
echo "Warm reboot machine if the FPGA wasn't initially setup with an XDMA bitstream."
|
|
|
@ -7,11 +7,11 @@ set root_dir [file dirname $script_dir]
|
||||||
|
|
||||||
# Loading options
|
# Loading options
|
||||||
# bitstream_path Path to the bitstream
|
# bitstream_path Path to the bitstream
|
||||||
# board Board name
|
# serial Serial number of FPGA board (without trailing A)
|
||||||
array set options {
|
array set options {
|
||||||
-bitstream_path ""
|
-bitstream_path ""
|
||||||
-probes_path ""
|
-probes_path ""
|
||||||
-board au50
|
-serial ""
|
||||||
}
|
}
|
||||||
|
|
||||||
# Expect arguments in the form of `-argument value`
|
# Expect arguments in the form of `-argument value`
|
||||||
|
@ -31,34 +31,39 @@ foreach {key value} [array get options] {
|
||||||
set [string range $key 1 end] $value
|
set [string range $key 1 end] $value
|
||||||
}
|
}
|
||||||
|
|
||||||
source ${script_dir}/${board}.tcl
|
|
||||||
|
|
||||||
puts "Program file: $options(-bitstream_path)"
|
puts "Program file: $options(-bitstream_path)"
|
||||||
puts "Probes file: $options(-probes_path)"
|
puts "Probes file: $options(-probes_path)"
|
||||||
puts "Board: $options(-board)"
|
puts "Serial Number: $options(-serial)"
|
||||||
puts "HW device: $hw_device"
|
|
||||||
|
set_param labtools.enable_cs_server false
|
||||||
|
|
||||||
open_hw_manager
|
open_hw_manager
|
||||||
connect_hw_server -allow_non_jtag
|
connect_hw_server -allow_non_jtag
|
||||||
|
|
||||||
## by default vivado opens a default hw target
|
# by default vivado opens a default hw target
|
||||||
#close_hw_target
|
close_hw_target
|
||||||
|
|
||||||
# note: helps view amount of fpgas
|
# check if serial is in hw targets
|
||||||
get_hw_targets
|
set final_hw_target ""
|
||||||
|
foreach {hw_target} [get_hw_targets] {
|
||||||
|
if {[string first $serial $hw_target] != -1} {
|
||||||
|
set final_hw_target $hw_target
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# TODO:
|
if {$final_hw_target == ""} {
|
||||||
# when no FPGA is programmed
|
puts "Unable to find $serial in available HW targets. See available HW targets below:"
|
||||||
# can use id to index into get_hw_targets and program that specific FPGA
|
get_hw_targets
|
||||||
# when you notice a PCI-ID associated w/ the FPGA
|
exit 1
|
||||||
# TODO: how do you determine the ID of the FPGA from the PCI-BDF
|
}
|
||||||
open_hw_target
|
|
||||||
current_hw_device [get_hw_devices $hw_device]
|
puts "Programming $final_hw_target with ${options(-bitstream_path)}"
|
||||||
refresh_hw_device -update_hw_probes false [lindex [get_hw_devices $hw_device] 0]
|
open_hw_target $final_hw_target
|
||||||
set_property PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
set_property PROBES.FILE ${options(-probes_path)} [get_hw_device]
|
||||||
set_property FULL_PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
set_property FULL_PROBES.FILE ${options(-probes_path)} [get_hw_device]
|
||||||
set_property PROGRAM.FILE ${options(-bitstream_path)} [get_hw_devices $hw_device]
|
set_property PROGRAM.FILE ${options(-bitstream_path)} [get_hw_device]
|
||||||
program_hw_devices [get_hw_devices $hw_device]
|
program_hw_devices [get_hw_device]
|
||||||
refresh_hw_device [lindex [get_hw_devices $hw_device] 0]
|
refresh_hw_device [get_hw_device]
|
||||||
|
close_hw_target
|
||||||
|
|
||||||
exit
|
exit
|
||||||
|
|
|
@ -0,0 +1,19 @@
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
from typing import List
|
||||||
|
|
||||||
|
def call_vivado(vivado: Path, args: List[str]) -> tuple[int, str, str]:
|
||||||
|
pVivado = subprocess.Popen(
|
||||||
|
[
|
||||||
|
str(vivado),
|
||||||
|
'-mode', 'tcl',
|
||||||
|
'-nolog', '-nojournal', '-notrace',
|
||||||
|
] + args,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
|
||||||
|
sout, serr = pVivado.communicate()
|
||||||
|
|
||||||
|
return (pVivado.returncode, sout.decode('utf-8'), serr.decode('utf-8'))
|
|
@ -21,4 +21,3 @@
|
||||||
set part xcu280-fsvh2892-2L-e
|
set part xcu280-fsvh2892-2L-e
|
||||||
set board_part xilinx.com:au280:part0:1.2
|
set board_part xilinx.com:au280:part0:1.2
|
||||||
set zynq_family 0
|
set zynq_family 0
|
||||||
set hw_device xcu280_u55c_0
|
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
../xilinx_alveo_u250/scripts
|
|
@ -1 +0,0 @@
|
||||||
../cl_firesim/scripts/au280.tcl
|
|
|
@ -1,58 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
# Adapted from https://github.com/Xilinx/open-nic-shell
|
|
||||||
|
|
||||||
echo $#
|
|
||||||
if [[ $# -le 1 ]] || [[ -z EXTENDED_DEVICE_BDF1 ]] || [[ -z $XILINX_VIVADO ]]; then
|
|
||||||
echo "Usage: EXTENDED_DEVICE_BDF1=<EXTENDED_DEVICE_BDF1> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please export EXTENDED_DEVICE_BDF1 and [EXTENDED_DEVICE_BDF2 (if needed for 2 port boards)]"
|
|
||||||
echo "Example: EXTENDED_DEVICE_BDF1=<0000:86:00.0> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please ensure vivado is loaded into system path."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
bridge_bdf=""
|
|
||||||
bitstream_path=$1
|
|
||||||
board=$2
|
|
||||||
probes_path="${3:-}"
|
|
||||||
# ^^ Probes are used for specifying hardware debugger symbols.
|
|
||||||
|
|
||||||
# Infer bridge
|
|
||||||
if [ -e "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1" ]; then
|
|
||||||
bridge_bdf=$(basename $(dirname $(readlink "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1")))
|
|
||||||
# Both devices will be on the same bridge as they are on the same FPGA board.
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Remove
|
|
||||||
if [[ $bridge_bdf != "" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF1}/remove" > /dev/null
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]] && [[ -e "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}/remove" > /dev/null
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Could not find bridge_bdf for the device $EXTENDED_DEVICE_BDF1"
|
|
||||||
echo "If remove was called on the device already, then manually set bridge_bdf here and comment 'exit 1'."
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Program fpga
|
|
||||||
vivado -mode tcl -source $SCRIPT_DIR/program_fpga.tcl \
|
|
||||||
-tclargs -board $board \
|
|
||||||
-bitstream_path $bitstream_path \
|
|
||||||
-probes_path $probes_path
|
|
||||||
|
|
||||||
# Rescan
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/rescan" > /dev/null
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF1 COMMAND=0x02
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]]; then
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF2 COMMAND=0x02
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "program_fpga.sh completed"
|
|
||||||
echo "Warm reboot machine if the FPGA wasn't initially setup with an XDMA bitstream."
|
|
|
@ -1,52 +0,0 @@
|
||||||
# Adapted from https://github.com/Xilinx/open-nic-shell
|
|
||||||
|
|
||||||
# Directory variables
|
|
||||||
set script_path [file normalize [info script]]
|
|
||||||
set script_dir [file dirname $script_path]
|
|
||||||
set root_dir [file dirname $script_dir]
|
|
||||||
|
|
||||||
# Loading options
|
|
||||||
# bitstream_path Path to the bitstream
|
|
||||||
# board Board name
|
|
||||||
array set options {
|
|
||||||
-bitstream_path ""
|
|
||||||
-probes_path ""
|
|
||||||
-board au50
|
|
||||||
}
|
|
||||||
|
|
||||||
# Expect arguments in the form of `-argument value`
|
|
||||||
for {set i 0} {$i < $argc} {incr i 2} {
|
|
||||||
set arg [lindex $argv $i]
|
|
||||||
set val [lindex $argv [expr $i+1]]
|
|
||||||
if {[info exists options($arg)]} {
|
|
||||||
set options($arg) $val
|
|
||||||
puts "Set option $arg to $val"
|
|
||||||
} else {
|
|
||||||
puts "Skip unknown argument $arg and its value $val"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Settings based on defaults or passed in values
|
|
||||||
foreach {key value} [array get options] {
|
|
||||||
set [string range $key 1 end] $value
|
|
||||||
}
|
|
||||||
|
|
||||||
source ${script_dir}/${board}.tcl
|
|
||||||
|
|
||||||
puts "Program file: $options(-bitstream_path)"
|
|
||||||
puts "Probes file: $options(-probes_path)"
|
|
||||||
puts "Board: $options(-board)"
|
|
||||||
puts "HW device: $hw_device"
|
|
||||||
|
|
||||||
open_hw_manager
|
|
||||||
connect_hw_server -allow_non_jtag
|
|
||||||
open_hw_target
|
|
||||||
current_hw_device [get_hw_devices $hw_device]
|
|
||||||
refresh_hw_device -update_hw_probes false [lindex [get_hw_devices $hw_device] 0]
|
|
||||||
set_property PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
|
||||||
set_property FULL_PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
|
||||||
set_property PROGRAM.FILE ${options(-bitstream_path)} [get_hw_devices $hw_device]
|
|
||||||
program_hw_devices [get_hw_devices $hw_device]
|
|
||||||
refresh_hw_device [lindex [get_hw_devices $hw_device] 0]
|
|
||||||
|
|
||||||
exit
|
|
|
@ -110,7 +110,7 @@ dromajo_t::dromajo_t(simif_t &sim,
|
||||||
* Destructor for Dromajo
|
* Destructor for Dromajo
|
||||||
*/
|
*/
|
||||||
dromajo_t::~dromajo_t() {
|
dromajo_t::~dromajo_t() {
|
||||||
if (this->dromajo_state != NULL)
|
if (this->dromajo_state)
|
||||||
dromajo_cosim_fini(this->dromajo_state);
|
dromajo_cosim_fini(this->dromajo_state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,7 +151,7 @@ void dromajo_t::init() {
|
||||||
|
|
||||||
printf("[INFO] Dromajo command: \n");
|
printf("[INFO] Dromajo command: \n");
|
||||||
char *dromajo_argv[dromajo_args.size()];
|
char *dromajo_argv[dromajo_args.size()];
|
||||||
for (int i = 0; i < dromajo_args.size(); ++i) {
|
for (size_t i = 0; i < dromajo_args.size(); ++i) {
|
||||||
dromajo_argv[i] = const_cast<char *>(dromajo_args[i].c_str());
|
dromajo_argv[i] = const_cast<char *>(dromajo_args[i].c_str());
|
||||||
printf("%s ", dromajo_argv[i]);
|
printf("%s ", dromajo_argv[i]);
|
||||||
}
|
}
|
||||||
|
|
|
@ -216,10 +216,13 @@ simplenic_t::~simplenic_t() {
|
||||||
fclose(this->niclog);
|
fclose(this->niclog);
|
||||||
if (loopback) {
|
if (loopback) {
|
||||||
for (auto &pcis_read_buf : pcis_read_bufs)
|
for (auto &pcis_read_buf : pcis_read_bufs)
|
||||||
|
if (pcis_read_buf)
|
||||||
free(pcis_read_buf);
|
free(pcis_read_buf);
|
||||||
} else {
|
} else {
|
||||||
for (int j = 0; j < 2; j++) {
|
for (int j = 0; j < 2; j++) {
|
||||||
|
if (pcis_read_bufs[j])
|
||||||
munmap(pcis_read_bufs[j], BUFBYTES + EXTRABYTES);
|
munmap(pcis_read_bufs[j], BUFBYTES + EXTRABYTES);
|
||||||
|
if (pcis_write_bufs[j])
|
||||||
munmap(pcis_write_bufs[j], BUFBYTES + EXTRABYTES);
|
munmap(pcis_write_bufs[j], BUFBYTES + EXTRABYTES);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -252,7 +252,7 @@ void tracerv_t::serialize(
|
||||||
const bool fireperf) {
|
const bool fireperf) {
|
||||||
const int max_consider = std::min(max_core_ipc, 7);
|
const int max_consider = std::min(max_core_ipc, 7);
|
||||||
if (human_readable || test_output) {
|
if (human_readable || test_output) {
|
||||||
for (int i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
for (size_t i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
||||||
if (test_output) {
|
if (test_output) {
|
||||||
fprintf(tracefile, "%016lx", OUTBUF[i + 7]);
|
fprintf(tracefile, "%016lx", OUTBUF[i + 7]);
|
||||||
fprintf(tracefile, "%016lx", OUTBUF[i + 6]);
|
fprintf(tracefile, "%016lx", OUTBUF[i + 6]);
|
||||||
|
@ -277,7 +277,7 @@ void tracerv_t::serialize(
|
||||||
}
|
}
|
||||||
} else if (fireperf) {
|
} else if (fireperf) {
|
||||||
|
|
||||||
for (int i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
for (size_t i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
||||||
uint64_t cycle_internal = OUTBUF[i + 0];
|
uint64_t cycle_internal = OUTBUF[i + 0];
|
||||||
|
|
||||||
for (int q = 0; q < max_consider; q++) {
|
for (int q = 0; q < max_consider; q++) {
|
||||||
|
@ -293,7 +293,7 @@ void tracerv_t::serialize(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for (int i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
for (size_t i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
||||||
// this stores as raw binary. stored as little endian.
|
// this stores as raw binary. stored as little endian.
|
||||||
// e.g. to get the same thing as the human readable above,
|
// e.g. to get the same thing as the human readable above,
|
||||||
// flip all the bytes in each 512-bit line.
|
// flip all the bytes in each 512-bit line.
|
||||||
|
|
|
@ -66,9 +66,11 @@ tsibridge_t::tsibridge_t(simif_t &simif,
|
||||||
}
|
}
|
||||||
|
|
||||||
tsibridge_t::~tsibridge_t() {
|
tsibridge_t::~tsibridge_t() {
|
||||||
|
if (fesvr)
|
||||||
delete fesvr;
|
delete fesvr;
|
||||||
if (tsi_argv) {
|
if (tsi_argv) {
|
||||||
for (int i = 0; i < tsi_argc; ++i) {
|
for (int i = 0; i < tsi_argc; ++i) {
|
||||||
|
if (tsi_argv[i])
|
||||||
delete[] tsi_argv[i];
|
delete[] tsi_argv[i];
|
||||||
}
|
}
|
||||||
delete[] tsi_argv;
|
delete[] tsi_argv;
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
#include "master.h"
|
#include "master.h"
|
||||||
#include "core/simif.h"
|
#include "core/simif.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
char master_t::KIND;
|
char master_t::KIND;
|
||||||
|
|
||||||
|
@ -13,4 +14,14 @@ master_t::master_t(simif_t &simif,
|
||||||
assert(index == 0 && "only one simulation master is allowed");
|
assert(index == 0 && "only one simulation master is allowed");
|
||||||
}
|
}
|
||||||
|
|
||||||
bool master_t::is_init_done() { return simif.read(mmio_addrs.INIT_DONE); }
|
bool master_t::is_init_done() { return simif.read(mmio_addrs.INIT_DONE) == 1; }
|
||||||
|
|
||||||
|
bool master_t::check_fingerprint() {
|
||||||
|
uint32_t presence = simif.read(mmio_addrs.PRESENCE_READ);
|
||||||
|
printf("FireSim fingerprint: 0x%x\n", presence);
|
||||||
|
return presence != 0x46697265;
|
||||||
|
}
|
||||||
|
|
||||||
|
void master_t::write_fingerprint(uint32_t data) {
|
||||||
|
simif.write(mmio_addrs.PRESENCE_WRITE, data);
|
||||||
|
}
|
||||||
|
|
|
@ -13,6 +13,8 @@ class simif_t;
|
||||||
|
|
||||||
struct SIMULATIONMASTER_struct {
|
struct SIMULATIONMASTER_struct {
|
||||||
uint64_t INIT_DONE;
|
uint64_t INIT_DONE;
|
||||||
|
uint64_t PRESENCE_READ;
|
||||||
|
uint64_t PRESENCE_WRITE;
|
||||||
};
|
};
|
||||||
|
|
||||||
class master_t final : public widget_t {
|
class master_t final : public widget_t {
|
||||||
|
@ -25,6 +27,16 @@ public:
|
||||||
unsigned index,
|
unsigned index,
|
||||||
const std::vector<std::string> &args);
|
const std::vector<std::string> &args);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether the device has FireSim fingerprint string.
|
||||||
|
*/
|
||||||
|
bool check_fingerprint();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write new value to FireSim fingerprint string.
|
||||||
|
*/
|
||||||
|
void write_fingerprint(uint32_t data);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check whether the device is initialised.
|
* Check whether the device is initialised.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -132,6 +132,7 @@ synthesized_prints_t::synthesized_prints_t(
|
||||||
|
|
||||||
synthesized_prints_t::~synthesized_prints_t() {
|
synthesized_prints_t::~synthesized_prints_t() {
|
||||||
for (size_t i = 0; i < prints.size(); i++) {
|
for (size_t i = 0; i < prints.size(); i++) {
|
||||||
|
if (masks[i])
|
||||||
delete masks[i];
|
delete masks[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,6 +27,12 @@ simulation_t::simulation_t(widget_registry_t ®istry,
|
||||||
if (arg.find("+zero-out-dram") == 0) {
|
if (arg.find("+zero-out-dram") == 0) {
|
||||||
do_zero_out_dram = true;
|
do_zero_out_dram = true;
|
||||||
}
|
}
|
||||||
|
if (arg.find("+check-fingerprint") == 0) {
|
||||||
|
check_fingerprint_only = true;
|
||||||
|
}
|
||||||
|
if (arg.find("+write-fingerprint=") == 0) {
|
||||||
|
write_fingerprint_only = atoi(arg.c_str() + 19);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fastloadmem)
|
if (fastloadmem)
|
||||||
|
@ -47,9 +53,10 @@ void simulation_t::record_end_times() {
|
||||||
void simulation_t::print_simulation_performance_summary() {
|
void simulation_t::print_simulation_performance_summary() {
|
||||||
// Must call record_start_times and record_end_times before invoking this
|
// Must call record_start_times and record_end_times before invoking this
|
||||||
// function
|
// function
|
||||||
assert(start_hcycle != -1 && end_hcycle != 0 && "simulation not executed");
|
assert(start_hcycle.has_value() && end_hcycle.has_value() &&
|
||||||
|
"simulation not executed");
|
||||||
|
|
||||||
const uint64_t hcycles = end_hcycle - start_hcycle;
|
const uint64_t hcycles = *end_hcycle - *start_hcycle;
|
||||||
const double sim_time = diff_secs(end_time, start_time);
|
const double sim_time = diff_secs(end_time, start_time);
|
||||||
const double sim_speed = ((double)end_tcycle) / (sim_time * 1000.0);
|
const double sim_speed = ((double)end_tcycle) / (sim_time * 1000.0);
|
||||||
const double measured_host_frequency =
|
const double measured_host_frequency =
|
||||||
|
@ -95,6 +102,30 @@ void simulation_t::simulation_finish() {
|
||||||
int simulation_t::execute_simulation_flow() {
|
int simulation_t::execute_simulation_flow() {
|
||||||
wait_for_init();
|
wait_for_init();
|
||||||
|
|
||||||
|
// following fingerprint logic uses 'exit' instead of 'return' to avoid
|
||||||
|
// issues w/ deconstructors not having initialized values
|
||||||
|
auto &master = registry.get_widget<master_t>();
|
||||||
|
if (check_fingerprint_only || write_fingerprint_only.has_value()) {
|
||||||
|
if (check_fingerprint_only && write_fingerprint_only.has_value()) {
|
||||||
|
fprintf(stderr, "Unable to both check/write FireSim fingerprint\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (check_fingerprint_only && master.check_fingerprint()) {
|
||||||
|
fprintf(stderr, "Invalid FireSim fingerprint\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
if (write_fingerprint_only.has_value()) {
|
||||||
|
master.write_fingerprint(write_fingerprint_only.value());
|
||||||
|
}
|
||||||
|
exit(EXIT_SUCCESS);
|
||||||
|
} else {
|
||||||
|
if (master.check_fingerprint()) {
|
||||||
|
fprintf(stderr, "Invalid FireSim fingerprint\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (auto *stream = registry.get_stream_engine()) {
|
if (auto *stream = registry.get_stream_engine()) {
|
||||||
stream->init();
|
stream->init();
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
#ifndef __SIMULATION_H
|
#ifndef __SIMULATION_H
|
||||||
#define __SIMULATION_H
|
#define __SIMULATION_H
|
||||||
|
|
||||||
|
#include <optional>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
@ -129,10 +130,21 @@ private:
|
||||||
*/
|
*/
|
||||||
bool do_zero_out_dram = false;
|
bool do_zero_out_dram = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If set, read the presence register, check it, and exit the simulation
|
||||||
|
* cleanly
|
||||||
|
*/
|
||||||
|
bool check_fingerprint_only = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If set, write the presence register and exit the simulation cleanly
|
||||||
|
*/
|
||||||
|
std::optional<uint32_t> write_fingerprint_only;
|
||||||
|
|
||||||
midas_time_t start_time;
|
midas_time_t start_time;
|
||||||
midas_time_t end_time;
|
midas_time_t end_time;
|
||||||
uint64_t start_hcycle = -1;
|
std::optional<uint64_t> start_hcycle;
|
||||||
uint64_t end_hcycle = 0;
|
std::optional<uint64_t> end_hcycle;
|
||||||
uint64_t end_tcycle = 0;
|
uint64_t end_tcycle = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -54,7 +54,10 @@ void mm_t::init(size_t sz, int lsz) {
|
||||||
size = sz;
|
size = sz;
|
||||||
}
|
}
|
||||||
|
|
||||||
mm_t::~mm_t() { munmap(data, this->size); }
|
mm_t::~mm_t() {
|
||||||
|
if (data)
|
||||||
|
munmap(data, this->size);
|
||||||
|
}
|
||||||
|
|
||||||
void mm_magic_t::init(size_t sz, int lsz) {
|
void mm_magic_t::init(size_t sz, int lsz) {
|
||||||
mm_t::init(sz, lsz);
|
mm_t::init(sz, lsz);
|
||||||
|
|
|
@ -10,7 +10,7 @@
|
||||||
#include "bridges/cpu_managed_stream.h"
|
#include "bridges/cpu_managed_stream.h"
|
||||||
#include "core/simif.h"
|
#include "core/simif.h"
|
||||||
|
|
||||||
#define PCI_DEV_FMT "%04x:%02x:%02x.%d"
|
#define PCI_DEV_FMT "%04x:%02d:%02x.%d"
|
||||||
|
|
||||||
class simif_xilinx_alveo_u250_t final : public simif_t,
|
class simif_xilinx_alveo_u250_t final : public simif_t,
|
||||||
public CPUManagedStreamIO {
|
public CPUManagedStreamIO {
|
||||||
|
@ -49,7 +49,11 @@ private:
|
||||||
};
|
};
|
||||||
|
|
||||||
static int fpga_pci_check_file_id(char *path, uint16_t id) {
|
static int fpga_pci_check_file_id(char *path, uint16_t id) {
|
||||||
|
if (path) {
|
||||||
|
fprintf(stdout, "Opening %s\n", path);
|
||||||
|
} else {
|
||||||
assert(path);
|
assert(path);
|
||||||
|
}
|
||||||
int ret = 0;
|
int ret = 0;
|
||||||
FILE *fp = fopen(path, "r");
|
FILE *fp = fopen(path, "r");
|
||||||
assert(fp);
|
assert(fp);
|
||||||
|
@ -112,8 +116,10 @@ void simif_xilinx_alveo_u250_t::check_rc(int rc, char *infostr) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void simif_xilinx_alveo_u250_t::fpga_shutdown() {
|
void simif_xilinx_alveo_u250_t::fpga_shutdown() {
|
||||||
|
if (bar0_base) {
|
||||||
int ret = munmap(bar0_base, bar0_size);
|
int ret = munmap(bar0_base, bar0_size);
|
||||||
assert(ret == 0);
|
assert(ret == 0);
|
||||||
|
}
|
||||||
close(edma_write_fd);
|
close(edma_write_fd);
|
||||||
close(edma_read_fd);
|
close(edma_read_fd);
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,16 @@ class SimulationMaster(implicit p: Parameters) extends Widget()(p) {
|
||||||
when (initDelay =/= 0.U) { initDelay := initDelay - 1.U }
|
when (initDelay =/= 0.U) { initDelay := initDelay - 1.U }
|
||||||
genRORegInit(initDelay === 0.U, "INIT_DONE", 0.U)
|
genRORegInit(initDelay === 0.U, "INIT_DONE", 0.U)
|
||||||
|
|
||||||
|
// add fingerprint to see if device is FireSim-enabled
|
||||||
|
val fingerprint = 0x46697265
|
||||||
|
val rFingerprint = RegInit(fingerprint.U(32.W))
|
||||||
|
genROReg(rFingerprint, "PRESENCE_READ")
|
||||||
|
|
||||||
|
val wFingerprint = genWORegInit(Wire(UInt(32.W)), "PRESENCE_WRITE", fingerprint.U(32.W))
|
||||||
|
when (wFingerprint =/= rFingerprint) {
|
||||||
|
rFingerprint := wFingerprint
|
||||||
|
}
|
||||||
|
|
||||||
genCRFile()
|
genCRFile()
|
||||||
|
|
||||||
override def genHeader(base: BigInt, memoryRegions: Map[String, BigInt], sb: StringBuilder): Unit = {
|
override def genHeader(base: BigInt, memoryRegions: Map[String, BigInt], sb: StringBuilder): Unit = {
|
||||||
|
|
|
@ -1,9 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
RUNBIN=../target-design/chipyard/tests/blkdev.riscv
|
|
||||||
#RUNBIN=../riscv-tools-install/riscv64-unknown-elf/share/riscv-tests/isa/rv64ui-p-add
|
|
||||||
|
|
||||||
# generate a fresh test.stuff disk, all zeroed
|
|
||||||
dd if=/dev/zero bs=1M count=128 of=test.disk
|
|
||||||
|
|
||||||
./generated-src/f1/FireSimRocketConfig/FireSim +mm_readLatency=10 +mm_writeLatency=10 +mm_readMaxReqs=4 +mm_writeMaxReqs=4 +blkdev0=test.disk $RUNBIN
|
|
|
@ -44,28 +44,28 @@ sim_binary_basename := $(basename $(notdir $(SIM_BINARY)))
|
||||||
|
|
||||||
run-verilator: $(verilator)
|
run-verilator: $(verilator)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(verilator) +permissive $(verilator_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(verilator) +permissive $(verilator_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out </dev/null
|
||||||
|
|
||||||
run-verilator-debug: $(verilator_debug)
|
run-verilator-debug: $(verilator_debug)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(verilator_debug) +permissive $(verilator_args) +waveform=$(sim_binary_basename).vpd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(verilator_debug) +permissive $(verilator_args) +waveform=$(sim_binary_basename).vcd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out </dev/null
|
||||||
|
|
||||||
run-vcs: $(vcs)
|
run-vcs: $(vcs)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(vcs) +permissive $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(vcs) +permissive $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out
|
||||||
|
|
||||||
run-vcs-debug: $(vcs_debug)
|
run-vcs-debug: $(vcs_debug)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(vcs_debug) +permissive $(vcs_args) +waveform=$(sim_binary_basename).vpd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(vcs_debug) +permissive $(vcs_args) +waveform=$(sim_binary_basename).vpd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out </dev/null
|
||||||
|
|
||||||
.PHONY: run-xsim
|
.PHONY: run-xsim
|
||||||
run-xsim: $(xsim)
|
run-xsim: $(xsim)
|
||||||
cd $(dir $<) && ./$(notdir $<) +permissive $(COMMON_SIM_ARGS) $(FPGA_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
cd $(dir $<) && ./$(notdir $<) +permissive $(COMMON_SIM_ARGS) $(FPGA_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
||||||
+permissive-off $(abspath $(SIM_BINARY))
|
+permissive-off $(abspath $(SIM_BINARY)) </dev/null
|
||||||
|
|
||||||
############################################
|
############################################
|
||||||
# Midas-Level Simulation Execution Recipes #
|
# Midas-Level Simulation Execution Recipes #
|
||||||
|
|
Loading…
Reference in New Issue