More U250/U280 Fixes
This commit is contained in:
parent
f7dd99e4d4
commit
ee9ca40f50
|
@ -303,6 +303,11 @@ def builddriver(runtime_conf: RuntimeConfig) -> None:
|
||||||
an entire metasim)."""
|
an entire metasim)."""
|
||||||
runtime_conf.build_driver()
|
runtime_conf.build_driver()
|
||||||
|
|
||||||
|
@register_task
|
||||||
|
def enumeratefpgas(runtime_conf: RuntimeConfig) -> None:
|
||||||
|
""" For all run hosts, create the /opt/firesim-db.json file """
|
||||||
|
runtime_conf.enumerate_fpgas()
|
||||||
|
|
||||||
@register_task
|
@register_task
|
||||||
# XXX this needs to be renamed or rethought, perhaps this is a backend-specific task?
|
# XXX this needs to be renamed or rethought, perhaps this is a backend-specific task?
|
||||||
def tar2afi(build_config_file: BuildConfigFile) -> None:
|
def tar2afi(build_config_file: BuildConfigFile) -> None:
|
||||||
|
|
|
@ -447,6 +447,27 @@ class FireSimTopologyWithPasses:
|
||||||
|
|
||||||
execute(infrasetup_node_wrapper, self.run_farm, uridir, hosts=all_run_farm_ips)
|
execute(infrasetup_node_wrapper, self.run_farm, uridir, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
|
def enumerate_fpgas_passes(self, use_mock_instances_for_testing: bool) -> None:
|
||||||
|
""" extra passes needed to do enumerate_fpgas """
|
||||||
|
self.run_farm.post_launch_binding(use_mock_instances_for_testing)
|
||||||
|
|
||||||
|
@parallel
|
||||||
|
def enumerate_fpgas_node_wrapper(run_farm: RunFarm, dir: str) -> None:
|
||||||
|
my_node = run_farm.lookup_by_host(env.host_string)
|
||||||
|
assert my_node is not None
|
||||||
|
assert my_node.instance_deploy_manager is not None
|
||||||
|
my_node.instance_deploy_manager.enumerate_fpgas(dir)
|
||||||
|
|
||||||
|
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
|
||||||
|
execute(instance_liveness, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
|
# Steps occur within the context of a tempdir.
|
||||||
|
# This allows URI's to survive until after deploy, and cleanup upon error
|
||||||
|
with TemporaryDirectory() as uridir:
|
||||||
|
self.pass_fetch_URI_resolve_runtime_cfg(uridir)
|
||||||
|
self.pass_build_required_drivers()
|
||||||
|
execute(enumerate_fpgas_node_wrapper, self.run_farm, uridir, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
def build_driver_passes(self) -> None:
|
def build_driver_passes(self) -> None:
|
||||||
""" Only run passes to build drivers. """
|
""" Only run passes to build drivers. """
|
||||||
|
|
||||||
|
@ -496,8 +517,6 @@ class FireSimTopologyWithPasses:
|
||||||
""" Passes that kill the simulator. """
|
""" Passes that kill the simulator. """
|
||||||
self.run_farm.post_launch_binding(use_mock_instances_for_testing)
|
self.run_farm.post_launch_binding(use_mock_instances_for_testing)
|
||||||
|
|
||||||
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
|
|
||||||
|
|
||||||
@parallel
|
@parallel
|
||||||
def kill_switch_wrapper(run_farm: RunFarm) -> None:
|
def kill_switch_wrapper(run_farm: RunFarm) -> None:
|
||||||
my_node = run_farm.lookup_by_host(env.host_string)
|
my_node = run_farm.lookup_by_host(env.host_string)
|
||||||
|
@ -510,6 +529,13 @@ class FireSimTopologyWithPasses:
|
||||||
assert my_node.instance_deploy_manager is not None
|
assert my_node.instance_deploy_manager is not None
|
||||||
my_node.instance_deploy_manager.kill_simulations_instance(disconnect_all_nbds=disconnect_all_nbds)
|
my_node.instance_deploy_manager.kill_simulations_instance(disconnect_all_nbds=disconnect_all_nbds)
|
||||||
|
|
||||||
|
# Steps occur within the context of a tempdir.
|
||||||
|
# This allows URI's to survive until after deploy, and cleanup upon error
|
||||||
|
with TemporaryDirectory() as uridir:
|
||||||
|
self.pass_fetch_URI_resolve_runtime_cfg(uridir)
|
||||||
|
|
||||||
|
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
|
||||||
|
|
||||||
execute(kill_switch_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
execute(kill_switch_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
||||||
execute(kill_simulation_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
execute(kill_simulation_wrapper, self.run_farm, hosts=all_run_farm_ips)
|
||||||
|
|
||||||
|
|
|
@ -84,6 +84,11 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
|
||||||
"""
|
"""
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
"""Run platform specific implementation of how to enumerate FPGAs for FireSim."""
|
||||||
|
raise NotImplementedError
|
||||||
|
|
||||||
@abc.abstractmethod
|
@abc.abstractmethod
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
"""Run platform specific implementation of how to terminate host
|
"""Run platform specific implementation of how to terminate host
|
||||||
|
@ -658,6 +663,10 @@ class EC2InstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" FPGAs are enumerated already with F1 """
|
||||||
|
return
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
self.instance_logger("Terminating instance", debug=True)
|
self.instance_logger("Terminating instance", debug=True)
|
||||||
self.parent_node.terminate_self()
|
self.parent_node.terminate_self()
|
||||||
|
@ -717,6 +726,10 @@ class VitisInstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" FPGAs are enumerated already with Vitis """
|
||||||
|
return
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
""" VitisInstanceDeployManager machines cannot be terminated. """
|
""" VitisInstanceDeployManager machines cannot be terminated. """
|
||||||
return
|
return
|
||||||
|
@ -724,7 +737,7 @@ class VitisInstanceDeployManager(InstanceDeployManager):
|
||||||
class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
""" This class manages a Xilinx Alveo-enabled instance """
|
""" This class manages a Xilinx Alveo-enabled instance """
|
||||||
PLATFORM_NAME: Optional[str]
|
PLATFORM_NAME: Optional[str]
|
||||||
BOARD_NAME: Optional[str]
|
JSON_DB: str = "/opt/firesim-db.json"
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def sim_command_requires_sudo(cls) -> bool:
|
def sim_command_requires_sudo(cls) -> bool:
|
||||||
|
@ -734,33 +747,32 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = None
|
self.PLATFORM_NAME = None
|
||||||
self.BOARD_NAME = None
|
|
||||||
|
|
||||||
def unload_xdma(self) -> None:
|
|
||||||
if self.instance_assigned_simulations():
|
|
||||||
self.instance_logger("Unloading XDMA Driver Kernel Module.")
|
|
||||||
|
|
||||||
with warn_only():
|
|
||||||
remote_kmsg("removing_xdma_start")
|
|
||||||
run('sudo rmmod xdma')
|
|
||||||
remote_kmsg("removing_xdma_end")
|
|
||||||
|
|
||||||
def load_xdma(self) -> None:
|
def load_xdma(self) -> None:
|
||||||
""" load the xdma kernel module. """
|
""" load the xdma kernel module. """
|
||||||
if self.instance_assigned_simulations():
|
if self.instance_assigned_simulations():
|
||||||
# unload first
|
# load xdma if unloaded
|
||||||
self.unload_xdma()
|
if run('lsmod | grep -wq xdma', warn_only=True).return_code != 0:
|
||||||
# load xdma
|
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
||||||
self.instance_logger("Loading XDMA Driver Kernel Module.")
|
# must be installed to this path on sim. machine
|
||||||
# must be installed to this path on sim. machine
|
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
||||||
run(f"sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1", shell=True)
|
else:
|
||||||
|
self.instance_logger("XDMA Driver Kernel Module already loaded.")
|
||||||
|
|
||||||
|
def slot_to_bdf(self, slotno: int) -> str:
|
||||||
|
# get fpga information from db
|
||||||
|
self.instance_logger(f"""Determine BDF for {slotno}""")
|
||||||
|
collect = run(f'cat {self.JSON_DB}')
|
||||||
|
db = json.loads(collect)
|
||||||
|
assert slotno < len(db), f"Less FPGAs available than slots ({slotno} >= {len(db)})"
|
||||||
|
return db[slotno]['bdf']
|
||||||
|
|
||||||
def flash_fpgas(self) -> None:
|
def flash_fpgas(self) -> None:
|
||||||
if self.instance_assigned_simulations():
|
if self.instance_assigned_simulations():
|
||||||
self.instance_logger("""Flash all FPGA Slots.""")
|
self.instance_logger("""Flash all FPGA Slots.""")
|
||||||
|
|
||||||
for slotno, firesimservernode in enumerate(self.parent_node.sim_slots):
|
for slotno, firesimservernode in enumerate(self.parent_node.sim_slots):
|
||||||
serv = self.parent_node.sim_slots[slotno]
|
serv = firesimservernode
|
||||||
hwcfg = serv.get_resolved_server_hardware_config()
|
hwcfg = serv.get_resolved_server_hardware_config()
|
||||||
|
|
||||||
bitstream_tar = hwcfg.get_bitstream_tar_filename()
|
bitstream_tar = hwcfg.get_bitstream_tar_filename()
|
||||||
|
@ -782,13 +794,10 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
rootLogger.debug(rsync_cap)
|
rootLogger.debug(rsync_cap)
|
||||||
rootLogger.debug(rsync_cap.stderr)
|
rootLogger.debug(rsync_cap.stderr)
|
||||||
|
|
||||||
self.instance_logger(f"""Determine BDF for {slotno}""")
|
bdf = self.slot_to_bdf(slotno)
|
||||||
collect = run('lspci | grep -i serial.*xilinx')
|
|
||||||
bdfs = [ "0000:" + i[:7] for i in collect.splitlines() if len(i.strip()) >= 0 ]
|
|
||||||
bdf = bdfs[slotno]
|
|
||||||
|
|
||||||
self.instance_logger(f"""Flashing FPGA Slot: {slotno} with bit: {bit}""")
|
self.instance_logger(f"""Flashing FPGA Slot: {slotno} ({bdf}) with bitstream: {bit}""")
|
||||||
run(f"""EXTENDED_DEVICE_BDF1={bdf} {remote_sim_dir}/scripts/program_fpga.sh {bit} {self.BOARD_NAME}""")
|
run(f"""{remote_sim_dir}/scripts/program_fpga.py --bitstream {bit} --bdf {bdf}""")
|
||||||
|
|
||||||
def infrasetup_instance(self, uridir: str) -> None:
|
def infrasetup_instance(self, uridir: str) -> None:
|
||||||
""" Handle infrastructure setup for this platform. """
|
""" Handle infrastructure setup for this platform. """
|
||||||
|
@ -813,6 +822,70 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
for slotno in range(len(self.parent_node.switch_slots)):
|
for slotno in range(len(self.parent_node.switch_slots)):
|
||||||
self.copy_switch_slot_infrastructure(slotno)
|
self.copy_switch_slot_infrastructure(slotno)
|
||||||
|
|
||||||
|
def create_fpga_database(self, uridir: str) -> None:
|
||||||
|
self.instance_logger(f"""Creating FPGA database""")
|
||||||
|
|
||||||
|
remote_home_dir = self.parent_node.get_sim_dir()
|
||||||
|
remote_sim_dir = f"{remote_home_dir}/enumerate_fpgas_staging"
|
||||||
|
remote_sim_rsync_dir = f"{remote_sim_dir}/rsyncdir/"
|
||||||
|
run(f"mkdir -p {remote_sim_rsync_dir}")
|
||||||
|
|
||||||
|
# only use the collateral from 1 driver (no need to copy all things)
|
||||||
|
assert len(self.parent_node.sim_slots) > 0
|
||||||
|
serv = self.parent_node.sim_slots[0]
|
||||||
|
|
||||||
|
files_to_copy = serv.get_required_files_local_paths()
|
||||||
|
|
||||||
|
# Append required URI paths to the end of this list
|
||||||
|
hwcfg = serv.get_resolved_server_hardware_config()
|
||||||
|
files_to_copy.extend(hwcfg.get_local_uri_paths(uridir))
|
||||||
|
|
||||||
|
for local_path, remote_path in files_to_copy:
|
||||||
|
# -z --inplace
|
||||||
|
rsync_cap = rsync_project(local_dir=local_path, remote_dir=pjoin(remote_sim_rsync_dir, remote_path),
|
||||||
|
ssh_opts="-o StrictHostKeyChecking=no", extra_opts="-L", capture=True)
|
||||||
|
rootLogger.debug(rsync_cap)
|
||||||
|
rootLogger.debug(rsync_cap.stderr)
|
||||||
|
|
||||||
|
run(f"cp -r {remote_sim_rsync_dir}/* {remote_sim_dir}/", shell=True)
|
||||||
|
|
||||||
|
rsync_cap = rsync_project(
|
||||||
|
local_dir=f'../platforms/{self.PLATFORM_NAME}/scripts',
|
||||||
|
remote_dir=remote_sim_dir + "/",
|
||||||
|
ssh_opts="-o StrictHostKeyChecking=no",
|
||||||
|
extra_opts="-L -p",
|
||||||
|
capture=True)
|
||||||
|
rootLogger.debug(rsync_cap)
|
||||||
|
rootLogger.debug(rsync_cap.stderr)
|
||||||
|
|
||||||
|
bitstream_tar = hwcfg.get_bitstream_tar_filename()
|
||||||
|
bitstream_tar_unpack_dir = f"{remote_sim_dir}/{self.PLATFORM_NAME}"
|
||||||
|
bitstream = f"{remote_sim_dir}/{self.PLATFORM_NAME}/firesim.bit"
|
||||||
|
|
||||||
|
with cd(remote_sim_dir):
|
||||||
|
run(f"tar -xf {hwcfg.get_driver_tar_filename()}")
|
||||||
|
|
||||||
|
# at this point the tar file is in the sim slot
|
||||||
|
run(f"rm -rf {bitstream_tar_unpack_dir}")
|
||||||
|
run(f"tar xvf {remote_sim_dir}/{bitstream_tar} -C {remote_sim_dir}")
|
||||||
|
|
||||||
|
driver = f"{remote_sim_dir}/FireSim-{self.PLATFORM_NAME}"
|
||||||
|
|
||||||
|
with cd(remote_sim_dir):
|
||||||
|
run(f"""./scripts/generate-fpga-db.py --working-bitstream {bitstream} --driver {driver} --out-db-json {json}""")
|
||||||
|
|
||||||
|
def enumerate_fpgas(self, uridir: str) -> None:
|
||||||
|
""" Handle fpga setup for this platform. """
|
||||||
|
|
||||||
|
if self.instance_assigned_simulations():
|
||||||
|
# This is a sim-host node.
|
||||||
|
|
||||||
|
# load xdma driver
|
||||||
|
self.load_xdma()
|
||||||
|
|
||||||
|
# run the passes
|
||||||
|
self.create_fpga_database(uridir)
|
||||||
|
|
||||||
def terminate_instance(self) -> None:
|
def terminate_instance(self) -> None:
|
||||||
""" XilinxAlveoInstanceDeployManager machines cannot be terminated. """
|
""" XilinxAlveoInstanceDeployManager machines cannot be terminated. """
|
||||||
return
|
return
|
||||||
|
@ -826,10 +899,7 @@ class XilinxAlveoInstanceDeployManager(InstanceDeployManager):
|
||||||
assert slotno < len(self.parent_node.sim_slots), f"{slotno} can not index into sim_slots {len(self.parent_node.sim_slots)} on {self.parent_node.host}"
|
assert slotno < len(self.parent_node.sim_slots), f"{slotno} can not index into sim_slots {len(self.parent_node.sim_slots)} on {self.parent_node.host}"
|
||||||
server = self.parent_node.sim_slots[slotno]
|
server = self.parent_node.sim_slots[slotno]
|
||||||
|
|
||||||
self.instance_logger(f"""Determine BDF for {slotno}""")
|
bdf = self.slot_to_bdf(slotno)
|
||||||
collect = run('lspci | grep -i serial.*xilinx')
|
|
||||||
bdfs = [ i[:2] for i in collect.splitlines() if len(i.strip()) >= 0 ]
|
|
||||||
bdf = bdfs[slotno]
|
|
||||||
|
|
||||||
# make the local job results dir for this sim slot
|
# make the local job results dir for this sim slot
|
||||||
server.mkdir_and_prep_local_job_results_dir()
|
server.mkdir_and_prep_local_job_results_dir()
|
||||||
|
@ -844,10 +914,8 @@ class XilinxAlveoU250InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = "xilinx_alveo_u250"
|
self.PLATFORM_NAME = "xilinx_alveo_u250"
|
||||||
self.BOARD_NAME = "au250"
|
|
||||||
|
|
||||||
class XilinxAlveoU280InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
class XilinxAlveoU280InstanceDeployManager(XilinxAlveoInstanceDeployManager):
|
||||||
def __init__(self, parent_node: Inst) -> None:
|
def __init__(self, parent_node: Inst) -> None:
|
||||||
super().__init__(parent_node)
|
super().__init__(parent_node)
|
||||||
self.PLATFORM_NAME = "xilinx_alveo_u280"
|
self.PLATFORM_NAME = "xilinx_alveo_u280"
|
||||||
self.BOARD_NAME = "au280"
|
|
||||||
|
|
|
@ -918,6 +918,11 @@ class RuntimeConfig:
|
||||||
""" directly called by top-level builddriver command. """
|
""" directly called by top-level builddriver command. """
|
||||||
self.firesim_topology_with_passes.build_driver_passes()
|
self.firesim_topology_with_passes.build_driver_passes()
|
||||||
|
|
||||||
|
def enumerate_fpgas(self) -> None:
|
||||||
|
""" directly called by top-level enumeratefpgas command. """
|
||||||
|
use_mock_instances_for_testing = False
|
||||||
|
self.firesim_topology_with_passes.enumerate_fpgas_passes(use_mock_instances_for_testing)
|
||||||
|
|
||||||
def boot(self) -> None:
|
def boot(self) -> None:
|
||||||
""" directly called by top-level boot command. """
|
""" directly called by top-level boot command. """
|
||||||
use_mock_instances_for_testing = False
|
use_mock_instances_for_testing = False
|
||||||
|
|
|
@ -0,0 +1,97 @@
|
||||||
|
FPGA Board Setup
|
||||||
|
===================
|
||||||
|
|
||||||
|
FPGA Setup
|
||||||
|
----------
|
||||||
|
|
||||||
|
.. warning:: Currently, FireSim only supports a single type of FPGA (i.e only |fpga_name| FPGAs) installed on a machine.
|
||||||
|
This includes not mixing the use of Xilinx Vitis/XRT-enabled FPGAs on the system.
|
||||||
|
|
||||||
|
.. Warning:: Power-users can skip this setup and just create the database file listed below by hand if you want to target specific fpgas.
|
||||||
|
|
||||||
|
We need to flash the |fpga_name| FPGA(s) with a dummy XDMA-enabled design and determine the PCI-e ID (or BDF) associated with the serial number of the FPGA.
|
||||||
|
First, we need to flash the FPGA's with the dummy XDMA-enabled design so that the PCI-e subsystem can be initially configured.
|
||||||
|
Afterwards, we will generate the mapping from FPGA serial number to BDF.
|
||||||
|
We provide a a set of scripts to do this.
|
||||||
|
|
||||||
|
First lets obtain the sample bitstream, let's find the URL to download the file to the machine with the FPGA.
|
||||||
|
Below find the HWDB entry called |hwdb_entry_name|.
|
||||||
|
|
||||||
|
.. literalinclude:: /../deploy/sample-backup-configs/sample_config_hwdb.yaml
|
||||||
|
:language: yaml
|
||||||
|
:start-after: DOCREF START: Xilinx Alveo HWDB Entries
|
||||||
|
:end-before: DOCREF END: Xilinx Alveo HWDB Entries
|
||||||
|
|
||||||
|
Look for the ``bitstream_tar: <URL>`` line within |hwdb_entry_name| and keep note of the URL.
|
||||||
|
We will replace the ``BITSTREAM_TAR`` bash variable below with that URL.
|
||||||
|
|
||||||
|
Next, lets flash all FPGAs in the system with the dummy bitstream.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
:substitutions:
|
||||||
|
|
||||||
|
# enter the firesim directory checked out
|
||||||
|
cd firesim
|
||||||
|
|
||||||
|
cd platforms/|platform_name|/scripts
|
||||||
|
|
||||||
|
vivado -mode tcl -source get_serial_dev_for_fpgas.tcl
|
||||||
|
# get the UID/serial number's from this script
|
||||||
|
|
||||||
|
BITSTREAM_TAR=<# replace me!>
|
||||||
|
tar xvf $BITSTREAM_TAR
|
||||||
|
./program_fpga.py --serial_no $SERIAL_NO |platform_name|/*.bit
|
||||||
|
|
||||||
|
Next, **warm reboot** the computer.
|
||||||
|
This will reconfigure your PCI-E settings such that FireSim can detect the XDMA-enabled bitstream.
|
||||||
|
After the machine is rebooted, you may need to re-insert the XDMA kernel module.
|
||||||
|
Then verify that you can see the XDMA module with:
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
lsmod | grep -i xdma
|
||||||
|
|
||||||
|
Also, verify that the FPGA programming worked by looking at the ``lspci`` output.
|
||||||
|
For example, we should see ``Serial controller`` for BDF's that were flashed.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
|
||||||
|
lspci | grep -i xilinx
|
||||||
|
|
||||||
|
# example output
|
||||||
|
04:00.0 Serial controller: Xilinx Corporation Device 903f (rev ff)
|
||||||
|
83:00.0 Serial controller: Xilinx Corporation Device 903f (rev ff)
|
||||||
|
|
||||||
|
.. Warning:: Anytime the host computer is rebooted you may need to re-run parts of the setup process (i.e. re-insert XDMA kernel module).
|
||||||
|
Before continuing to FireSim simulations after a host computer reboot, ensure that ``cat /proc/devices | grep xdma`` command is successful.
|
||||||
|
Also ensure that you see ``Serial controller`` for the BDF of the FPGA you would like to use in ``lspci | grep -i xilinx`` (otherwise, re-run this setup).
|
||||||
|
|
||||||
|
Next, let's generate the mapping from FPGA serial numbers to the BDF.
|
||||||
|
Re-enter the FireSim repository and run the following commands to re-setup the repo after reboot.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
:substitutions:
|
||||||
|
|
||||||
|
cd firesim
|
||||||
|
|
||||||
|
# rerunning this since the machine rebooted
|
||||||
|
source sourceme-f1-manager.sh --skip-ssh-setup
|
||||||
|
|
||||||
|
Next, open up the ``deploy/config_runtime.yaml`` file and replace the following keys to be the following:
|
||||||
|
|
||||||
|
* ``default_platform`` should be |deploy_manager_code|
|
||||||
|
|
||||||
|
* ``default_simulation_dir`` should point to a temporary simulation directory of your choice
|
||||||
|
|
||||||
|
* ``default_hw_config`` should be |hwdb_entry_name|
|
||||||
|
|
||||||
|
Then, run the following command to generate a mapping from a PCI-E BDF to FPGA UID/serial number.
|
||||||
|
|
||||||
|
.. code-block:: bash
|
||||||
|
:substitutions:
|
||||||
|
|
||||||
|
firesim enumeratefpgas
|
||||||
|
|
||||||
|
This will generate a database file in ``/opt/firesim-db.json`` that has this mapping.
|
||||||
|
|
||||||
|
Now you're ready to continue with other FireSim setup!
|
|
@ -1,4 +1,4 @@
|
||||||
FPGA and Tool Setup
|
FPGA Software Setup
|
||||||
===================
|
===================
|
||||||
|
|
||||||
Requirements and Installations
|
Requirements and Installations
|
||||||
|
@ -21,13 +21,11 @@ We require the following programs/packages installed from the Xilinx website in
|
||||||
Importantly, using this FPGA with FireSim requires that you have ``sudo`` **passwordless** access to the machine with the FPGA.
|
Importantly, using this FPGA with FireSim requires that you have ``sudo`` **passwordless** access to the machine with the FPGA.
|
||||||
This is needed to flash the FPGA bitstream onto the FPGA.
|
This is needed to flash the FPGA bitstream onto the FPGA.
|
||||||
|
|
||||||
FPGA Setup
|
XDMA Setup
|
||||||
----------
|
----------
|
||||||
|
|
||||||
.. warning:: Currently, FireSim only supports a single |fpga_name| installed on a machine. Future support will address this.
|
To communicate with the FPGA over PCI-e, we need to install the Xilinx XDMA kernel module.
|
||||||
|
First, lets install the XDMA kernel module into a FireSim-known location:
|
||||||
After installing the |fpga_name| using the Xilinx instructions and installing the specific version of Vivado, we need to flash the |fpga_name| with a dummy XDMA-enabled design to finish setup.
|
|
||||||
First, lets install the XDMA kernel module in a FireSim known location:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
|
@ -46,81 +44,13 @@ Next, lets add the kernel module:
|
||||||
# by the `make install` previously run
|
# by the `make install` previously run
|
||||||
sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1
|
sudo insmod /lib/modules/$(uname -r)/extra/xdma.ko poll_mode=1
|
||||||
|
|
||||||
Next, let's determine the BDF's (unique ID) of the/any FPGA you want to use with FireSim.
|
By default, FireSim will refer to this location to check if the XDMA driver is loaded.
|
||||||
|
Verify that you can see the XDMA module with:
|
||||||
.. code-block:: bash
|
|
||||||
:substitutions:
|
|
||||||
|
|
||||||
# determine BDF of FPGA that you want to use / re-flash
|
|
||||||
lspci | grep -i xilinx
|
|
||||||
|
|
||||||
# example output of a 2 |fpga_name| FPGA system:
|
|
||||||
# 04:00.0 Processing accelerators: Xilinx Corporation Device 5004
|
|
||||||
# 04:00.1 Processing accelerators: Xilinx Corporation Device 5005
|
|
||||||
# 83:00.0 Processing accelerators: Xilinx Corporation Device 5004
|
|
||||||
# 83:00.1 Processing accelerators: Xilinx Corporation Device 5005
|
|
||||||
|
|
||||||
# BDF would be 04:00.0 if you want to flash the '04' FPGA
|
|
||||||
# the extended BDF would be 0000: + the BDF from before (i.e. 0000:04:00.0)
|
|
||||||
# note: that you BDF to use is the one ending in .0
|
|
||||||
|
|
||||||
Keep note of the **extended BDF** of the FPGA you would like to setup.
|
|
||||||
Next, let's flash each |fpga_name| that you would like to use with the dummy bitstream.
|
|
||||||
To obtain the sample bitstream, let's find the URL to download the file to the machine with the FPGA.
|
|
||||||
Below find the HWDB entry called |hwdb_entry_name|.
|
|
||||||
|
|
||||||
.. literalinclude:: /../deploy/sample-backup-configs/sample_config_hwdb.yaml
|
|
||||||
:language: yaml
|
|
||||||
:start-after: DOCREF START: Xilinx Alveo HWDB Entries
|
|
||||||
:end-before: DOCREF END: Xilinx Alveo HWDB Entries
|
|
||||||
|
|
||||||
Look for the ``bitstream_tar: <URL>`` line within |hwdb_entry_name| and keep note of the URL.
|
|
||||||
Next, we will do the following for each FPGA that will be used with FireSim.
|
|
||||||
|
|
||||||
#. Create a temporary flashing area that we will delete after flashing the FPGA.
|
|
||||||
#. Download the bitstream file.
|
|
||||||
#. Download a temporary FireSim repository to have access to the flashing scripts.
|
|
||||||
#. Flash the FPGA (with the extended BDF obtained) and the bitstream file.
|
|
||||||
#. Delete the temporary flashing area.
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
:substitutions:
|
|
||||||
|
|
||||||
mkdir /tmp/tempdownload
|
|
||||||
cd /tmp/tempdownload
|
|
||||||
wget <BIT_TAR URL SAVED FROM PREVIOUSLY>
|
|
||||||
tar xvf firesim.tar.gz
|
|
||||||
cd |platform_name|
|
|
||||||
|
|
||||||
git clone --branch |overall_version| https://github.com/firesim/firesim
|
|
||||||
EXTENDED_DEVICE_BDF1=<YOUR BDF HERE> ./firesim/platforms/|platform_name|/scripts/program_fpga.sh ./firesim.bit |board_name|
|
|
||||||
|
|
||||||
rm -rf /tmp/tempdownload
|
|
||||||
|
|
||||||
Next, **warm reboot** the computer.
|
|
||||||
This will reconfigure your PCI-E settings such that FireSim can detect the XDMA-enabled bitstream.
|
|
||||||
After the machine is rebooted, you may need to re-insert the XDMA kernel module.
|
|
||||||
Then verify that you can see the XDMA module with:
|
|
||||||
|
|
||||||
.. code-block:: bash
|
.. code-block:: bash
|
||||||
|
|
||||||
cat /proc/devices | grep xdma
|
lsmod | grep -wq xdma
|
||||||
|
|
||||||
Also, verify that the FPGA programming worked by seeing if the ``lspci`` output has changed.
|
.. warning:: After the machine is rebooted, you may need to re-insert the XDMA kernel module.
|
||||||
For example, we should see ``Serial controller`` for BDF's that were flashed.
|
|
||||||
|
|
||||||
|
|
||||||
.. code-block:: bash
|
|
||||||
|
|
||||||
lspci | grep -i xilinx
|
|
||||||
|
|
||||||
# example output if only the 0000:04:00.0 FPGA was programmed
|
|
||||||
04:00.0 Serial controller: Xilinx Corporation Device 903f (rev ff)
|
|
||||||
83:00.0 Processing accelerators: Xilinx Corporation Device 5004
|
|
||||||
83:00.1 Processing accelerators: Xilinx Corporation Device 5005
|
|
||||||
|
|
||||||
.. Warning:: Anytime the host computer is rebooted you may need to re-run parts of the setup process (i.e. re-insert XDMA kernel module).
|
|
||||||
Before continuing to FireSim simulations after a host computer reboot, ensure that the previously mentioned ``cat /proc/devices | grep xdma`` command is successful.
|
|
||||||
Also ensure that you see ``Serial controller`` for the BDF of the FPGA you would like to use (otherwise, re-run this setup).
|
|
||||||
|
|
||||||
Now you're ready to continue with other FireSim setup!
|
Now you're ready to continue with other FireSim setup!
|
||||||
|
|
|
@ -0,0 +1,7 @@
|
||||||
|
.. |fpga_name| replace:: Xilinx Alveo U250
|
||||||
|
.. |hwdb_entry_name| replace:: ``alveo_u250_firesim_rocket_singlecore_no_nic``
|
||||||
|
.. |platform_name| replace:: xilinx_alveo_u250
|
||||||
|
.. |board_name| replace:: au250
|
||||||
|
.. |deploy_manager_code| replace:: ``XilinxAlveoU250InstanceDeployManager``
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Template-Part2.rst
|
|
@ -0,0 +1,7 @@
|
||||||
|
.. |fpga_name| replace:: Xilinx Alveo U280
|
||||||
|
.. |hwdb_entry_name| replace:: ``alveo_u280_firesim_rocket_singlecore_no_nic``
|
||||||
|
.. |platform_name| replace:: xilinx_alveo_u280
|
||||||
|
.. |board_name| replace:: au280
|
||||||
|
.. |deploy_manager_code| replace:: ``XilinxAlveoU280InstanceDeployManager``
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Template-Part2.rst
|
|
@ -158,4 +158,4 @@ To run it, do the following:
|
||||||
It will create initial configuration files, which we will edit in later
|
It will create initial configuration files, which we will edit in later
|
||||||
sections.
|
sections.
|
||||||
|
|
||||||
Now you're ready to launch FireSim simulations! Hit Next to learn how to run single-node simulations.
|
Hit Next to continue with the guide.
|
||||||
|
|
|
@ -14,23 +14,3 @@ After you complete these tutorials, you can look at the "Advanced Docs"
|
||||||
in the sidebar to the left.
|
in the sidebar to the left.
|
||||||
|
|
||||||
Here's a high-level outline of what we'll be doing in our tutorials:
|
Here's a high-level outline of what we'll be doing in our tutorials:
|
||||||
|
|
||||||
#. **FPGA Setup**: Installing the FPGA board and relevant software.
|
|
||||||
|
|
||||||
#. **On-Premises Machine Setup**
|
|
||||||
|
|
||||||
#. Setting up a "Manager Machine" from which you will coordinate building
|
|
||||||
and deploying simulations locally.
|
|
||||||
|
|
||||||
#. **Single-node simulation tutorial**: This tutorial guides you through the
|
|
||||||
process of running one simulation locally consisting of a single
|
|
||||||
|fpga_name|, using our pre-built public FireSim |bit_type| bitstream.
|
|
||||||
|
|
||||||
#. **Building your own hardware designs tutorial (Chisel to FPGA Image)**:
|
|
||||||
This tutorial guides you through the full process of taking Rocket Chip RTL
|
|
||||||
and any custom RTL plugged into Rocket Chip and producing a FireSim bitstream
|
|
||||||
to plug into your simulations. This automatically runs Chisel elaboration,
|
|
||||||
FAME-1 Transformation, and the |build_type| FPGA flow.
|
|
||||||
|
|
||||||
Generally speaking, you only need to follow Step 4 if you're modifying Chisel
|
|
||||||
RTL or changing non-runtime configurable hardware parameters.
|
|
||||||
|
|
|
@ -0,0 +1,21 @@
|
||||||
|
#. **FPGA Software Setup**: Installing the relevant FPGA software.
|
||||||
|
|
||||||
|
#. **On-Premises Machine Setup**
|
||||||
|
|
||||||
|
#. Setting up a "Manager Machine" from which you will coordinate building
|
||||||
|
and deploying simulations locally.
|
||||||
|
|
||||||
|
#. **FPGA Board Setup**: Finish initial programming/setting up FPGA boards.
|
||||||
|
|
||||||
|
#. **Single-node simulation tutorial**: This tutorial guides you through the
|
||||||
|
process of running one simulation locally consisting of a single
|
||||||
|
|fpga_name|, using our pre-built public FireSim |bit_type| bitstream.
|
||||||
|
|
||||||
|
#. **Building your own hardware designs tutorial (Chisel to FPGA Image)**:
|
||||||
|
This tutorial guides you through the full process of taking Rocket Chip RTL
|
||||||
|
and any custom RTL plugged into Rocket Chip and producing a FireSim bitstream
|
||||||
|
to plug into your simulations. This automatically runs Chisel elaboration,
|
||||||
|
FAME-1 Transformation, and the |build_type| FPGA flow.
|
||||||
|
|
||||||
|
Generally speaking, you only need to follow Step 5 if you're modifying Chisel
|
||||||
|
RTL or changing non-runtime configurable hardware parameters.
|
|
@ -4,10 +4,13 @@
|
||||||
|
|
||||||
.. include:: Intro-Template.rst
|
.. include:: Intro-Template.rst
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Outline-Template.rst
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
FPGA-Setup/Xilinx-Alveo-U250
|
FPGA-Setup/Xilinx-Alveo-U250
|
||||||
Initial-Setup/Setting-Up-Xilinx-Alveo-U250
|
Initial-Setup/Setting-Up-Xilinx-Alveo-U250
|
||||||
|
FPGA-Setup/Xilinx-Alveo-U250-Part2
|
||||||
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U250
|
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U250
|
||||||
Building-a-FireSim-Bitstream/Xilinx-Alveo-U250
|
Building-a-FireSim-Bitstream/Xilinx-Alveo-U250
|
||||||
|
|
|
@ -4,10 +4,13 @@
|
||||||
|
|
||||||
.. include:: Intro-Template.rst
|
.. include:: Intro-Template.rst
|
||||||
|
|
||||||
|
.. include:: Xilinx-Alveo-Outline-Template.rst
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
FPGA-Setup/Xilinx-Alveo-U280
|
FPGA-Setup/Xilinx-Alveo-U280
|
||||||
Initial-Setup/Setting-Up-Xilinx-Alveo-U280
|
Initial-Setup/Setting-Up-Xilinx-Alveo-U280
|
||||||
|
FPGA-Setup/Xilinx-Alveo-U280-Part2
|
||||||
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U280
|
Running-Simulations/Running-Single-Node-Simulation-Xilinx-Alveo-U280
|
||||||
Building-a-FireSim-Bitstream/Xilinx-Alveo-U280
|
Building-a-FireSim-Bitstream/Xilinx-Alveo-U280
|
||||||
|
|
|
@ -4,6 +4,26 @@
|
||||||
|
|
||||||
.. include:: Intro-Template.rst
|
.. include:: Intro-Template.rst
|
||||||
|
|
||||||
|
#. **FPGA Setup**: Installing the FPGA board and relevant software.
|
||||||
|
|
||||||
|
#. **On-Premises Machine Setup**
|
||||||
|
|
||||||
|
#. Setting up a "Manager Machine" from which you will coordinate building
|
||||||
|
and deploying simulations locally.
|
||||||
|
|
||||||
|
#. **Single-node simulation tutorial**: This tutorial guides you through the
|
||||||
|
process of running one simulation locally consisting of a single
|
||||||
|
|fpga_name|, using our pre-built public FireSim |bit_type| bitstream.
|
||||||
|
|
||||||
|
#. **Building your own hardware designs tutorial (Chisel to FPGA Image)**:
|
||||||
|
This tutorial guides you through the full process of taking Rocket Chip RTL
|
||||||
|
and any custom RTL plugged into Rocket Chip and producing a FireSim bitstream
|
||||||
|
to plug into your simulations. This automatically runs Chisel elaboration,
|
||||||
|
FAME-1 Transformation, and the |build_type| FPGA flow.
|
||||||
|
|
||||||
|
Generally speaking, you only need to follow Step 4 if you're modifying Chisel
|
||||||
|
RTL or changing non-runtime configurable hardware parameters.
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
|
|
||||||
|
|
|
@ -21,4 +21,3 @@
|
||||||
set part xcu250-figd2104-2l-e
|
set part xcu250-figd2104-2l-e
|
||||||
set board_part xilinx.com:au250:part0:1.3
|
set board_part xilinx.com:au250:part0:1.3
|
||||||
set zynq_family 0
|
set zynq_family 0
|
||||||
set hw_device xcu250_0
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
../cl_firesim/scripts/au250.tcl
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
BITSTREAM=$SCRATCH_HOME/firesim-private/deploy/results-build/2023-05-12--07-11-08-alveou250_firesim_rocket_singlecore_no_nic_w_pres/cl_xilinx_alveo_u250-firesim-FireSim-FireSimRocketConfig-BaseXilinxAlveoConfig/xilinx_alveo_u250/firesim.bit
|
||||||
|
|
||||||
|
./program_fpga.py --bdf 04:00.0 --disconnect-bdf
|
||||||
|
./program_fpga.py --bdf 83:00.0 --disconnect-bdf
|
||||||
|
|
||||||
|
if lspci | grep -i xilinx; then
|
||||||
|
echo "Something went wrong"
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
echo "Missing them"
|
||||||
|
fi
|
||||||
|
|
||||||
|
./program_fpga.py --serial_no Xilinx/21320733400EA --bitstream $BITSTREAM
|
||||||
|
./program_fpga.py --serial_no Xilinx/213207334001A --bitstream $BITSTREAM
|
||||||
|
|
||||||
|
if lspci | grep -i xilinx; then
|
||||||
|
echo "Something went wrong"
|
||||||
|
exit 1
|
||||||
|
else
|
||||||
|
echo "Missing them"
|
||||||
|
fi
|
||||||
|
|
||||||
|
./program_fpga.py --bdf 04:00.0 --reconnect-bdf
|
||||||
|
./program_fpga.py --bdf 83:00.0 --reconnect-bdf
|
||||||
|
|
||||||
|
if lspci | grep -i xilinx; then
|
||||||
|
echo "Found them"
|
||||||
|
else
|
||||||
|
echo "Something went wrong"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
## from scratch
|
||||||
|
#
|
||||||
|
#./program_fpga.py --bdf 04:00.0 --disconnect-bdf
|
||||||
|
#./program_fpga.py --bdf 04:00.1 --disconnect-bdf
|
||||||
|
#./program_fpga.py --bdf 83:00.0 --disconnect-bdf
|
||||||
|
#./program_fpga.py --bdf 83:00.1 --disconnect-bdf
|
||||||
|
#
|
||||||
|
#./program_fpga.py --serial_no Xilinx/21320733400EA --bitstream $BITSTREAM
|
||||||
|
#./program_fpga.py --serial_no Xilinx/213207334001A --bitstream $BITSTREAM
|
||||||
|
#
|
||||||
|
##./program_fpga.py --bdf 04:00.0 --reconnect-bdf
|
||||||
|
##./program_fpga.py --bdf 04:00.1 --reconnect-bdf
|
||||||
|
##./program_fpga.py --bdf 83:00.0 --reconnect-bdf
|
||||||
|
##./program_fpga.py --bdf 83:00.1 --reconnect-bdf
|
|
@ -0,0 +1,334 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import pwd
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from typing import Optional, Dict, Any, List
|
||||||
|
|
||||||
|
scriptPath = Path(__file__).resolve().parent
|
||||||
|
|
||||||
|
def get_bdfs() -> List[str]:
|
||||||
|
pLspci= subprocess.Popen(['lspci'], stdout=subprocess.PIPE)
|
||||||
|
pGrep = subprocess.Popen(['grep', '-i', 'serial.*xilinx'], stdin=pLspci.stdout, stdout=subprocess.PIPE)
|
||||||
|
if pLspci.stdout is not None:
|
||||||
|
pLspci.stdout.close()
|
||||||
|
|
||||||
|
sout, serr = pGrep.communicate()
|
||||||
|
|
||||||
|
if pGrep.returncode != 0:
|
||||||
|
print(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
outputLines = sout.decode('utf-8').splitlines()
|
||||||
|
bdfs = [ i[:7] for i in outputLines if len(i.strip()) >= 0]
|
||||||
|
return bdfs
|
||||||
|
|
||||||
|
def disconnect_bdf(bdf: str, vivado: str, hw_server: str) -> None:
|
||||||
|
print(f"Disconnecting BDF: {bdf}")
|
||||||
|
progScript = scriptPath / 'program_fpga.py'
|
||||||
|
assert progScript.exists()
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
[
|
||||||
|
str(progScript),
|
||||||
|
"--bdf", bdf,
|
||||||
|
"--disconnect-bdf",
|
||||||
|
"--vivado-bin", vivado,
|
||||||
|
"--hw-server-bin", hw_server,
|
||||||
|
],
|
||||||
|
stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
if pProg.returncode != 0:
|
||||||
|
print(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def reconnect_bdf(bdf: str, vivado: str, hw_server: str) -> None:
|
||||||
|
print(f"Reconnecting BDF: {bdf}")
|
||||||
|
progScript = scriptPath / 'program_fpga.py'
|
||||||
|
assert progScript.exists()
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
[
|
||||||
|
str(progScript),
|
||||||
|
"--bdf", bdf,
|
||||||
|
"--reconnect-bdf",
|
||||||
|
"--vivado-bin", vivado,
|
||||||
|
"--hw-server-bin", hw_server,
|
||||||
|
],
|
||||||
|
stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
if pProg.returncode != 0:
|
||||||
|
print(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def program_fpga(serial: str, bitstream: str, vivado: str, hw_server: str) -> None:
|
||||||
|
print(f"Programming {serial} with {bitstream}")
|
||||||
|
progScript = scriptPath / 'program_fpga.py'
|
||||||
|
assert progScript.exists()
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
[
|
||||||
|
str(progScript),
|
||||||
|
"--serial_no", serial,
|
||||||
|
"--bitstream", bitstream,
|
||||||
|
"--vivado-bin", vivado,
|
||||||
|
"--hw-server-bin", hw_server,
|
||||||
|
],
|
||||||
|
stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
if pProg.returncode != 0:
|
||||||
|
print(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
def get_serial_numbers_and_fpga_types(vivado: str) -> Dict[str, str]:
|
||||||
|
tclScript = scriptPath / 'get_serial_dev_for_fpgas.tcl'
|
||||||
|
assert tclScript.exists()
|
||||||
|
pVivado = subprocess.Popen(
|
||||||
|
[
|
||||||
|
vivado,
|
||||||
|
'-mode', 'tcl',
|
||||||
|
'-nolog', '-nojournal', '-notrace',
|
||||||
|
'-source', str(tclScript),
|
||||||
|
],
|
||||||
|
stdout=subprocess.PIPE
|
||||||
|
)
|
||||||
|
|
||||||
|
sout, serr = pVivado.communicate()
|
||||||
|
|
||||||
|
if pVivado.returncode != 0:
|
||||||
|
print(f":ERROR: It failed with stdout: {sout.decode('utf-8')} stderr: {serr.decode('utf-8')}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
outputLines = sout.decode('utf-8').splitlines()
|
||||||
|
relevantLines= [s for s in outputLines if ("hw_dev" in s) or ("hw_uid" in s)]
|
||||||
|
devs = []
|
||||||
|
uids = []
|
||||||
|
|
||||||
|
for line in relevantLines:
|
||||||
|
m = re.match(r"^hw_dev: (.*)$", line)
|
||||||
|
if m:
|
||||||
|
devs.append(m.group(1))
|
||||||
|
|
||||||
|
m = re.match(r"^hw_uid: (.*)$", line)
|
||||||
|
if m:
|
||||||
|
uids.append(m.group(1))
|
||||||
|
|
||||||
|
uid2dev = {}
|
||||||
|
for uid, dev in zip(uids, devs):
|
||||||
|
uid2dev[uid] = dev
|
||||||
|
|
||||||
|
return uid2dev
|
||||||
|
|
||||||
|
def run_driver_check_fingerprint(bdf: str, driver: Path, write_val: int) -> int:
|
||||||
|
bus_id = bdf[:2]
|
||||||
|
print(f"Running check fingerprint driver call with {bus_id} corresponding to {bdf}")
|
||||||
|
|
||||||
|
driverPath = driver.resolve().absolute()
|
||||||
|
assert driverPath.exists()
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
str(driverPath),
|
||||||
|
"+permissive",
|
||||||
|
f"+slotid={bus_id}",
|
||||||
|
"+check-fingerprint",
|
||||||
|
"+permissive-off",
|
||||||
|
"+prog0=none",
|
||||||
|
]
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
sout, serr = pProg.communicate(timeout=5)
|
||||||
|
except:
|
||||||
|
# spam any amount of flush signals
|
||||||
|
pProg.send_signal(signal.SIGPIPE)
|
||||||
|
pProg.send_signal(signal.SIGUSR1)
|
||||||
|
pProg.send_signal(signal.SIGUSR2)
|
||||||
|
|
||||||
|
# spam any amount of kill signals
|
||||||
|
pProg.kill()
|
||||||
|
pProg.send_signal(signal.SIGINT)
|
||||||
|
pProg.send_signal(signal.SIGTERM)
|
||||||
|
|
||||||
|
# retrieve flushed output
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
stdout = sout.decode('utf-8').splitlines()
|
||||||
|
|
||||||
|
if pProg.returncode == 124 or pProg.returncode is None:
|
||||||
|
print(":ERROR: Timed out...", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
elif pProg.returncode != 0:
|
||||||
|
print(f":WARNING: Running the driver failed...", file=sys.stderr)
|
||||||
|
print(f":DEBUG: bdf: {bdf} bus_id: {bus_id} stdout: {stdout}", file=sys.stderr)
|
||||||
|
return pProg.returncode
|
||||||
|
|
||||||
|
# successfully read fingerprint
|
||||||
|
print(f":DEBUG: bdf: {bdf} bus_id: {bus_id} stdout: {stdout}", file=sys.stderr)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def run_driver_write_fingerprint(bdf: str, driver: Path, write_val: int) -> None:
|
||||||
|
bus_id = bdf[:2]
|
||||||
|
print(f"Running write fingerprint driver call with {bus_id} corresponding to {bdf}")
|
||||||
|
|
||||||
|
driverPath = driver.resolve().absolute()
|
||||||
|
assert driverPath.exists()
|
||||||
|
|
||||||
|
cmd = [
|
||||||
|
str(driverPath),
|
||||||
|
"+permissive",
|
||||||
|
f"+slotid={bus_id}",
|
||||||
|
f"+write-fingerprint={write_val}",
|
||||||
|
"+permissive-off",
|
||||||
|
"+prog0=none",
|
||||||
|
]
|
||||||
|
pProg = subprocess.Popen(
|
||||||
|
cmd,
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
sout, serr = pProg.communicate(timeout=5)
|
||||||
|
except:
|
||||||
|
# spam any amount of flush signals
|
||||||
|
pProg.send_signal(signal.SIGPIPE)
|
||||||
|
pProg.send_signal(signal.SIGUSR1)
|
||||||
|
pProg.send_signal(signal.SIGUSR2)
|
||||||
|
|
||||||
|
# spam any amount of kill signals
|
||||||
|
pProg.kill()
|
||||||
|
pProg.send_signal(signal.SIGINT)
|
||||||
|
pProg.send_signal(signal.SIGTERM)
|
||||||
|
|
||||||
|
# retrieve flushed output
|
||||||
|
sout, serr = pProg.communicate()
|
||||||
|
|
||||||
|
stdout = sout.decode('utf-8').splitlines()
|
||||||
|
|
||||||
|
if pProg.returncode == 124 or pProg.returncode is None:
|
||||||
|
print(":ERROR: Timed out...", file=sys.stderr)
|
||||||
|
print(f":DEBUG: bdf: {bdf} bus_id: {bus_id} stdout: {stdout}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
elif pProg.returncode != 0:
|
||||||
|
print(f":ERROR: Running the driver failed...", file=sys.stderr)
|
||||||
|
print(f":DEBUG: bdf: {bdf} bus_id: {bus_id} stdout: {stdout}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
# TODO: Maybe confirm that the write went through?
|
||||||
|
|
||||||
|
print(f":DEBUG: bdf: {bdf} bus_id: {bus_id} stdout: {stdout}", file=sys.stderr)
|
||||||
|
return
|
||||||
|
|
||||||
|
def main(args: List[str]) -> int:
|
||||||
|
parser = argparse.ArgumentParser(description="")
|
||||||
|
parser.add_argument("--vivado-bin", help="Explicit path to 'vivado'", type=Path)
|
||||||
|
parser.add_argument("--hw-server-bin", help="Explicit path to 'hw_server'", type=Path)
|
||||||
|
parser.add_argument("--working-bitstream", help="Bitstream to flash and verify with a driver", type=Path, required=True)
|
||||||
|
parser.add_argument("--out-db-json", help="Where to dump the output database mapping", type=Path, required=True)
|
||||||
|
parser.add_argument("--driver", help="Driver to test with", type=Path, required=True)
|
||||||
|
parsed_args = parser.parse_args(args)
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
parsed_args.hw_server_bin = shutil.which('hw_server')
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
parsed_args.vivado_bin = shutil.which('vivado')
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Hardware Server!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Vivado!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
parsed_args.vivado_bin = Path(parsed_args.vivado_bin).absolute()
|
||||||
|
parsed_args.hw_server_bin = Path(parsed_args.hw_server_bin).absolute()
|
||||||
|
|
||||||
|
eUserId = os.geteuid()
|
||||||
|
sudoUserId = os.getenv('SUDO_UID')
|
||||||
|
isAdmin = (eUserId == 0) and (sudoUserId is None)
|
||||||
|
userId = eUserId if sudoUserId is None else int(sudoUserId)
|
||||||
|
|
||||||
|
# if not sudoer, spawn w/ sudo
|
||||||
|
if eUserId != 0:
|
||||||
|
execvArgs = ['/usr/bin/sudo', str(Path(__file__).absolute())] + sys.argv[1:]
|
||||||
|
execvArgs += ['--vivado-bin', str(parsed_args.vivado_bin), '--hw-server-bin', str(parsed_args.hw_server_bin)]
|
||||||
|
print(f"Running: {execvArgs}")
|
||||||
|
os.execv(execvArgs[0], execvArgs)
|
||||||
|
|
||||||
|
# expects that fpgas are programmed with working bitstreams (w/ xdma)
|
||||||
|
|
||||||
|
sno2fpga = get_serial_numbers_and_fpga_types(str(parsed_args.vivado_bin))
|
||||||
|
serials = sno2fpga.keys()
|
||||||
|
bdfs = get_bdfs()
|
||||||
|
|
||||||
|
serial2BDF: Dict[str, str] = {}
|
||||||
|
|
||||||
|
write_val = 0xDEADBEEF
|
||||||
|
|
||||||
|
# write to all fingerprints based on bdfs
|
||||||
|
for bdf in bdfs:
|
||||||
|
run_driver_write_fingerprint(bdf, parsed_args.driver, write_val)
|
||||||
|
|
||||||
|
for serial in serials:
|
||||||
|
# disconnect all
|
||||||
|
for bdf in bdfs:
|
||||||
|
disconnect_bdf(bdf, str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
program_fpga(serial, str(parsed_args.working_bitstream.resolve().absolute()), str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
# reconnect all
|
||||||
|
for bdf in bdfs:
|
||||||
|
reconnect_bdf(bdf, str(parsed_args.vivado_bin), str(parsed_args.hw_server_bin))
|
||||||
|
|
||||||
|
# read all fingerprints to find the good one
|
||||||
|
for bdf in bdfs:
|
||||||
|
if not (bdf in serial2BDF.values()):
|
||||||
|
rc = run_driver_check_fingerprint(bdf, parsed_args.driver, write_val)
|
||||||
|
if rc == 0:
|
||||||
|
serial2BDF[serial] = bdf
|
||||||
|
break
|
||||||
|
|
||||||
|
if not (serial in serial2BDF):
|
||||||
|
print(f":ERROR: Unable to determine BDF for {serial} FPGA. Something went wrong", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
print(f"Mapping: {serial2BDF}")
|
||||||
|
|
||||||
|
finalMap = []
|
||||||
|
for s, b in serial2BDF.items():
|
||||||
|
finalMap.append({
|
||||||
|
"uid" : s,
|
||||||
|
"device" : sno2fpga[s],
|
||||||
|
"bdf" : b
|
||||||
|
})
|
||||||
|
|
||||||
|
with open(parsed_args.out_db_json, 'w') as f:
|
||||||
|
json.dump(finalMap, f, indent=2)
|
||||||
|
|
||||||
|
print(f"Successfully wrote to {parsed_args.out_db_json}")
|
||||||
|
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main(sys.argv[1:]))
|
|
@ -0,0 +1,23 @@
|
||||||
|
# Directory variables
|
||||||
|
set script_path [file normalize [info script]]
|
||||||
|
set script_dir [file dirname $script_path]
|
||||||
|
set root_dir [file dirname $script_dir]
|
||||||
|
|
||||||
|
set_param labtools.enable_cs_server false
|
||||||
|
|
||||||
|
open_hw_manager
|
||||||
|
connect_hw_server -allow_non_jtag
|
||||||
|
|
||||||
|
# by default vivado opens a default hw target
|
||||||
|
close_hw_target
|
||||||
|
|
||||||
|
foreach {hw_target} [get_hw_targets] {
|
||||||
|
open_hw_target $hw_target
|
||||||
|
set hw_dev [get_hw_device]
|
||||||
|
set hw_uid [get_property UID $hw_target]
|
||||||
|
puts "hw_dev: $hw_dev"
|
||||||
|
puts "hw_uid: $hw_uid"
|
||||||
|
close_hw_target
|
||||||
|
}
|
||||||
|
|
||||||
|
exit
|
|
@ -6,30 +6,79 @@ import subprocess
|
||||||
import sys
|
import sys
|
||||||
import pwd
|
import pwd
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from typing import Optional, Dict, Any, List
|
from typing import Optional, Dict, Any, List
|
||||||
|
|
||||||
pciDevicesPath = Path('/sys/bus/pci/devices')
|
pciDevicesPath = Path('/sys/bus/pci/devices')
|
||||||
|
scriptPath = Path(__file__).resolve().parent
|
||||||
|
|
||||||
def get_bridge_bdf(id: str) -> str:
|
# obtain device paths/bdfs
|
||||||
for entry in pciDevicesPath.iterdir():
|
|
||||||
if re.match('^0000:' + re.escape(id) + ':[a-fA-F0-9]{2}\.[0-7]$', entry.name):
|
|
||||||
bridgePath = entry.resolve().absolute().parent
|
|
||||||
if bridgePath.exists():
|
|
||||||
return bridgePath.name
|
|
||||||
print(":ERROR: Unable to obtain bridge BDF")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def get_fpga_bdfs(id: str) -> List[str]:
|
def get_device_paths(id: str) -> List[Path]:
|
||||||
result = []
|
result = []
|
||||||
for entry in pciDevicesPath.iterdir():
|
for entry in pciDevicesPath.iterdir():
|
||||||
if re.match('^0000:' + re.escape(id) + ':[a-fA-F0-9]{2}\.[0-7]$', entry.name):
|
if re.match('^0000:' + re.escape(id) + ':[a-fA-F0-9]{2}\.[0-7]$', entry.name):
|
||||||
result.append(entry.name)
|
result.append(entry)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
def get_device_extended_bdfs(id: str) -> List[str]:
|
||||||
|
return [e.name for e in get_device_paths(id)]
|
||||||
|
|
||||||
|
def get_singular_device_path(id: str) -> Path:
|
||||||
|
devicePaths = get_device_paths(id)
|
||||||
|
if len(devicePaths) == 0:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Device BDF path for {id}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if len(devicePaths) != 1:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Device BDF path for {id} since too many Extended Device BDFs match: {devicePaths}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return devicePaths[0]
|
||||||
|
|
||||||
|
def get_singular_device_extended_bdf(id: str) -> str:
|
||||||
|
deviceBDFs = get_device_extended_bdfs(id)
|
||||||
|
if len(deviceBDFs) == 0:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Device BDF for {id}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if len(deviceBDFs) != 1:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Device BDF for {id} since too many Extended Device BDFs match: {deviceBDFs}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return deviceBDFs[0]
|
||||||
|
|
||||||
|
# obtain bridge paths/bdfs
|
||||||
|
|
||||||
|
def get_bridge_paths(id: str) -> List[Path]:
|
||||||
|
return [e.resolve().absolute().parent for e in get_device_paths(id)]
|
||||||
|
|
||||||
|
def get_bridge_extended_bdfs(id: str) -> List[str]:
|
||||||
|
return [e.name for e in get_bridge_paths(id)]
|
||||||
|
|
||||||
|
def get_singular_bridge_path(id: str) -> Path:
|
||||||
|
bridgePaths = get_bridge_paths(id)
|
||||||
|
if len(bridgePaths) == 0:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Bridge BDF path for {id}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if len(bridgePaths) != 1:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Bridge BDF path for {id} since too many Extended Bridge BDFs match: {bridgePaths}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return bridgePaths[0]
|
||||||
|
|
||||||
|
def get_singular_bridge_extended_bdf(id: str) -> str:
|
||||||
|
bridgeBDFs = get_bridge_extended_bdfs(id)
|
||||||
|
if len(bridgeBDFs) == 0:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Bridge BDF for {id}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
if len(bridgeBDFs) != 1:
|
||||||
|
print(f":ERROR: Unable to obtain Extended Bridge BDF for {id} since too many Extended Bridge BDFs match: {bridgeBDFs}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
return bridgeBDFs[0]
|
||||||
|
|
||||||
|
# misc
|
||||||
|
|
||||||
def get_fpga_devs(id) -> List[Path]:
|
def get_fpga_devs(id) -> List[Path]:
|
||||||
def readUevent(path: Path) -> Dict[Any, Any]:
|
def readUevent(path: Path) -> Dict[str, str]:
|
||||||
if not (path / 'uevent').exists():
|
if not (path / 'uevent').exists():
|
||||||
return {}
|
return {}
|
||||||
return { entry[0]: entry[1] for entry in [line.strip('\n\r ').split('=') for line in open(f'{path}/uevent', 'r').readlines()] if len(entry) >= 2 }
|
return { entry[0]: entry[1] for entry in [line.strip('\n\r ').split('=') for line in open(f'{path}/uevent', 'r').readlines()] if len(entry) >= 2 }
|
||||||
|
@ -51,7 +100,7 @@ def get_fpga_devs(id) -> List[Path]:
|
||||||
}
|
}
|
||||||
|
|
||||||
returnDevs = []
|
returnDevs = []
|
||||||
fpgaDevices = get_fpga_bdfs(id)
|
fpgaDevices = get_device_extended_bdfs(id)
|
||||||
for fpgaDev in fpgaDevices:
|
for fpgaDev in fpgaDevices:
|
||||||
path = pciDevicesPath / fpgaDev
|
path = pciDevicesPath / fpgaDev
|
||||||
fpgaDevUevent = readUevent(path)
|
fpgaDevUevent = readUevent(path)
|
||||||
|
@ -66,67 +115,61 @@ def get_fpga_devs(id) -> List[Path]:
|
||||||
|
|
||||||
# clear SERR bit in command register
|
# clear SERR bit in command register
|
||||||
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
||||||
def clear_serr(id: str) -> None:
|
def clear_serr_bits(id: str) -> None:
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
for bridgeBDF in get_bridge_extended_bdfs(id):
|
||||||
run = subprocess.run(['setpci', '-s', bridgeBDF, 'COMMAND=0000:0100'])
|
run = subprocess.run(['setpci', '-s', bridgeBDF, 'COMMAND=0000:0100'])
|
||||||
if run.returncode != 0:
|
if run.returncode != 0:
|
||||||
print(":ERROR: Unable to clear SERR bit")
|
print(f":ERROR: Unable to clear SERR bit for {bridgeBDF}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
# clear fatal error reporting enable bit in the device control register
|
# clear fatal error reporting enable bit in the device control register
|
||||||
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
# https://support.xilinx.com/s/question/0D52E00006hpjPHSAY/dell-r720-poweredge-server-reboots-on-fpga-reprogramming?language=en_US
|
||||||
def clear_fatal_error_reporting(id: str) -> None:
|
def clear_fatal_error_reporting_bits(id: str) -> None:
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
for bridgeBDF in get_bridge_extended_bdfs(id):
|
||||||
run = subprocess.run(['setpci', '-s', bridgeBDF, 'CAP_EXP+8.w=0000:0004'])
|
run = subprocess.run(['setpci', '-s', bridgeBDF, 'CAP_EXP+8.w=0000:0004'])
|
||||||
if run.returncode != 0:
|
if run.returncode != 0:
|
||||||
print(":ERROR: Unable to clear SERR bit")
|
print(f":ERROR: Unable to clear error reporting bit for {bridgeBDF}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def write_to_linux_device_path(path: Path, data: str = '1\n') -> None:
|
def write_to_linux_device_path(path: Path, data: str = '1\n') -> None:
|
||||||
try:
|
try:
|
||||||
|
print(f"Writing to {path}: {data.strip()}")
|
||||||
open(path, 'w').write(data)
|
open(path, 'w').write(data)
|
||||||
except:
|
except:
|
||||||
print(f":ERROR: Cannot write to {path} value: {data}")
|
print(f":ERROR: Cannot write to {path} value: {data}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def remove(id: str) -> None:
|
def remove(id: str) -> None:
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
for devicePaths in get_device_paths(id):
|
||||||
deviceBDFs = get_fpga_bdfs(id)
|
removePath = devicePaths.resolve().absolute() / 'remove'
|
||||||
for deviceBDF in deviceBDFs:
|
|
||||||
removePath = pciDevicesPath / bridgeBDF / deviceBDF / 'remove'
|
|
||||||
if removePath.exists():
|
if removePath.exists():
|
||||||
write_to_linux_device_path(removePath)
|
write_to_linux_device_path(removePath)
|
||||||
|
|
||||||
def rescan(id: str) -> None:
|
def rescan(id: str) -> None:
|
||||||
bridgeBDF = get_bridge_bdf(id)
|
for bridgePath in get_bridge_paths(id):
|
||||||
if bridgeBDF is not None:
|
rescanPath = bridgePath / 'rescan'
|
||||||
rescanPath = pciDevicesPath / bridgeBDF / 'rescan'
|
if rescanPath.exists():
|
||||||
write_to_linux_device_path(rescanPath)
|
write_to_linux_device_path(rescanPath)
|
||||||
else:
|
write_to_linux_device_path(Path('/sys/bus/pci/rescan'))
|
||||||
write_to_linux_device_path('/sys/bus/pci/rescan')
|
|
||||||
|
|
||||||
# enable memory mapped transfers for the fpga
|
# enable memory mapped transfers for the fpga
|
||||||
# https://support.xilinx.com/s/question/0D52E00006iHlNoSAK/lspci-reports-bar-0-disabled?language=en_US
|
# https://support.xilinx.com/s/question/0D52E00006iHlNoSAK/lspci-reports-bar-0-disabled?language=en_US
|
||||||
def enable_memmapped_transfers(id: str) -> None:
|
def enable_memmapped_transfers(id: str) -> None:
|
||||||
deviceBDFs = get_fpga_bdfs(id)
|
for deviceBDF in get_device_extended_bdfs(id):
|
||||||
for deviceBDF in deviceBDFs:
|
|
||||||
run = subprocess.run(['setpci', '-s', deviceBDF, 'COMMAND=0x02'])
|
run = subprocess.run(['setpci', '-s', deviceBDF, 'COMMAND=0x02'])
|
||||||
if run.returncode != 0:
|
if run.returncode != 0:
|
||||||
print(f":ERROR: Unable to enable memmapped transfers on {deviceBDF}")
|
print(f":ERROR: Unable to enable memmapped transfers on {deviceBDF}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def program_fpga(serial: str, board: str, bitstream: str) -> None:
|
def program_fpga(vivado: str, serial: str, bitstream: str) -> None:
|
||||||
print(":WARNING: This only can target the 1st FPGA on a machine currently...")
|
|
||||||
|
|
||||||
pVivado = subprocess.Popen(
|
pVivado = subprocess.Popen(
|
||||||
[
|
[
|
||||||
'vivado',
|
vivado,
|
||||||
'-mode', 'tcl',
|
'-mode', 'tcl',
|
||||||
'-nolog', '-nojournal', '-notrace',
|
'-nolog', '-nojournal', '-notrace',
|
||||||
'-source', scriptPath / 'program_fpga.tcl',
|
'-source', scriptPath / 'program_fpga.tcl',
|
||||||
'-tclargs',
|
'-tclargs',
|
||||||
'-board', board,
|
'-serial', serial,
|
||||||
'-bitstream_path', bitstream,
|
'-bitstream_path', bitstream,
|
||||||
],
|
],
|
||||||
stdin=subprocess.DEVNULL
|
stdin=subprocess.DEVNULL
|
||||||
|
@ -135,58 +178,186 @@ def program_fpga(serial: str, board: str, bitstream: str) -> None:
|
||||||
pVivado.wait()
|
pVivado.wait()
|
||||||
|
|
||||||
if pVivado.returncode != 0:
|
if pVivado.returncode != 0:
|
||||||
print(":ERROR: Unable to flash FPGA")
|
print(f":ERROR: Unable to flash FPGA {serial} with {bitstream}", file=sys.stderr)
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
def get_serial_from_bdf(id: str) -> str:
|
# mapping functions
|
||||||
deviceBDFs = get_fpga_bdfs(parsed_args.bus_id)
|
|
||||||
if len(deviceBDFs) == 0:
|
def get_fpga_db() -> Dict[Any, Any]:
|
||||||
print(f":ERROR: Unable to obtain Extended Device BDF for {parsed_args.bus_id}")
|
db_file = Path("/opt/firesim-db.json")
|
||||||
sys.exit(1)
|
if db_file.exists():
|
||||||
return "TODO"
|
with open(db_file, 'r') as f:
|
||||||
|
db = json.load(f)
|
||||||
|
return db
|
||||||
|
else:
|
||||||
|
print(f":ERROR: Unable to open {db_file}. Does it exist? Did you run 'firesim enumeratefpgas'?")
|
||||||
|
|
||||||
|
print(f":ERROR: Unable to create FPGA database from {db_file}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def get_serial_from_bus_id(id: str) -> str:
|
||||||
|
deviceBDF = get_bdf_from_extended_bdf(get_singular_device_extended_bdf(id))
|
||||||
|
db = get_fpga_db()
|
||||||
|
for e in db:
|
||||||
|
if deviceBDF == e['bdf']:
|
||||||
|
return e['uid']
|
||||||
|
print(":ERROR: Unable to get serial number from bus id", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def get_serials() -> List[str]:
|
||||||
|
db = get_fpga_db()
|
||||||
|
serials = []
|
||||||
|
for e in db:
|
||||||
|
serials.append(e['uid'])
|
||||||
|
return serials
|
||||||
|
|
||||||
|
def get_extended_bdfs() -> List[str]:
|
||||||
|
db = get_fpga_db()
|
||||||
|
bdfs = []
|
||||||
|
for e in db:
|
||||||
|
bdfs.append(convert_bdf_to_extended_bdf(e['bdfs']))
|
||||||
|
return bdfs
|
||||||
|
|
||||||
|
def convert_bdf_to_extended_bdf(bdf: str) -> str:
|
||||||
|
return '0000:' + bdf
|
||||||
|
|
||||||
|
def get_bus_id_from_extended_bdf(extended_bdf: str) -> str:
|
||||||
|
return extended_bdf[5:7]
|
||||||
|
|
||||||
|
def get_bdf_from_extended_bdf(extended_bdf: str) -> str:
|
||||||
|
return extended_bdf[5:]
|
||||||
|
|
||||||
|
# main
|
||||||
|
|
||||||
def main(args: List[str]) -> int:
|
def main(args: List[str]) -> int:
|
||||||
parser = argparse.ArgumentParser(description="Program a Xilinx XDMA-enabled FPGA")
|
parser = argparse.ArgumentParser(description="Program a Xilinx XDMA-enabled FPGA")
|
||||||
megroup = parser.add_mutually_exclusive_group(required=True)
|
megroup = parser.add_mutually_exclusive_group(required=True)
|
||||||
megroup.add_argument("--bus_id", help="Bus number of FPGA to flash (i.e. ****:<THIS>:**.*)")
|
megroup.add_argument("--bus_id", help="Bus number of FPGA (i.e. ****:<THIS>:**.*)")
|
||||||
megroup.add_argument("--serial_no", help="Serial number of FPGA to flash (i.e. what 'get_hw_target' shows in Vivado)")
|
megroup.add_argument("--bdf", help="BDF of FPGA (i.e. ****:<THIS>)")
|
||||||
parser.add_argument("--bitstream", help="Bitstream to flash onto FPGA", required=True, type=Path)
|
megroup.add_argument("--extended-bdf", help="Extended BDF of FPGA (i.e. all of this - ****:**:**.*)")
|
||||||
parser.add_argument("--board", help="FPGA board to flash", required=True)
|
megroup.add_argument("--serial_no", help="Serial number of FPGA (i.e. what 'get_hw_target' shows in Vivado)")
|
||||||
|
megroup.add_argument("--all-serials", help="Use all serial numbers (no PCI-E manipulation)", action="store_true")
|
||||||
|
megroup.add_argument("--all-bdfs", help="Use all BDFs (PCI-E manipulation)", action="store_true")
|
||||||
|
parser.add_argument("--vivado-bin", help="Explicit path to 'vivado'", type=Path)
|
||||||
|
parser.add_argument("--hw-server-bin", help="Explicit path to 'hw_server'", type=Path)
|
||||||
|
megroup2 = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
megroup2.add_argument("--bitstream", help="The bitstream to flash onto FPGA(s)", type=Path)
|
||||||
|
megroup2.add_argument("--disconnect-bdf", help="Disconnect BDF(s)", action="store_true")
|
||||||
|
megroup2.add_argument("--reconnect-bdf", help="Reconnect BDF(s)", action="store_true")
|
||||||
parsed_args = parser.parse_args(args)
|
parsed_args = parser.parse_args(args)
|
||||||
|
|
||||||
scriptPath = Path(__file__).resolve().parent
|
if parsed_args.hw_server_bin is None:
|
||||||
|
parsed_args.hw_server_bin = shutil.which('hw_server')
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
parsed_args.vivado_bin = shutil.which('vivado')
|
||||||
|
|
||||||
|
if parsed_args.hw_server_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Hardware Server!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
if parsed_args.vivado_bin is None:
|
||||||
|
print(':ERROR: Could not find Xilinx Vivado!', file=sys.stderr)
|
||||||
|
exit(1)
|
||||||
|
|
||||||
|
parsed_args.vivado_bin = Path(parsed_args.vivado_bin).absolute()
|
||||||
|
parsed_args.hw_server_bin = Path(parsed_args.hw_server_bin).absolute()
|
||||||
|
|
||||||
eUserId = os.geteuid()
|
eUserId = os.geteuid()
|
||||||
sudoUserId = os.getenv('SUDO_UID')
|
sudoUserId = os.getenv('SUDO_UID')
|
||||||
isAdmin = (eUserId == 0) and (sudoUserId is None)
|
isAdmin = (eUserId == 0) and (sudoUserId is None)
|
||||||
userId = eUserId if sudoUserId is None else int(sudoUserId)
|
userId = eUserId if sudoUserId is None else int(sudoUserId)
|
||||||
|
|
||||||
if not isAdmin:
|
# if not sudoer, spawn w/ sudo
|
||||||
print(":ERROR: Requires running script with 'sudo'")
|
if eUserId != 0:
|
||||||
sys.exit(1)
|
execvArgs = ['/usr/bin/sudo', str(Path(__file__).absolute())] + sys.argv[1:]
|
||||||
|
execvArgs += ['--vivado-bin', str(parsed_args.vivado_bin), '--hw-server-bin', str(parsed_args.hw_server_bin)]
|
||||||
|
print(f"Running: {execvArgs}")
|
||||||
|
os.execv(execvArgs[0], execvArgs)
|
||||||
|
|
||||||
if not parsed_args.bitstream.is_file() or not parsed_args.bitstream.exists():
|
# program based on bitstream
|
||||||
print(f":ERROR: Invalid bitstream: {parsed_args.bitstream}")
|
if parsed_args.bitstream is not None:
|
||||||
sys.exit(1)
|
if not parsed_args.bitstream.is_file() or not parsed_args.bitstream.exists():
|
||||||
else:
|
print(f":ERROR: Invalid bitstream: {parsed_args.bitstream}")
|
||||||
parsed_args.bitstream = parsed_args.bitstream.absolute()
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
parsed_args.bitstream = parsed_args.bitstream.absolute()
|
||||||
|
|
||||||
if parsed_args.bus_id:
|
if parsed_args.bus_id or parsed_args.bdf or parsed_args.extended_bdf or parsed_args.all_bdfs:
|
||||||
serialNumber = get_serial_from_bdf(id)
|
bus_ids = []
|
||||||
clear_serr(parsed_args.bus_id)
|
if parsed_args.bus_id:
|
||||||
clear_fatal_error_reporting(parsed_args.bus_id)
|
bus_ids.append(parsed_args.bus_id)
|
||||||
remove(parsed_args.bus_id)
|
if parsed_args.bdf:
|
||||||
program_fpga(serialNumber, parsed_args.board, parsed_args.bitstream)
|
bus_ids.append(get_bus_id_from_extended_bdf(convert_bdf_to_extended_bdf(parsed_args.bdf)))
|
||||||
rescan(parsed_args.bus_id)
|
if parsed_args.extended_bdf:
|
||||||
enable_memmapped_transfers(parsed_args.bus_id)
|
bus_ids.append(get_bus_id_from_extended_bdf(parsed_args.extended_bdf))
|
||||||
|
if parsed_args.all_bdfs:
|
||||||
|
bus_ids.extend([get_bus_id_from_extended_bdf(bdf) for bdf in get_extended_bdfs()])
|
||||||
|
|
||||||
print(f"Successfully programmed FPGA {parsed_args.bus_id} with {parsed_args.bitstream}")
|
# must be called before the remove otherwise it will not find a serial number
|
||||||
|
serialNums = []
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
serialNums.append(get_serial_from_bus_id(bus_id))
|
||||||
|
|
||||||
if parsed_args.serial_no:
|
for bus_id in bus_ids:
|
||||||
program_fpga(parsed_args.serial_no, parsed_args.board, parsed_args.bitstream)
|
clear_serr_bits(bus_id)
|
||||||
|
clear_fatal_error_reporting_bits(bus_id)
|
||||||
|
remove(bus_id)
|
||||||
|
|
||||||
print(f"Successfully programmed FPGA {parsed_args.serial_no} with {parsed_args.bitstream}")
|
# program fpga(s) separately if doing multiple bdfs
|
||||||
print(":WARNING: Please warm reboot the machine")
|
for i, bus_id in enumerate(bus_ids):
|
||||||
|
serialNumber = serialNums[i]
|
||||||
|
program_fpga(str(parsed_args.vivado_bin), serialNumber, parsed_args.bitstream)
|
||||||
|
print(f"Successfully programmed FPGA {bus_id} with {parsed_args.bitstream}")
|
||||||
|
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
rescan(bus_id)
|
||||||
|
enable_memmapped_transfers(bus_id)
|
||||||
|
|
||||||
|
if parsed_args.serial_no or parsed_args.all_serials:
|
||||||
|
serial_nos = []
|
||||||
|
if parsed_args.serial_no:
|
||||||
|
serial_nos.append(parsed_args.serial_no)
|
||||||
|
if parsed_args.all_serials:
|
||||||
|
serial_nos.extend(get_serials())
|
||||||
|
|
||||||
|
for serial in serial_nos:
|
||||||
|
program_fpga(str(parsed_args.vivado_bin), serial, parsed_args.bitstream)
|
||||||
|
print(f"Successfully programmed FPGA {serial} with {parsed_args.bitstream}")
|
||||||
|
print(":WARNING: Please warm reboot the machine")
|
||||||
|
|
||||||
|
# disconnect bdfs
|
||||||
|
if parsed_args.disconnect_bdf:
|
||||||
|
if parsed_args.bus_id or parsed_args.all_bdfs or parsed_args.bdf or parsed_args.extended_bdf:
|
||||||
|
bus_ids = []
|
||||||
|
if parsed_args.bus_id:
|
||||||
|
bus_ids.append(parsed_args.bus_id)
|
||||||
|
if parsed_args.bdf:
|
||||||
|
bus_ids.append(get_bus_id_from_extended_bdf(convert_bdf_to_extended_bdf(parsed_args.bdf)))
|
||||||
|
if parsed_args.extended_bdf:
|
||||||
|
bus_ids.append(get_bus_id_from_extended_bdf(parsed_args.extended_bdf))
|
||||||
|
if parsed_args.all_bdfs:
|
||||||
|
bus_ids.extend([get_bus_id_from_extended_bdf(bdf) for bdf in get_extended_bdfs()])
|
||||||
|
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
clear_serr_bits(bus_id)
|
||||||
|
clear_fatal_error_reporting_bits(bus_id)
|
||||||
|
remove(bus_id)
|
||||||
|
|
||||||
|
# reconnect bdfs
|
||||||
|
if parsed_args.reconnect_bdf:
|
||||||
|
if parsed_args.bus_id or parsed_args.all_bdfs or parsed_args.bdf or parsed_args.extended_bdf:
|
||||||
|
bus_ids = []
|
||||||
|
if parsed_args.bus_id:
|
||||||
|
bus_ids.append(parsed_args.bus_id)
|
||||||
|
if parsed_args.bdf:
|
||||||
|
bus_ids.append(get_bus_id_from_extended_bdf(convert_bdf_to_extended_bdf(parsed_args.bdf)))
|
||||||
|
if parsed_args.extended_bdf:
|
||||||
|
bus_ids.append(get_bus_id_from_extended_bdf(parsed_args.extended_bdf))
|
||||||
|
if parsed_args.all_bdfs:
|
||||||
|
bus_ids.extend([get_bus_id_from_extended_bdf(bdf) for bdf in get_extended_bdfs()])
|
||||||
|
|
||||||
|
for bus_id in bus_ids:
|
||||||
|
rescan(bus_id)
|
||||||
|
enable_memmapped_transfers(bus_id)
|
||||||
|
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
|
@ -1,58 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
# Adapted from https://github.com/Xilinx/open-nic-shell
|
|
||||||
|
|
||||||
echo $#
|
|
||||||
if [[ $# -le 1 ]] || [[ -z EXTENDED_DEVICE_BDF1 ]] || [[ -z $XILINX_VIVADO ]]; then
|
|
||||||
echo "Usage: EXTENDED_DEVICE_BDF1=<EXTENDED_DEVICE_BDF1> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please export EXTENDED_DEVICE_BDF1 and [EXTENDED_DEVICE_BDF2 (if needed for 2 port boards)]"
|
|
||||||
echo "Example: EXTENDED_DEVICE_BDF1=<0000:86:00.0> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please ensure vivado is loaded into system path."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
bridge_bdf=""
|
|
||||||
bitstream_path=$1
|
|
||||||
board=$2
|
|
||||||
probes_path="${3:-}"
|
|
||||||
# ^^ Probes are used for specifying hardware debugger symbols.
|
|
||||||
|
|
||||||
# Infer bridge
|
|
||||||
if [ -e "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1" ]; then
|
|
||||||
bridge_bdf=$(basename $(dirname $(readlink "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1")))
|
|
||||||
# Both devices will be on the same bridge as they are on the same FPGA board.
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Remove
|
|
||||||
if [[ $bridge_bdf != "" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF1}/remove" > /dev/null
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]] && [[ -e "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}/remove" > /dev/null
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Could not find bridge_bdf for the device $EXTENDED_DEVICE_BDF1"
|
|
||||||
echo "If remove was called on the device already, then manually set bridge_bdf here and comment 'exit 1'."
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Program fpga
|
|
||||||
vivado -mode tcl -source $SCRIPT_DIR/program_fpga.tcl \
|
|
||||||
-tclargs -board $board \
|
|
||||||
-bitstream_path $bitstream_path \
|
|
||||||
-probes_path $probes_path
|
|
||||||
|
|
||||||
# Rescan
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/rescan" > /dev/null
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF1 COMMAND=0x02
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]]; then
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF2 COMMAND=0x02
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "program_fpga.sh completed"
|
|
||||||
echo "Warm reboot machine if the FPGA wasn't initially setup with an XDMA bitstream."
|
|
|
@ -7,11 +7,11 @@ set root_dir [file dirname $script_dir]
|
||||||
|
|
||||||
# Loading options
|
# Loading options
|
||||||
# bitstream_path Path to the bitstream
|
# bitstream_path Path to the bitstream
|
||||||
# board Board name
|
# serial Serial number of FPGA board (without trailing A)
|
||||||
array set options {
|
array set options {
|
||||||
-bitstream_path ""
|
-bitstream_path ""
|
||||||
-probes_path ""
|
-probes_path ""
|
||||||
-board au50
|
-serial ""
|
||||||
}
|
}
|
||||||
|
|
||||||
# Expect arguments in the form of `-argument value`
|
# Expect arguments in the form of `-argument value`
|
||||||
|
@ -31,34 +31,39 @@ foreach {key value} [array get options] {
|
||||||
set [string range $key 1 end] $value
|
set [string range $key 1 end] $value
|
||||||
}
|
}
|
||||||
|
|
||||||
source ${script_dir}/${board}.tcl
|
|
||||||
|
|
||||||
puts "Program file: $options(-bitstream_path)"
|
puts "Program file: $options(-bitstream_path)"
|
||||||
puts "Probes file: $options(-probes_path)"
|
puts "Probes file: $options(-probes_path)"
|
||||||
puts "Board: $options(-board)"
|
puts "Serial Number: $options(-serial)"
|
||||||
puts "HW device: $hw_device"
|
|
||||||
|
set_param labtools.enable_cs_server false
|
||||||
|
|
||||||
open_hw_manager
|
open_hw_manager
|
||||||
connect_hw_server -allow_non_jtag
|
connect_hw_server -allow_non_jtag
|
||||||
|
|
||||||
## by default vivado opens a default hw target
|
# by default vivado opens a default hw target
|
||||||
#close_hw_target
|
close_hw_target
|
||||||
|
|
||||||
# note: helps view amount of fpgas
|
# check if serial is in hw targets
|
||||||
get_hw_targets
|
set final_hw_target ""
|
||||||
|
foreach {hw_target} [get_hw_targets] {
|
||||||
|
if {[string first $serial $hw_target] != -1} {
|
||||||
|
set final_hw_target $hw_target
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
# TODO:
|
if {$final_hw_target == ""} {
|
||||||
# when no FPGA is programmed
|
puts "Unable to find $serial in available HW targets. See available HW targets below:"
|
||||||
# can use id to index into get_hw_targets and program that specific FPGA
|
get_hw_targets
|
||||||
# when you notice a PCI-ID associated w/ the FPGA
|
exit 1
|
||||||
# TODO: how do you determine the ID of the FPGA from the PCI-BDF
|
}
|
||||||
open_hw_target
|
|
||||||
current_hw_device [get_hw_devices $hw_device]
|
puts "Programming $final_hw_target with ${options(-bitstream_path)}"
|
||||||
refresh_hw_device -update_hw_probes false [lindex [get_hw_devices $hw_device] 0]
|
open_hw_target $final_hw_target
|
||||||
set_property PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
set_property PROBES.FILE ${options(-probes_path)} [get_hw_device]
|
||||||
set_property FULL_PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
set_property FULL_PROBES.FILE ${options(-probes_path)} [get_hw_device]
|
||||||
set_property PROGRAM.FILE ${options(-bitstream_path)} [get_hw_devices $hw_device]
|
set_property PROGRAM.FILE ${options(-bitstream_path)} [get_hw_device]
|
||||||
program_hw_devices [get_hw_devices $hw_device]
|
program_hw_devices [get_hw_device]
|
||||||
refresh_hw_device [lindex [get_hw_devices $hw_device] 0]
|
refresh_hw_device [get_hw_device]
|
||||||
|
close_hw_target
|
||||||
|
|
||||||
exit
|
exit
|
||||||
|
|
|
@ -21,4 +21,3 @@
|
||||||
set part xcu280-fsvh2892-2L-e
|
set part xcu280-fsvh2892-2L-e
|
||||||
set board_part xilinx.com:au280:part0:1.2
|
set board_part xilinx.com:au280:part0:1.2
|
||||||
set zynq_family 0
|
set zynq_family 0
|
||||||
set hw_device xcu280_u55c_0
|
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
../xilinx_alveo_u250/scripts
|
|
@ -1 +0,0 @@
|
||||||
../cl_firesim/scripts/au280.tcl
|
|
|
@ -1,58 +0,0 @@
|
||||||
#!/usr/bin/env bash
|
|
||||||
|
|
||||||
# Adapted from https://github.com/Xilinx/open-nic-shell
|
|
||||||
|
|
||||||
echo $#
|
|
||||||
if [[ $# -le 1 ]] || [[ -z EXTENDED_DEVICE_BDF1 ]] || [[ -z $XILINX_VIVADO ]]; then
|
|
||||||
echo "Usage: EXTENDED_DEVICE_BDF1=<EXTENDED_DEVICE_BDF1> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please export EXTENDED_DEVICE_BDF1 and [EXTENDED_DEVICE_BDF2 (if needed for 2 port boards)]"
|
|
||||||
echo "Example: EXTENDED_DEVICE_BDF1=<0000:86:00.0> program_fpga.sh BITSTREAM_PATH BOARD [PROBES_PATH]"
|
|
||||||
echo "Please ensure vivado is loaded into system path."
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
|
|
||||||
|
|
||||||
set -Eeuo pipefail
|
|
||||||
set -x
|
|
||||||
|
|
||||||
bridge_bdf=""
|
|
||||||
bitstream_path=$1
|
|
||||||
board=$2
|
|
||||||
probes_path="${3:-}"
|
|
||||||
# ^^ Probes are used for specifying hardware debugger symbols.
|
|
||||||
|
|
||||||
# Infer bridge
|
|
||||||
if [ -e "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1" ]; then
|
|
||||||
bridge_bdf=$(basename $(dirname $(readlink "/sys/bus/pci/devices/$EXTENDED_DEVICE_BDF1")))
|
|
||||||
# Both devices will be on the same bridge as they are on the same FPGA board.
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Remove
|
|
||||||
if [[ $bridge_bdf != "" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF1}/remove" > /dev/null
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]] && [[ -e "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}" ]]; then
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/${EXTENDED_DEVICE_BDF2}/remove" > /dev/null
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
echo "Could not find bridge_bdf for the device $EXTENDED_DEVICE_BDF1"
|
|
||||||
echo "If remove was called on the device already, then manually set bridge_bdf here and comment 'exit 1'."
|
|
||||||
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Program fpga
|
|
||||||
vivado -mode tcl -source $SCRIPT_DIR/program_fpga.tcl \
|
|
||||||
-tclargs -board $board \
|
|
||||||
-bitstream_path $bitstream_path \
|
|
||||||
-probes_path $probes_path
|
|
||||||
|
|
||||||
# Rescan
|
|
||||||
echo 1 | sudo tee "/sys/bus/pci/devices/${bridge_bdf}/rescan" > /dev/null
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF1 COMMAND=0x02
|
|
||||||
if [[ -n "${EXTENDED_DEVICE_BDF2:-}" ]]; then
|
|
||||||
sudo setpci -s $EXTENDED_DEVICE_BDF2 COMMAND=0x02
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "program_fpga.sh completed"
|
|
||||||
echo "Warm reboot machine if the FPGA wasn't initially setup with an XDMA bitstream."
|
|
|
@ -1,52 +0,0 @@
|
||||||
# Adapted from https://github.com/Xilinx/open-nic-shell
|
|
||||||
|
|
||||||
# Directory variables
|
|
||||||
set script_path [file normalize [info script]]
|
|
||||||
set script_dir [file dirname $script_path]
|
|
||||||
set root_dir [file dirname $script_dir]
|
|
||||||
|
|
||||||
# Loading options
|
|
||||||
# bitstream_path Path to the bitstream
|
|
||||||
# board Board name
|
|
||||||
array set options {
|
|
||||||
-bitstream_path ""
|
|
||||||
-probes_path ""
|
|
||||||
-board au50
|
|
||||||
}
|
|
||||||
|
|
||||||
# Expect arguments in the form of `-argument value`
|
|
||||||
for {set i 0} {$i < $argc} {incr i 2} {
|
|
||||||
set arg [lindex $argv $i]
|
|
||||||
set val [lindex $argv [expr $i+1]]
|
|
||||||
if {[info exists options($arg)]} {
|
|
||||||
set options($arg) $val
|
|
||||||
puts "Set option $arg to $val"
|
|
||||||
} else {
|
|
||||||
puts "Skip unknown argument $arg and its value $val"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Settings based on defaults or passed in values
|
|
||||||
foreach {key value} [array get options] {
|
|
||||||
set [string range $key 1 end] $value
|
|
||||||
}
|
|
||||||
|
|
||||||
source ${script_dir}/${board}.tcl
|
|
||||||
|
|
||||||
puts "Program file: $options(-bitstream_path)"
|
|
||||||
puts "Probes file: $options(-probes_path)"
|
|
||||||
puts "Board: $options(-board)"
|
|
||||||
puts "HW device: $hw_device"
|
|
||||||
|
|
||||||
open_hw_manager
|
|
||||||
connect_hw_server -allow_non_jtag
|
|
||||||
open_hw_target
|
|
||||||
current_hw_device [get_hw_devices $hw_device]
|
|
||||||
refresh_hw_device -update_hw_probes false [lindex [get_hw_devices $hw_device] 0]
|
|
||||||
set_property PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
|
||||||
set_property FULL_PROBES.FILE ${options(-probes_path)} [get_hw_devices $hw_device]
|
|
||||||
set_property PROGRAM.FILE ${options(-bitstream_path)} [get_hw_devices $hw_device]
|
|
||||||
program_hw_devices [get_hw_devices $hw_device]
|
|
||||||
refresh_hw_device [lindex [get_hw_devices $hw_device] 0]
|
|
||||||
|
|
||||||
exit
|
|
|
@ -110,7 +110,7 @@ dromajo_t::dromajo_t(simif_t &sim,
|
||||||
* Destructor for Dromajo
|
* Destructor for Dromajo
|
||||||
*/
|
*/
|
||||||
dromajo_t::~dromajo_t() {
|
dromajo_t::~dromajo_t() {
|
||||||
if (this->dromajo_state != NULL)
|
if (this->dromajo_state)
|
||||||
dromajo_cosim_fini(this->dromajo_state);
|
dromajo_cosim_fini(this->dromajo_state);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,7 +151,7 @@ void dromajo_t::init() {
|
||||||
|
|
||||||
printf("[INFO] Dromajo command: \n");
|
printf("[INFO] Dromajo command: \n");
|
||||||
char *dromajo_argv[dromajo_args.size()];
|
char *dromajo_argv[dromajo_args.size()];
|
||||||
for (int i = 0; i < dromajo_args.size(); ++i) {
|
for (size_t i = 0; i < dromajo_args.size(); ++i) {
|
||||||
dromajo_argv[i] = const_cast<char *>(dromajo_args[i].c_str());
|
dromajo_argv[i] = const_cast<char *>(dromajo_args[i].c_str());
|
||||||
printf("%s ", dromajo_argv[i]);
|
printf("%s ", dromajo_argv[i]);
|
||||||
}
|
}
|
||||||
|
|
|
@ -216,11 +216,14 @@ simplenic_t::~simplenic_t() {
|
||||||
fclose(this->niclog);
|
fclose(this->niclog);
|
||||||
if (loopback) {
|
if (loopback) {
|
||||||
for (auto &pcis_read_buf : pcis_read_bufs)
|
for (auto &pcis_read_buf : pcis_read_bufs)
|
||||||
free(pcis_read_buf);
|
if (pcis_read_buf)
|
||||||
|
free(pcis_read_buf);
|
||||||
} else {
|
} else {
|
||||||
for (int j = 0; j < 2; j++) {
|
for (int j = 0; j < 2; j++) {
|
||||||
munmap(pcis_read_bufs[j], BUFBYTES + EXTRABYTES);
|
if (pcis_read_bufs[j])
|
||||||
munmap(pcis_write_bufs[j], BUFBYTES + EXTRABYTES);
|
munmap(pcis_read_bufs[j], BUFBYTES + EXTRABYTES);
|
||||||
|
if (pcis_write_bufs[j])
|
||||||
|
munmap(pcis_write_bufs[j], BUFBYTES + EXTRABYTES);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -252,7 +252,7 @@ void tracerv_t::serialize(
|
||||||
const bool fireperf) {
|
const bool fireperf) {
|
||||||
const int max_consider = std::min(max_core_ipc, 7);
|
const int max_consider = std::min(max_core_ipc, 7);
|
||||||
if (human_readable || test_output) {
|
if (human_readable || test_output) {
|
||||||
for (int i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
for (size_t i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
||||||
if (test_output) {
|
if (test_output) {
|
||||||
fprintf(tracefile, "%016lx", OUTBUF[i + 7]);
|
fprintf(tracefile, "%016lx", OUTBUF[i + 7]);
|
||||||
fprintf(tracefile, "%016lx", OUTBUF[i + 6]);
|
fprintf(tracefile, "%016lx", OUTBUF[i + 6]);
|
||||||
|
@ -277,7 +277,7 @@ void tracerv_t::serialize(
|
||||||
}
|
}
|
||||||
} else if (fireperf) {
|
} else if (fireperf) {
|
||||||
|
|
||||||
for (int i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
for (size_t i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
||||||
uint64_t cycle_internal = OUTBUF[i + 0];
|
uint64_t cycle_internal = OUTBUF[i + 0];
|
||||||
|
|
||||||
for (int q = 0; q < max_consider; q++) {
|
for (int q = 0; q < max_consider; q++) {
|
||||||
|
@ -293,7 +293,7 @@ void tracerv_t::serialize(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
for (int i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
for (size_t i = 0; i < (bytes_received / sizeof(uint64_t)); i += 8) {
|
||||||
// this stores as raw binary. stored as little endian.
|
// this stores as raw binary. stored as little endian.
|
||||||
// e.g. to get the same thing as the human readable above,
|
// e.g. to get the same thing as the human readable above,
|
||||||
// flip all the bytes in each 512-bit line.
|
// flip all the bytes in each 512-bit line.
|
||||||
|
|
|
@ -66,10 +66,12 @@ tsibridge_t::tsibridge_t(simif_t &simif,
|
||||||
}
|
}
|
||||||
|
|
||||||
tsibridge_t::~tsibridge_t() {
|
tsibridge_t::~tsibridge_t() {
|
||||||
delete fesvr;
|
if (fesvr)
|
||||||
|
delete fesvr;
|
||||||
if (tsi_argv) {
|
if (tsi_argv) {
|
||||||
for (int i = 0; i < tsi_argc; ++i) {
|
for (int i = 0; i < tsi_argc; ++i) {
|
||||||
delete[] tsi_argv[i];
|
if (tsi_argv[i])
|
||||||
|
delete[] tsi_argv[i];
|
||||||
}
|
}
|
||||||
delete[] tsi_argv;
|
delete[] tsi_argv;
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
#include "master.h"
|
#include "master.h"
|
||||||
#include "core/simif.h"
|
#include "core/simif.h"
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
char master_t::KIND;
|
char master_t::KIND;
|
||||||
|
|
||||||
|
@ -13,4 +14,14 @@ master_t::master_t(simif_t &simif,
|
||||||
assert(index == 0 && "only one simulation master is allowed");
|
assert(index == 0 && "only one simulation master is allowed");
|
||||||
}
|
}
|
||||||
|
|
||||||
bool master_t::is_init_done() { return simif.read(mmio_addrs.INIT_DONE); }
|
bool master_t::is_init_done() { return simif.read(mmio_addrs.INIT_DONE) == 1; }
|
||||||
|
|
||||||
|
bool master_t::check_fingerprint() {
|
||||||
|
uint32_t presence = simif.read(mmio_addrs.PRESENCE_READ);
|
||||||
|
printf("FireSim fingerprint: 0x%x\n", presence);
|
||||||
|
return presence != 0x46697265;
|
||||||
|
}
|
||||||
|
|
||||||
|
void master_t::write_fingerprint(uint32_t data) {
|
||||||
|
simif.write(mmio_addrs.PRESENCE_WRITE, data);
|
||||||
|
}
|
||||||
|
|
|
@ -13,6 +13,8 @@ class simif_t;
|
||||||
|
|
||||||
struct SIMULATIONMASTER_struct {
|
struct SIMULATIONMASTER_struct {
|
||||||
uint64_t INIT_DONE;
|
uint64_t INIT_DONE;
|
||||||
|
uint64_t PRESENCE_READ;
|
||||||
|
uint64_t PRESENCE_WRITE;
|
||||||
};
|
};
|
||||||
|
|
||||||
class master_t final : public widget_t {
|
class master_t final : public widget_t {
|
||||||
|
@ -25,6 +27,16 @@ public:
|
||||||
unsigned index,
|
unsigned index,
|
||||||
const std::vector<std::string> &args);
|
const std::vector<std::string> &args);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check whether the device has FireSim fingerprint string.
|
||||||
|
*/
|
||||||
|
bool check_fingerprint();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Write new value to FireSim fingerprint string.
|
||||||
|
*/
|
||||||
|
void write_fingerprint(uint32_t data);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check whether the device is initialised.
|
* Check whether the device is initialised.
|
||||||
*/
|
*/
|
||||||
|
|
|
@ -132,7 +132,8 @@ synthesized_prints_t::synthesized_prints_t(
|
||||||
|
|
||||||
synthesized_prints_t::~synthesized_prints_t() {
|
synthesized_prints_t::~synthesized_prints_t() {
|
||||||
for (size_t i = 0; i < prints.size(); i++) {
|
for (size_t i = 0; i < prints.size(); i++) {
|
||||||
delete masks[i];
|
if (masks[i])
|
||||||
|
delete masks[i];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -27,6 +27,12 @@ simulation_t::simulation_t(widget_registry_t ®istry,
|
||||||
if (arg.find("+zero-out-dram") == 0) {
|
if (arg.find("+zero-out-dram") == 0) {
|
||||||
do_zero_out_dram = true;
|
do_zero_out_dram = true;
|
||||||
}
|
}
|
||||||
|
if (arg.find("+check-fingerprint") == 0) {
|
||||||
|
check_fingerprint_only = true;
|
||||||
|
}
|
||||||
|
if (arg.find("+write-fingerprint=") == 0) {
|
||||||
|
write_fingerprint_only = atoi(arg.c_str() + 19);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (fastloadmem)
|
if (fastloadmem)
|
||||||
|
@ -47,9 +53,10 @@ void simulation_t::record_end_times() {
|
||||||
void simulation_t::print_simulation_performance_summary() {
|
void simulation_t::print_simulation_performance_summary() {
|
||||||
// Must call record_start_times and record_end_times before invoking this
|
// Must call record_start_times and record_end_times before invoking this
|
||||||
// function
|
// function
|
||||||
assert(start_hcycle != -1 && end_hcycle != 0 && "simulation not executed");
|
assert(start_hcycle.has_value() && end_hcycle.has_value() &&
|
||||||
|
"simulation not executed");
|
||||||
|
|
||||||
const uint64_t hcycles = end_hcycle - start_hcycle;
|
const uint64_t hcycles = *end_hcycle - *start_hcycle;
|
||||||
const double sim_time = diff_secs(end_time, start_time);
|
const double sim_time = diff_secs(end_time, start_time);
|
||||||
const double sim_speed = ((double)end_tcycle) / (sim_time * 1000.0);
|
const double sim_speed = ((double)end_tcycle) / (sim_time * 1000.0);
|
||||||
const double measured_host_frequency =
|
const double measured_host_frequency =
|
||||||
|
@ -95,6 +102,30 @@ void simulation_t::simulation_finish() {
|
||||||
int simulation_t::execute_simulation_flow() {
|
int simulation_t::execute_simulation_flow() {
|
||||||
wait_for_init();
|
wait_for_init();
|
||||||
|
|
||||||
|
// following fingerprint logic uses 'exit' instead of 'return' to avoid
|
||||||
|
// issues w/ deconstructors not having initialized values
|
||||||
|
auto &master = registry.get_widget<master_t>();
|
||||||
|
if (check_fingerprint_only || write_fingerprint_only.has_value()) {
|
||||||
|
if (check_fingerprint_only && write_fingerprint_only.has_value()) {
|
||||||
|
fprintf(stderr, "Unable to both check/write FireSim fingerprint\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (check_fingerprint_only && master.check_fingerprint()) {
|
||||||
|
fprintf(stderr, "Invalid FireSim fingerprint\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
if (write_fingerprint_only.has_value()) {
|
||||||
|
master.write_fingerprint(write_fingerprint_only.value());
|
||||||
|
}
|
||||||
|
exit(EXIT_SUCCESS);
|
||||||
|
} else {
|
||||||
|
if (master.check_fingerprint()) {
|
||||||
|
fprintf(stderr, "Invalid FireSim fingerprint\n");
|
||||||
|
exit(EXIT_FAILURE);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (auto *stream = registry.get_stream_engine()) {
|
if (auto *stream = registry.get_stream_engine()) {
|
||||||
stream->init();
|
stream->init();
|
||||||
}
|
}
|
||||||
|
|
|
@ -3,6 +3,7 @@
|
||||||
#ifndef __SIMULATION_H
|
#ifndef __SIMULATION_H
|
||||||
#define __SIMULATION_H
|
#define __SIMULATION_H
|
||||||
|
|
||||||
|
#include <optional>
|
||||||
#include <string>
|
#include <string>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
||||||
|
@ -129,10 +130,20 @@ private:
|
||||||
*/
|
*/
|
||||||
bool do_zero_out_dram = false;
|
bool do_zero_out_dram = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If set, read the presence register, check it, and exit the simulation cleanly
|
||||||
|
*/
|
||||||
|
bool check_fingerprint_only = false;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If set, write the presence register and exit the simulation cleanly
|
||||||
|
*/
|
||||||
|
std::optional<uint32_t> write_fingerprint_only;
|
||||||
|
|
||||||
midas_time_t start_time;
|
midas_time_t start_time;
|
||||||
midas_time_t end_time;
|
midas_time_t end_time;
|
||||||
uint64_t start_hcycle = -1;
|
std::optional<uint64_t> start_hcycle;
|
||||||
uint64_t end_hcycle = 0;
|
std::optional<uint64_t> end_hcycle;
|
||||||
uint64_t end_tcycle = 0;
|
uint64_t end_tcycle = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -54,7 +54,10 @@ void mm_t::init(size_t sz, int lsz) {
|
||||||
size = sz;
|
size = sz;
|
||||||
}
|
}
|
||||||
|
|
||||||
mm_t::~mm_t() { munmap(data, this->size); }
|
mm_t::~mm_t() {
|
||||||
|
if (data)
|
||||||
|
munmap(data, this->size);
|
||||||
|
}
|
||||||
|
|
||||||
void mm_magic_t::init(size_t sz, int lsz) {
|
void mm_magic_t::init(size_t sz, int lsz) {
|
||||||
mm_t::init(sz, lsz);
|
mm_t::init(sz, lsz);
|
||||||
|
|
|
@ -56,7 +56,7 @@ $(OUT_DIR)/$(DRIVER_NAME)-debug: $(vcs_v) $(vcs_cc) $(emul_h)
|
||||||
mkdir -p $(OUT_DIR)
|
mkdir -p $(OUT_DIR)
|
||||||
rm -rf $(GEN_DIR)/$(DRIVER_NAME)-debug.csrc
|
rm -rf $(GEN_DIR)/$(DRIVER_NAME)-debug.csrc
|
||||||
rm -rf $(OUT_DIR)/$(DRIVER_NAME)-debug.daidir
|
rm -rf $(OUT_DIR)/$(DRIVER_NAME)-debug.daidir
|
||||||
$(VCS) $(vcs_rtl_flags) +define+DEBUG -o $@ $(vcs_v) $(vcs_cc)
|
$(VCS) $(vcs_rtl_flags) +define+FSDB +define+DEBUG -o $@ $(vcs_v) $(vcs_cc)
|
||||||
|
|
||||||
################################################################################
|
################################################################################
|
||||||
|
|
||||||
|
@ -83,4 +83,4 @@ $(OUT_DIR)/$(DRIVER_NAME)-post-synth-debug: $(vcs_v) $(vcs_cc) $(emul_h)
|
||||||
mkdir -p $(OUT_DIR)
|
mkdir -p $(OUT_DIR)
|
||||||
rm -rf $(GEN_DIR)/$(DRIVER_NAME)-post-synth-debug.csrc
|
rm -rf $(GEN_DIR)/$(DRIVER_NAME)-post-synth-debug.csrc
|
||||||
rm -rf $(OUT_DIR)/$(DRIVER_NAME)-post-synth-debug.daidir
|
rm -rf $(OUT_DIR)/$(DRIVER_NAME)-post-synth-debug.daidir
|
||||||
$(VCS) $(vcs_post_synth_flags) +define+DEBUG -o $@ $(vcs_v) $(vcs_cc)
|
$(VCS) $(vcs_post_synth_flags) +define+FSDB +define+DEBUG -o $@ $(vcs_v) $(vcs_cc)
|
||||||
|
|
|
@ -10,7 +10,7 @@
|
||||||
#include "bridges/cpu_managed_stream.h"
|
#include "bridges/cpu_managed_stream.h"
|
||||||
#include "core/simif.h"
|
#include "core/simif.h"
|
||||||
|
|
||||||
#define PCI_DEV_FMT "%04x:%02x:%02x.%d"
|
#define PCI_DEV_FMT "%04x:%02d:%02x.%d"
|
||||||
|
|
||||||
class simif_xilinx_alveo_u250_t final : public simif_t,
|
class simif_xilinx_alveo_u250_t final : public simif_t,
|
||||||
public CPUManagedStreamIO {
|
public CPUManagedStreamIO {
|
||||||
|
@ -49,7 +49,11 @@ private:
|
||||||
};
|
};
|
||||||
|
|
||||||
static int fpga_pci_check_file_id(char *path, uint16_t id) {
|
static int fpga_pci_check_file_id(char *path, uint16_t id) {
|
||||||
assert(path);
|
if (path) {
|
||||||
|
fprintf(stdout, "Opening %s\n", path);
|
||||||
|
} else {
|
||||||
|
assert(path);
|
||||||
|
}
|
||||||
int ret = 0;
|
int ret = 0;
|
||||||
FILE *fp = fopen(path, "r");
|
FILE *fp = fopen(path, "r");
|
||||||
assert(fp);
|
assert(fp);
|
||||||
|
@ -112,8 +116,10 @@ void simif_xilinx_alveo_u250_t::check_rc(int rc, char *infostr) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void simif_xilinx_alveo_u250_t::fpga_shutdown() {
|
void simif_xilinx_alveo_u250_t::fpga_shutdown() {
|
||||||
int ret = munmap(bar0_base, bar0_size);
|
if (bar0_base) {
|
||||||
assert(ret == 0);
|
int ret = munmap(bar0_base, bar0_size);
|
||||||
|
assert(ret == 0);
|
||||||
|
}
|
||||||
close(edma_write_fd);
|
close(edma_write_fd);
|
||||||
close(edma_read_fd);
|
close(edma_read_fd);
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,16 @@ class SimulationMaster(implicit p: Parameters) extends Widget()(p) {
|
||||||
when (initDelay =/= 0.U) { initDelay := initDelay - 1.U }
|
when (initDelay =/= 0.U) { initDelay := initDelay - 1.U }
|
||||||
genRORegInit(initDelay === 0.U, "INIT_DONE", 0.U)
|
genRORegInit(initDelay === 0.U, "INIT_DONE", 0.U)
|
||||||
|
|
||||||
|
// add fingerprint to see if device is FireSim-enabled
|
||||||
|
val fingerprint = 0x46697265
|
||||||
|
val rFingerprint = RegInit(fingerprint.U(32.W))
|
||||||
|
genROReg(rFingerprint, "PRESENCE_READ")
|
||||||
|
|
||||||
|
val wFingerprint = genWORegInit(Wire(UInt(32.W)), "PRESENCE_WRITE", fingerprint.U(32.W))
|
||||||
|
when (wFingerprint =/= rFingerprint) {
|
||||||
|
rFingerprint := wFingerprint
|
||||||
|
}
|
||||||
|
|
||||||
genCRFile()
|
genCRFile()
|
||||||
|
|
||||||
override def genHeader(base: BigInt, memoryRegions: Map[String, BigInt], sb: StringBuilder): Unit = {
|
override def genHeader(base: BigInt, memoryRegions: Map[String, BigInt], sb: StringBuilder): Unit = {
|
||||||
|
|
|
@ -141,27 +141,41 @@ module emul(
|
||||||
|
|
||||||
`ifndef VERILATOR
|
`ifndef VERILATOR
|
||||||
`ifdef DEBUG
|
`ifdef DEBUG
|
||||||
reg [2047:0] vcdplusfile = 2048'h0;
|
reg [2047:0] waveformfile = 2048'h0;
|
||||||
reg [63:0] dump_start = 64'h0;
|
reg [63:0] dump_start = 64'h0;
|
||||||
reg [63:0] dump_end = {64{1'b1}};
|
reg [63:0] dump_end = {64{1'b1}};
|
||||||
reg [63:0] dump_cycles = 64'h0;
|
reg [63:0] dump_cycles = 64'h0;
|
||||||
reg [63:0] trace_count = 64'h0;
|
reg [63:0] trace_count = 64'h0;
|
||||||
|
|
||||||
initial begin
|
initial begin
|
||||||
if ($value$plusargs("waveform=%s", vcdplusfile))
|
if ($value$plusargs("waveform=%s", waveformfile))
|
||||||
begin
|
begin
|
||||||
$value$plusargs("dump-start=%d", dump_start);
|
$value$plusargs("dump-start=%d", dump_start);
|
||||||
if ($value$plusargs("dump-cycles=%d", dump_cycles)) begin
|
if ($value$plusargs("dump-cycles=%d", dump_cycles)) begin
|
||||||
dump_end = dump_start + dump_cycles;
|
dump_end = dump_start + dump_cycles;
|
||||||
end
|
end
|
||||||
|
|
||||||
$vcdplusfile(vcdplusfile);
|
`ifdef FSDB
|
||||||
|
$fsdbDumpfile(waveformfile);
|
||||||
|
$fsdbDumpvars("+all");
|
||||||
|
`else
|
||||||
|
$vcdplusfile(waveformfile);
|
||||||
|
`endif
|
||||||
|
|
||||||
wait (trace_count >= dump_start) begin
|
wait (trace_count >= dump_start) begin
|
||||||
$vcdpluson(0);
|
`ifdef FSDB
|
||||||
$vcdplusmemon(0);
|
$fsdbDumpon;
|
||||||
|
`else
|
||||||
|
$vcdpluson(0);
|
||||||
|
$vcdplusmemon(0);
|
||||||
|
`endif
|
||||||
end
|
end
|
||||||
wait ((trace_count > dump_end) || fin) begin
|
wait ((trace_count > dump_end) || fin) begin
|
||||||
$vcdplusclose;
|
`ifdef FSDB
|
||||||
|
$fsdbDumpoff;
|
||||||
|
`else
|
||||||
|
$vcdplusclose;
|
||||||
|
`endif
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -44,28 +44,28 @@ sim_binary_basename := $(basename $(notdir $(SIM_BINARY)))
|
||||||
|
|
||||||
run-verilator: $(verilator)
|
run-verilator: $(verilator)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(verilator) +permissive $(verilator_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(verilator) +permissive $(verilator_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out </dev/null
|
||||||
|
|
||||||
run-verilator-debug: $(verilator_debug)
|
run-verilator-debug: $(verilator_debug)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(verilator_debug) +permissive $(verilator_args) +waveform=$(sim_binary_basename).vpd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(verilator_debug) +permissive $(verilator_args) +waveform=$(sim_binary_basename).vcd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out </dev/null
|
||||||
|
|
||||||
run-vcs: $(vcs)
|
run-vcs: $(vcs)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(vcs) +permissive $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(vcs) +permissive $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out
|
||||||
|
|
||||||
run-vcs-debug: $(vcs_debug)
|
run-vcs-debug: $(vcs_debug)
|
||||||
cd $(dir $<) && \
|
cd $(dir $<) && \
|
||||||
$(vcs_debug) +permissive $(vcs_args) +waveform=$(sim_binary_basename).vpd $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) \
|
$(vcs_debug) +permissive $(vcs_args) +waveform=$(sim_binary_basename).fsdb $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +permissive-off $(abspath $(SIM_BINARY)) </dev/null \
|
||||||
$(disasm) $(sim_binary_basename).out
|
$(disasm) $(sim_binary_basename).out </dev/null
|
||||||
|
|
||||||
.PHONY: run-xsim
|
.PHONY: run-xsim
|
||||||
run-xsim: $(xsim)
|
run-xsim: $(xsim)
|
||||||
cd $(dir $<) && ./$(notdir $<) +permissive $(COMMON_SIM_ARGS) $(FPGA_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
cd $(dir $<) && ./$(notdir $<) +permissive $(COMMON_SIM_ARGS) $(FPGA_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
||||||
+permissive-off $(abspath $(SIM_BINARY))
|
+permissive-off $(abspath $(SIM_BINARY)) </dev/null
|
||||||
|
|
||||||
############################################
|
############################################
|
||||||
# Midas-Level Simulation Execution Recipes #
|
# Midas-Level Simulation Execution Recipes #
|
||||||
|
@ -110,7 +110,12 @@ $(OUTPUT_DIR)/%.vpd: $(OUTPUT_DIR)/% $(EMUL)-debug
|
||||||
./$(notdir $($(EMUL)_debug)) $< +waveform=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
./$(notdir $($(EMUL)_debug)) $< +waveform=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
||||||
$(disasm) $(patsubst %.vpd,%.out,$@) && [ $$PIPESTATUS -eq 0 ]
|
$(disasm) $(patsubst %.vpd,%.out,$@) && [ $$PIPESTATUS -eq 0 ]
|
||||||
|
|
||||||
.PRECIOUS: $(OUTPUT_DIR)/%.vpd $(OUTPUT_DIR)/%.out $(OUTPUT_DIR)/%.run
|
$(OUTPUT_DIR)/%.fsdb: $(OUTPUT_DIR)/% $(EMUL)-debug
|
||||||
|
cd $(dir $($(EMUL)_debug)) && \
|
||||||
|
./$(notdir $($(EMUL)_debug)) $< +waveform=$@ $($*_ARGS) $($(EMUL)_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) \
|
||||||
|
$(disasm) $(patsubst %.fsdb,%.out,$@) && [ $$PIPESTATUS -eq 0 ]
|
||||||
|
|
||||||
|
.PRECIOUS: $(OUTPUT_DIR)/%.fsdb $(OUTPUT_DIR)/%.vpd $(OUTPUT_DIR)/%.out $(OUTPUT_DIR)/%.run
|
||||||
|
|
||||||
# TraceGen rules
|
# TraceGen rules
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
|
||||||
|
set -ex
|
||||||
|
|
||||||
|
make \
|
||||||
|
PLATFORM=xilinx_alveo_u250 \
|
||||||
|
TARGET_PROJECT=firesim \
|
||||||
|
DESIGN=FireSim \
|
||||||
|
TARGET_CONFIG=FireSimRocketConfig \
|
||||||
|
PLATFORM_CONFIG=BaseXilinxAlveoConfig \
|
||||||
|
SIM_BINARY=/scratch/abejgonza/firesim-private/sw/firesim-software/test/bare/hello \
|
||||||
|
clean
|
Loading…
Reference in New Issue