replica
/
firesim
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve fabric logging (#1159) 

Co-authored-by: Filip Stamenkovic <filip.stamenkovic@sifive.com>
Co-authored-by: Tim Snyder <timothy.snyder@sifive.com>
This commit is contained in:
mergify[bot] 2022-08-09 11:03:53 -05:00 committed by GitHub
parent f24a3e256e
commit 83e8083085
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 256 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/scripts/common.py vendored
View File

@ -7,8 +7,8 @@ from ci_variables import ci_firesim_dir, local_fsim_dir, ci_gha_api_url, ci_repo
# Reuse manager utilities
# Note: ci_firesim_dir must not be used here because the persistent clone my not be initialized yet.
sys.path.append(local_fsim_dir + "/deploy/awstools")
from awstools import get_instances_with_filter
sys.path.append(local_fsim_dir + "/deploy")
from awstools.awstools import get_instances_with_filter
# Github URL related constants
gha_api_url = f"{ci_gha_api_url}/repos/{ci_repo_name}/actions"

4
.github/scripts/cull-old-ci-instances.py vendored
View File

@ -12,8 +12,8 @@ from common import unique_tag_key, deregister_runner_if_exists
# Reuse manager utilities
from ci_variables import ci_workdir, ci_personal_api_token, ci_workflow_run_id
sys.path.append(ci_workdir + "/deploy/awstools")
from awstools import get_instances_with_filter
sys.path.append(ci_workdir + "/deploy")
from awstools.awstools import get_instances_with_filter
# The number of hours an instance may exist since its initial launch time
INSTANCE_LIFETIME_LIMIT_HOURS = 8

6
.github/scripts/launch-manager-instance.py vendored
View File

@ -9,8 +9,8 @@ from ci_variables import *
from common import *
# Reuse manager utilities
sys.path.append(ci_workdir + "/deploy/awstools")
import awstools
sys.path.append(ci_workdir + "/deploy")
import awstools.awstools
def main():
""" Spins up a new manager instance for our CI run """
@ -21,7 +21,7 @@ def main():
sys.exit(0)
print("Launching a fresh manager instance. This will take a couple minutes")
awstools.main([
awstools.awstools.main([
'launch',
'--inst_type', 'z1d.2xlarge',
'--market', 'spot',

7
.github/scripts/run-linux-poweroff-externally-provisioned.py vendored
View File

@ -8,10 +8,9 @@ from fabric.api import *
from common import manager_fsim_dir, set_fabric_firesim_pem
from ci_variables import ci_workdir, ci_workflow_run_id
sys.path.append(ci_workdir + "/deploy/awstools")
from awstools import get_instances_with_filter, get_private_ips_for_instances
sys.path.append(ci_workdir + "/deploy/util")
from filelineswap import file_line_swap
sys.path.append(ci_workdir + "/deploy")
from awstools.awstools import get_instances_with_filter, get_private_ips_for_instances
from util.filelineswap import file_line_swap
def run_linux_poweroff_externally_provisioned():
""" Runs Linux poweroff workloads using externally provisioned AWS run farm """

12
deploy/awstools/awstools.py
View File

@ -22,10 +22,16 @@ from mypy_boto3_ec2.service_resource import Instance as EC2InstanceResource
from mypy_boto3_ec2.type_defs import FilterTypeDef
from mypy_boto3_s3.literals import BucketLocationConstraintType
# setup basic config for logging
if __name__ == '__main__':
# setup basic config for logging
logging.basicConfig()
# use builtin.input because we aren't in a StreamLogger context
from builtins import input as firesim_input
else:
from util.io import firesim_input
rootLogger = logging.getLogger()
# this needs to be updated whenever the FPGA Dev AMI changes
@ -200,8 +206,6 @@ def awsinit() -> None:
# only run aws configure if we cannot already find valid creds
# this loops calling valid_aws_configure_creds until
rootLogger.info("Running aws configure. You must specify your AWS account info here to use the FireSim Manager.")
# DO NOT wrap this local call with StreamLogger, we don't want creds to get
# stored in the log
local("aws configure")
# check again
@ -209,7 +213,7 @@ def awsinit() -> None:
if not valid_creds:
rootLogger.info("Invalid AWS credentials. Try again.")
useremail = input("If you are a new user, supply your email address [abc@xyz.abc] for email notifications (leave blank if you do not want email notifications): ")
useremail = firesim_input("If you are a new user, supply your email address [abc@xyz.abc] for email notifications (leave blank if you do not want email notifications): ")
if useremail != "":
subscribe_to_firesim_topic(useremail)
else:

192
deploy/buildtools/bitbuilder.py
View File

@ -14,7 +14,6 @@ from fabric.contrib.project import rsync_project # type: ignore
from awstools.afitools import firesim_tags_to_description, copy_afi_to_all_regions
from awstools.awstools import send_firesim_notification, get_aws_userid, get_aws_region, auto_create_bucket, valid_aws_configure_creds, aws_resource_names, get_snsname_arn
from util.streamlogger import StreamLogger, InfoStreamLogger
# imports needed for python type checking
from typing import Optional, Dict, Any, TYPE_CHECKING
@ -29,8 +28,7 @@ def get_deploy_dir() -> str:
Returns:
Path to firesim/deploy directory.
"""
with StreamLogger('stdout'), StreamLogger('stderr'):
deploydir = local("pwd", capture=True)
deploydir = local("pwd", capture=True)
return deploydir
class BitBuilder(metaclass=abc.ABCMeta):
@ -115,9 +113,7 @@ class F1BitBuilder(BitBuilder):
prefix(f'export PATH={os.getenv("PATH", "")}'), \
prefix(f'export LD_LIBRARY_PATH={os.getenv("LD_LIBRARY_PATH", "")}'), \
prefix('source sourceme-f1-manager.sh'), \
prefix('cd sim/'), \
InfoStreamLogger('stdout'), \
InfoStreamLogger('stderr'):
prefix('cd sim/'):
run(self.build_config.make_recipe("PLATFORM=f1 replace-rtl"))
def build_driver(self) -> None:
@ -128,9 +124,7 @@ class F1BitBuilder(BitBuilder):
prefix(f'export PATH={os.getenv("PATH", "")}'), \
prefix(f'export LD_LIBRARY_PATH={os.getenv("LD_LIBRARY_PATH", "")}'), \
prefix('source sourceme-f1-manager.sh'), \
prefix('cd sim/'), \
InfoStreamLogger('stdout'), \
InfoStreamLogger('stderr'):
prefix('cd sim/'):
run(self.build_config.make_recipe("PLATFORM=f1 driver"))
def cl_dir_setup(self, chisel_triplet: str, dest_build_dir: str) -> str:
@ -155,24 +149,23 @@ class F1BitBuilder(BitBuilder):
# do the rsync, but ignore any checkpoints that might exist on this machine
# (in case builds were run locally)
# extra_opts -l preserves symlinks
with StreamLogger('stdout'), StreamLogger('stderr'):
run(f'mkdir -p {dest_f1_platform_dir}')
rsync_cap = rsync_project(
local_dir=local_awsfpga_dir,
remote_dir=dest_f1_platform_dir,
ssh_opts="-o StrictHostKeyChecking=no",
exclude=["hdk/cl/developer_designs/cl_*"],
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
rsync_cap = rsync_project(
local_dir=f"{local_awsfpga_dir}/{fpga_build_postfix}/*",
remote_dir=f'{dest_awsfpga_dir}/{fpga_build_postfix}',
exclude=["build/checkpoints"],
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
run(f'mkdir -p {dest_f1_platform_dir}')
rsync_cap = rsync_project(
local_dir=local_awsfpga_dir,
remote_dir=dest_f1_platform_dir,
ssh_opts="-o StrictHostKeyChecking=no",
exclude=["hdk/cl/developer_designs/cl_*"],
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
rsync_cap = rsync_project(
local_dir=f"{local_awsfpga_dir}/{fpga_build_postfix}/*",
remote_dir=f'{dest_awsfpga_dir}/{fpga_build_postfix}',
exclude=["build/checkpoints"],
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
return f"{dest_awsfpga_dir}/{fpga_build_postfix}"
@ -213,27 +206,27 @@ class F1BitBuilder(BitBuilder):
cl_dir = self.cl_dir_setup(self.build_config.get_chisel_triplet(), build_farm.get_build_host(self.build_config).dest_build_dir)
vivado_result = 0
with InfoStreamLogger('stdout'), InfoStreamLogger('stderr'):
# copy script to the cl_dir and execute
rsync_cap = rsync_project(
local_dir=f"{local_deploy_dir}/../platforms/f1/build-bitstream.sh",
remote_dir=f"{cl_dir}/",
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
# copy script to the cl_dir and execute
rsync_cap = rsync_project(
local_dir=f"{local_deploy_dir}/../platforms/f1/build-bitstream.sh",
remote_dir=f"{cl_dir}/",
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
vivado_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
# put build results in the result-build area
with StreamLogger('stdout'), StreamLogger('stderr'):
rsync_cap = rsync_project(
local_dir=f"{local_results_dir}/",
remote_dir=cl_dir,
ssh_opts="-o StrictHostKeyChecking=no", upload=False, extra_opts="-l",
capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
rsync_cap = rsync_project(
local_dir=f"{local_results_dir}/",
remote_dir=cl_dir,
ssh_opts="-o StrictHostKeyChecking=no", upload=False, extra_opts="-l",
capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
if vivado_result != 0:
on_build_failure()
@ -275,9 +268,8 @@ class F1BitBuilder(BitBuilder):
assert len(tag_buildtriplet) <= 255, "ERR: aws does not support tags longer than 256 chars for buildtriplet"
assert len(tag_deploytriplet) <= 255, "ERR: aws does not support tags longer than 256 chars for deploytriplet"
with StreamLogger('stdout'), StreamLogger('stderr'):
is_dirty_str = local("if [[ $(git status --porcelain) ]]; then echo '-dirty'; fi", capture=True)
hash = local("git rev-parse HEAD", capture=True)
is_dirty_str = local("if [[ $(git status --porcelain) ]]; then echo '-dirty'; fi", capture=True)
hash = local("git rev-parse HEAD", capture=True)
tag_fsimcommit = hash + is_dirty_str
assert len(tag_fsimcommit) <= 255, "ERR: aws does not support tags longer than 256 chars for fsimcommit"
@ -289,7 +281,7 @@ class F1BitBuilder(BitBuilder):
# append the build node IP + a random string to diff them in s3
global_append = "-" + str(env.host_string) + "-" + ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(10)) + ".tar"
with lcd(f"{local_results_dir}/cl_{tag_buildtriplet}/build/checkpoints/to_aws/"), StreamLogger('stdout'), StreamLogger('stderr'):
with lcd(f"{local_results_dir}/cl_{tag_buildtriplet}/build/checkpoints/to_aws/"):
files = local('ls *.tar', capture=True)
rootLogger.debug(files)
rootLogger.debug(files.stderr)
@ -310,7 +302,7 @@ class F1BitBuilder(BitBuilder):
rootLogger.info("Waiting for create-fpga-image completion.")
checkstate = "pending"
with lcd(local_results_dir), StreamLogger('stdout'), StreamLogger('stderr'):
with lcd(local_results_dir):
while checkstate == "pending":
imagestate = local(f"aws ec2 describe-fpga-images --fpga-image-id {afi} | tee AGFI_INFO", capture=True)
state_as_dict = json.loads(imagestate)
@ -344,10 +336,9 @@ class F1BitBuilder(BitBuilder):
outputfile.write(agfi_entry)
if self.build_config.post_build_hook:
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
rootLogger.info(f"Build complete! AFI ready. See {os.path.join(hwdb_entry_file_location,afiname)}.")
return True
@ -374,9 +365,7 @@ class VitisBitBuilder(BitBuilder):
prefix(f'export PATH={os.getenv("PATH", "")}'), \
prefix(f'export LD_LIBRARY_PATH={os.getenv("LD_LIBRARY_PATH", "")}'), \
prefix('source sourceme-f1-manager.sh'), \
prefix('cd sim/'), \
InfoStreamLogger('stdout'), \
InfoStreamLogger('stderr'):
prefix('cd sim/'):
run(self.build_config.make_recipe("PLATFORM=vitis replace-rtl"))
def build_driver(self):
@ -387,9 +376,7 @@ class VitisBitBuilder(BitBuilder):
prefix(f'export PATH={os.getenv("PATH", "")}'), \
prefix(f'export LD_LIBRARY_PATH={os.getenv("LD_LIBRARY_PATH", "")}'), \
prefix('source sourceme-f1-manager.sh'), \
prefix('cd sim/'), \
InfoStreamLogger('stdout'), \
InfoStreamLogger('stderr'):
prefix('cd sim/'):
run(self.build_config.make_recipe("PLATFORM=vitis driver"))
def cl_dir_setup(self, chisel_triplet: str, dest_build_dir: str) -> str:
@ -413,23 +400,23 @@ class VitisBitBuilder(BitBuilder):
# do the rsync, but ignore any checkpoints that might exist on this machine
# (in case builds were run locally)
# extra_opts -l preserves symlinks
with StreamLogger('stdout'), StreamLogger('stderr'):
run('mkdir -p {}'.format(dest_vitis_dir))
rsync_cap = rsync_project(
local_dir=local_vitis_dir,
remote_dir=dest_vitis_dir,
ssh_opts="-o StrictHostKeyChecking=no",
exclude="cl_*",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
rsync_cap = rsync_project(
local_dir="{}/{}/".format(local_vitis_dir, fpga_build_postfix),
remote_dir='{}/{}'.format(dest_vitis_dir, fpga_build_postfix),
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
run('mkdir -p {}'.format(dest_vitis_dir))
rsync_cap = rsync_project(
local_dir=local_vitis_dir,
remote_dir=dest_vitis_dir,
ssh_opts="-o StrictHostKeyChecking=no",
exclude="cl_*",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
rsync_cap = rsync_project(
local_dir="{}/{}/".format(local_vitis_dir, fpga_build_postfix),
remote_dir='{}/{}'.format(dest_vitis_dir, fpga_build_postfix),
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
return f"{dest_vitis_dir}/{fpga_build_postfix}"
@ -469,33 +456,32 @@ class VitisBitBuilder(BitBuilder):
# TODO: Does this still apply or is this done in the Makefile
## copy over generated RTL into local CL_DIR before remote
#with InfoStreamLogger('stdout'), InfoStreamLogger('stderr'):
# run("""mkdir -p {}""".format(local_results_dir))
# run("""cp {}/design/FireSim-generated.sv {}/FireSim-generated.sv""".format(cl_dir, local_results_dir))
#
#run("""mkdir -p {}""".format(local_results_dir))
#run("""cp {}/design/FireSim-generated.sv {}/FireSim-generated.sv""".format(cl_dir, local_results_dir))
vitis_result = 0
with InfoStreamLogger('stdout'), InfoStreamLogger('stderr'):
# TODO: Put script within Vitis area
# copy script to the cl_dir and execute
rsync_cap = rsync_project(
local_dir=f"{local_deploy_dir}/../platforms/vitis/build-bitstream.sh",
remote_dir=f"{cl_dir}/",
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
# TODO: Put script within Vitis area
# copy script to the cl_dir and execute
rsync_cap = rsync_project(
local_dir=f"{local_deploy_dir}/../platforms/vitis/build-bitstream.sh",
remote_dir=f"{cl_dir}/",
ssh_opts="-o StrictHostKeyChecking=no",
extra_opts="-l", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
vitis_result = run(f"{cl_dir}/build-bitstream.sh {cl_dir}").return_code
# put build results in the result-build area
with StreamLogger('stdout'), StreamLogger('stderr'):
rsync_cap = rsync_project(
local_dir=f"{local_results_dir}/",
remote_dir=cl_dir,
ssh_opts="-o StrictHostKeyChecking=no", upload=False, extra_opts="-l",
capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
rsync_cap = rsync_project(
local_dir=f"{local_results_dir}/",
remote_dir=cl_dir,
ssh_opts="-o StrictHostKeyChecking=no", upload=False, extra_opts="-l",
capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
if vitis_result != 0:
on_build_failure()
@ -526,10 +512,10 @@ class VitisBitBuilder(BitBuilder):
outputfile.write(hwdb_entry)
if self.build_config.post_build_hook:
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
localcap = local(f"{self.build_config.post_build_hook} {local_results_dir}", capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
rootLogger.info(f"Build complete! Vitis bitstream ready. See {os.path.join(hwdb_entry_file_location,hwdb_entry_name)}.")

34
deploy/firesim
View File

@ -34,8 +34,9 @@ from awstools.afitools import share_agfi_in_all_regions
from buildtools.buildconfigfile import BuildConfigFile
from buildtools.bitbuilder import F1BitBuilder
from util.streamlogger import StreamLogger
from util.streamlogger import StreamLogger, InfoStreamLogger
from util.filelineswap import file_line_swap
from util.io import firesim_input
from typing import Dict, Callable, Type, Optional, TypedDict, get_type_hints, Tuple, List
@ -260,7 +261,7 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None:
def release_build_hosts_handler(sig, frame) -> None:
""" Handler that prompts to release build farm hosts if you press ctrl-c. """
rootLogger.info("You pressed ctrl-c, so builds have been killed.")
userconfirm = input("Do you also want to terminate your build hosts? Type 'yes' to do so.\n")
userconfirm = firesim_input("Do you also want to terminate your build hosts? Type 'yes' to do so.\n")
if userconfirm == "yes":
build_config_file.release_build_hosts()
rootLogger.info("Build farm hosts released.")
@ -499,17 +500,18 @@ if __name__ == '__main__':
check_env()
# lastly - whenever you use run/local/put/etc from fabric, you need to wrap
# it up in "with util.StreamLogger('stdout'), util.StreamLogger('stdin').
# unfortunately there's no proper way to do it with fabric
exitcode = 0
try:
main(args)
except:
# log all exceptions that make it this far
rootLogger.exception("Fatal error.")
exitcode = 1
finally:
rootLogger.info("""The full log of this run is:\n{basedir}/{fulllog}""".format(basedir=dname, fulllog=full_log_filename))
sys.exit(exitcode)
# lastly - we want anything printed to stdout to be converted into a DEBUG
# level logging message and anything printed to stderr converted into INFO.
# This is primarily because fabric does not use logging, it prints explicitly
# to stdout and stderr. We want it's output to be logged.
with StreamLogger('stdout'), InfoStreamLogger('stderr'):
exitcode = 0
try:
main(args)
except:
# log all exceptions that make it this far
rootLogger.exception("Fatal error.")
exitcode = 1
finally:
rootLogger.info("""The full log of this run is:\n{basedir}/{fulllog}""".format(basedir=dname, fulllog=full_log_filename))
sys.exit(exitcode)

65
deploy/runtools/firesim_topology_elements.py
View File

@ -10,7 +10,6 @@ from fabric.exceptions import CommandTimeout # type: ignore
from runtools.switch_model_config import AbstractSwitchToSwitchConfig
from runtools.utils import get_local_shared_libraries
from util.streamlogger import StreamLogger
from runtools.simulation_data_classes import TracerVConfig, AutoCounterConfig, HostDebugConfig, SynthPrintConfig
from typing import Optional, List, Tuple, Sequence, Union, TYPE_CHECKING
@ -327,15 +326,15 @@ class FireSimServerNode(FireSimNode):
simserverindex = slotno
job_results_dir = self.get_job().parent_workload.job_results_dir
job_dir = """{}/{}/""".format(job_results_dir, jobinfo.jobname)
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local("""mkdir -p {}""".format(job_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
localcap = local("""mkdir -p {}""".format(job_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
# add hw config summary per job
localcap = local("""echo "{}" > {}/HW_CFG_SUMMARY""".format(str(self.server_hardware_config), job_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
# add hw config summary per job
localcap = local("""echo "{}" > {}/HW_CFG_SUMMARY""".format(str(self.server_hardware_config), job_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
dest_sim_dir = self.get_host_instance().get_sim_dir()
@ -365,25 +364,25 @@ class FireSimServerNode(FireSimNode):
if rfsname is not None:
is_qcow2 = rfsname.endswith(".qcow2")
mountpoint = """{}/sim_slot_{}/mountpoint""".format(dest_sim_dir, simserverindex)
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""{} mkdir -p {}""".format("sudo" if sudo else "", mountpoint))
run("""{} mkdir -p {}""".format("sudo" if sudo else "", mountpoint))
if is_qcow2:
host_inst = self.get_host_instance()
assert isinstance(host_inst.instance_deploy_manager, EC2InstanceDeployManager)
nbd_tracker = host_inst.instance_deploy_manager.nbd_tracker
assert nbd_tracker is not None
rfsname = nbd_tracker.get_nbd_for_imagename(rfsname)
else:
rfsname = """{}/sim_slot_{}/{}""".format(dest_sim_dir, simserverindex, rfsname)
if is_qcow2:
host_inst = self.get_host_instance()
assert isinstance(host_inst.instance_deploy_manager, EC2InstanceDeployManager)
nbd_tracker = host_inst.instance_deploy_manager.nbd_tracker
assert nbd_tracker is not None
rfsname = nbd_tracker.get_nbd_for_imagename(rfsname)
else:
rfsname = """{}/sim_slot_{}/{}""".format(dest_sim_dir, simserverindex, rfsname)
mount(rfsname, mountpoint, f"{dest_sim_dir}/sim_slot_{simserverindex}")
with warn_only():
# ignore if this errors. not all rootfses have /etc/sysconfig/nfs
run("""{} chattr -i {}/etc/sysconfig/nfs""".format("sudo" if sudo else "", mountpoint))
mount(rfsname, mountpoint, f"{dest_sim_dir}/sim_slot_{simserverindex}")
with warn_only():
# ignore if this errors. not all rootfses have /etc/sysconfig/nfs
run("""{} chattr -i {}/etc/sysconfig/nfs""".format("sudo" if sudo else "", mountpoint))
## copy back files from inside the rootfs
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
for outputfile in jobinfo.outputs:
rsync_cap = rsync_project(remote_dir=mountpoint + outputfile,
local_dir=job_dir,
@ -395,20 +394,18 @@ class FireSimServerNode(FireSimNode):
rootLogger.debug(rsync_cap.stderr)
## unmount
with StreamLogger('stdout'), StreamLogger('stderr'):
umount(mountpoint, f"{dest_sim_dir}/sim_slot_{simserverindex}")
umount(mountpoint, f"{dest_sim_dir}/sim_slot_{simserverindex}")
## if qcow2, detach .qcow2 image from the device, we're done with it
if is_qcow2:
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""sudo qemu-nbd -d {devname}""".format(devname=rfsname))
run("""sudo qemu-nbd -d {devname}""".format(devname=rfsname))
## copy output files generated by the simulator that live on the host:
## e.g. uartlog, memory_stats.csv, etc
remote_sim_run_dir = """{}/sim_slot_{}/""".format(dest_sim_dir, simserverindex)
for simoutputfile in jobinfo.simoutputs:
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
rsync_cap = rsync_project(remote_dir=remote_sim_run_dir + simoutputfile,
local_dir=job_dir,
ssh_opts="-o StrictHostKeyChecking=no",
@ -717,10 +714,9 @@ class FireSimSwitchNode(FireSimNode):
"""
job_dir = """{}/switch{}/""".format(job_results_dir, self.switch_id_internal)
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local("""mkdir -p {}""".format(job_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
localcap = local("""mkdir -p {}""".format(job_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
dest_sim_dir = self.get_host_instance().get_sim_dir()
@ -728,8 +724,7 @@ class FireSimSwitchNode(FireSimNode):
## e.g. uartlog, memory_stats.csv, etc
remote_sim_run_dir = """{}/switch_slot_{}/""".format(dest_sim_dir, switch_slot_no)
for simoutputfile in ["switchlog"]:
with StreamLogger('stdout'), StreamLogger('stderr'):
get(remote_path=remote_sim_run_dir + simoutputfile, local_path=job_dir)
get(remote_path=remote_sim_run_dir + simoutputfile, local_path=job_dir)
def diagramstr(self) -> str:
msg = f"FireSimSwitchNode:{self.switch_id_internal}\n"

41
deploy/runtools/firesim_topology_with_passes.py
View File

@ -15,7 +15,6 @@ from runtools.firesim_topology_elements import FireSimServerNode, FireSimDummySe
from runtools.firesim_topology_core import FireSimTopology
from runtools.utils import MacAddress
from runtools.simulation_data_classes import TracerVConfig, AutoCounterConfig, HostDebugConfig, SynthPrintConfig
from util.streamlogger import StreamLogger
from typing import Dict, Any, cast, List, TYPE_CHECKING, Callable
if TYPE_CHECKING:
@ -30,8 +29,7 @@ def instance_liveness() -> None:
""" Confirm that all instances are accessible (are running and can be ssh'ed into) first so that we don't run any
actual firesim-related commands on only some of the run farm machines."""
rootLogger.info("""[{}] Checking if host instance is up...""".format(env.host_string))
with StreamLogger('stdout'), StreamLogger('stderr'):
run("uname -a")
run("uname -a")
class FireSimTopologyWithPasses:
""" This class constructs a FireSimTopology, then performs a series of passes
@ -458,15 +456,14 @@ class FireSimTopologyWithPasses:
rootLogger.info("Confirming exit...")
# keep checking screen until it reports that there are no screens left
while True:
with StreamLogger('stdout'), StreamLogger('stderr'):
screenoutput = run("screen -ls")
# If AutoILA is enabled, use the following condition
if "2 Sockets in" in screenoutput and "hw_server" in screenoutput and "virtual_jtag" in screenoutput:
break
# If AutoILA is disabled, use the following condition
elif "No Sockets found" in screenoutput:
break
time.sleep(1)
screenoutput = run("screen -ls")
# If AutoILA is enabled, use the following condition
if "2 Sockets in" in screenoutput and "hw_server" in screenoutput and "virtual_jtag" in screenoutput:
break
# If AutoILA is disabled, use the following condition
elif "No Sockets found" in screenoutput:
break
time.sleep(1)
execute(screens, hosts=all_run_farm_ips)
@ -477,10 +474,9 @@ class FireSimTopologyWithPasses:
all_run_farm_ips = [x.get_host() for x in self.run_farm.get_all_bound_host_nodes()]
rootLogger.info("""Creating the directory: {}""".format(self.workload.job_results_dir))
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local("""mkdir -p {}""".format(self.workload.job_results_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
localcap = local("""mkdir -p {}""".format(self.workload.job_results_dir), capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
# boot up as usual
self.boot_simulation_passes(False, skip_instance_binding=True)
@ -634,13 +630,12 @@ class FireSimTopologyWithPasses:
# run post-workload hook, if one exists
if self.workload.post_run_hook is not None:
rootLogger.info("Running post_run_hook...")
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local("""cd {} && {} {}""".format(self.workload.workload_input_base_dir,
self.workload.post_run_hook,
self.workload.job_results_dir),
capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
localcap = local("""cd {} && {} {}""".format(self.workload.workload_input_base_dir,
self.workload.post_run_hook,
self.workload.job_results_dir),
capture=True)
rootLogger.debug("[localhost] " + str(localcap))
rootLogger.debug("[localhost] " + str(localcap.stderr))
rootLogger.info("FireSim Simulation Exited Successfully. See results in:\n" + str(self.workload.job_results_dir))

4
deploy/runtools/run_farm.py
View File

@ -13,8 +13,8 @@ import pprint
from collections import defaultdict
from awstools.awstools import instances_sorted_by_avail_ip, get_run_instances_by_tag_type, get_private_ips_for_instances, launch_run_instances, wait_on_instance_launches, terminate_instances, get_instance_ids_for_instances, aws_resource_names, MockBoto3Instance
from util.streamlogger import StreamLogger
from util.inheritors import inheritors
from util.io import firesim_input
from runtools.run_farm_deploy_managers import InstanceDeployManager, EC2InstanceDeployManager
from typing import Any, Dict, Optional, List, Union, Set, Type, Tuple, TYPE_CHECKING
@ -457,7 +457,7 @@ class AWSEC2F1(RunFarm):
if not forceterminate:
# --forceterminate was not supplied, so confirm with the user
userconfirm = input("Type yes, then press enter, to continue. Otherwise, the operation will be cancelled.\n")
userconfirm = firesim_input("Type yes, then press enter, to continue. Otherwise, the operation will be cancelled.\n")
else:
userconfirm = "yes"

133
deploy/runtools/run_farm_deploy_managers.py
View File

@ -11,7 +11,6 @@ from fabric.contrib.project import rsync_project # type: ignore
import time
from os.path import join as pjoin
from util.streamlogger import StreamLogger
from awstools.awstools import terminate_instances, get_instance_ids_for_instances
from runtools.utils import has_sudo
@ -108,12 +107,11 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
"""
if self.nbd_tracker is not None:
self.instance_logger("""Setting up remote node for qcow2 disk images.""")
with StreamLogger('stdout'), StreamLogger('stderr'):
# get qemu-nbd
### XXX Centos Specific
run('sudo yum -y install qemu-img')
# copy over kernel module
put('../build/nbd.ko', '/home/centos/nbd.ko', mirror_local_mode=True)
# get qemu-nbd
### XXX Centos Specific
run('sudo yum -y install qemu-img')
# copy over kernel module
put('../build/nbd.ko', '/home/centos/nbd.ko', mirror_local_mode=True)
def load_nbd_module(self) -> None:
""" If NBD is available, load the nbd module. always unload the module
@ -121,8 +119,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
if self.nbd_tracker is not None:
self.instance_logger("Loading NBD Kernel Module.")
self.unload_nbd_module()
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""sudo insmod /home/centos/nbd.ko nbds_max={}""".format(self.nbd_tracker.NBDS_MAX))
run("""sudo insmod /home/centos/nbd.ko nbds_max={}""".format(self.nbd_tracker.NBDS_MAX))
def unload_nbd_module(self) -> None:
""" If NBD is available, unload the nbd module. """
@ -131,7 +128,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
# disconnect all /dev/nbdX devices before rmmod
self.disconnect_all_nbds_instance()
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
run('sudo rmmod nbd')
def disconnect_all_nbds_instance(self) -> None:
@ -140,7 +137,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
self.instance_logger("Disconnecting all NBDs.")
# warn_only, so we can call this even if there are no nbds
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
# build up one large command with all the disconnects
fullcmd = []
for nbd_index in range(self.nbd_tracker.NBDS_MAX):
@ -160,20 +157,17 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
remote_sim_dir = """{}/sim_slot_{}/""".format(remote_home_dir, slotno)
remote_sim_rsync_dir = remote_sim_dir + "rsyncdir/"
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""mkdir -p {}""".format(remote_sim_rsync_dir))
run("""mkdir -p {}""".format(remote_sim_rsync_dir))
files_to_copy = serv.get_required_files_local_paths()
for local_path, remote_path in files_to_copy:
with StreamLogger('stdout'), StreamLogger('stderr'):
# -z --inplace
rsync_cap = rsync_project(local_dir=local_path, remote_dir=pjoin(remote_sim_rsync_dir, remote_path),
ssh_opts="-o StrictHostKeyChecking=no", extra_opts="-L", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
# -z --inplace
rsync_cap = rsync_project(local_dir=local_path, remote_dir=pjoin(remote_sim_rsync_dir, remote_path),
ssh_opts="-o StrictHostKeyChecking=no", extra_opts="-L", capture=True)
rootLogger.debug(rsync_cap)
rootLogger.debug(rsync_cap.stderr)
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""cp -r {}/* {}/""".format(remote_sim_rsync_dir, remote_sim_dir), shell=True)
run("""cp -r {}/* {}/""".format(remote_sim_rsync_dir, remote_sim_dir), shell=True)
def copy_switch_slot_infrastructure(self, switchslot: int) -> None:
""" copy all the switch infrastructure to the remote node. """
@ -181,15 +175,13 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
self.instance_logger("""Copying switch simulation infrastructure for switch slot: {}.""".format(switchslot))
remote_home_dir = self.parent_node.get_sim_dir()
remote_switch_dir = """{}/switch_slot_{}/""".format(remote_home_dir, switchslot)
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""mkdir -p {}""".format(remote_switch_dir))
run("""mkdir -p {}""".format(remote_switch_dir))
assert switchslot < len(self.parent_node.switch_slots)
switch = self.parent_node.switch_slots[switchslot]
files_to_copy = switch.get_required_files_local_paths()
for local_path, remote_path in files_to_copy:
with StreamLogger('stdout'), StreamLogger('stderr'):
put(local_path, pjoin(remote_switch_dir, remote_path), mirror_local_mode=True)
put(local_path, pjoin(remote_switch_dir, remote_path), mirror_local_mode=True)
def start_switch_slot(self, switchslot: int) -> None:
@ -200,7 +192,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
remote_switch_dir = """{}/switch_slot_{}/""".format(remote_home_dir, switchslot)
assert switchslot < len(self.parent_node.switch_slots)
switch = self.parent_node.switch_slots[switchslot]
with cd(remote_switch_dir), StreamLogger('stdout'), StreamLogger('stderr'):
with cd(remote_switch_dir):
run(switch.get_switch_start_command(has_sudo()))
def start_sim_slot(self, slotno: int) -> None:
@ -211,7 +203,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
remote_sim_dir = """{}/sim_slot_{}/""".format(remote_home_dir, slotno)
assert slotno < len(self.parent_node.sim_slots)
server = self.parent_node.sim_slots[slotno]
with cd(remote_sim_dir), StreamLogger('stdout'), StreamLogger('stderr'):
with cd(remote_sim_dir):
run(server.get_sim_start_command(slotno, has_sudo()))
@ -221,7 +213,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
self.instance_logger("""Killing switch simulation for switchslot: {}.""".format(switchslot))
assert switchslot < len(self.parent_node.switch_slots)
switch = self.parent_node.switch_slots[switchslot]
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
if has_sudo():
run("sudo " + switch.get_switch_kill_command())
else:
@ -233,7 +225,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
self.instance_logger(f"""Killing {self.sim_type_message} simulation for slot: {slotno}.""")
assert slotno < len(self.parent_node.sim_slots)
server = self.parent_node.sim_slots[slotno]
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
if has_sudo():
run("sudo " + server.get_sim_kill_command(slotno))
else:
@ -251,8 +243,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
"""Boot up all the switches on this host in screens."""
# remove shared mem pages used by switches
if self.instance_assigned_switches():
with StreamLogger('stdout'), StreamLogger('stderr'):
run("sudo rm -rf /dev/shm/*")
run("sudo rm -rf /dev/shm/*")
for slotno in range(len(self.parent_node.switch_slots)):
self.start_switch_slot(slotno)
@ -269,8 +260,7 @@ class InstanceDeployManager(metaclass=abc.ABCMeta):
if self.instance_assigned_switches():
for slotno in range(len(self.parent_node.switch_slots)):
self.kill_switch_slot(slotno)
with StreamLogger('stdout'), StreamLogger('stderr'):
run("sudo rm -rf /dev/shm/*")
run("sudo rm -rf /dev/shm/*")
def kill_simulations_instance(self, disconnect_all_nbds: bool = True) -> None:
""" Kill all simulations on this host. """
@ -445,18 +435,16 @@ class EC2InstanceDeployManager(InstanceDeployManager):
def get_and_install_aws_fpga_sdk(self) -> None:
""" Installs the aws-sdk. This gets us access to tools to flash the fpga. """
if self.instance_assigned_simulations():
with prefix('cd ../'), \
StreamLogger('stdout'), \
StreamLogger('stderr'):
with prefix('cd ../'):
# use local version of aws_fpga on run farm nodes
aws_fpga_upstream_version = local('git -C platforms/f1/aws-fpga describe --tags --always --dirty', capture=True)
if "-dirty" in aws_fpga_upstream_version:
rootLogger.critical("Unable to use local changes to aws-fpga. Continuing without them.")
self.instance_logger("""Installing AWS FPGA SDK on remote nodes. Upstream hash: {}""".format(aws_fpga_upstream_version))
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
run('git clone https://github.com/aws/aws-fpga')
run('cd aws-fpga && git checkout ' + aws_fpga_upstream_version)
with cd('/home/centos/aws-fpga'), StreamLogger('stdout'), StreamLogger('stderr'):
with cd('/home/centos/aws-fpga'):
run('source sdk_setup.sh')
def fpga_node_xdma(self) -> None:
@ -465,22 +453,21 @@ class EC2InstanceDeployManager(InstanceDeployManager):
"""
if self.instance_assigned_simulations():
self.instance_logger("""Copying AWS FPGA XDMA driver to remote node.""")
with StreamLogger('stdout'), StreamLogger('stderr'):
run('mkdir -p /home/centos/xdma/')
put('../platforms/f1/aws-fpga/sdk/linux_kernel_drivers',
'/home/centos/xdma/', mirror_local_mode=True)
with cd('/home/centos/xdma/linux_kernel_drivers/xdma/'), \
prefix("export PATH=/usr/bin:$PATH"):
# prefix only needed if conda env is earlier in PATH
# see build-setup-nolog.sh for explanation.
run('make clean')
run('make')
run('mkdir -p /home/centos/xdma/')
put('../platforms/f1/aws-fpga/sdk/linux_kernel_drivers',
'/home/centos/xdma/', mirror_local_mode=True)
with cd('/home/centos/xdma/linux_kernel_drivers/xdma/'), \
prefix("export PATH=/usr/bin:$PATH"):
# prefix only needed if conda env is earlier in PATH
# see build-setup-nolog.sh for explanation.
run('make clean')
run('make')
def unload_xrt_and_xocl(self) -> None:
if self.instance_assigned_simulations():
self.instance_logger("Unloading XRT-related Kernel Modules.")
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
# fpga mgmt tools seem to force load xocl after a flash now...
# so we just remove everything for good measure:
remote_kmsg("removing_xrt_start")
@ -492,7 +479,7 @@ class EC2InstanceDeployManager(InstanceDeployManager):
if self.instance_assigned_simulations():
self.instance_logger("Unloading XDMA Driver Kernel Module.")
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
# fpga mgmt tools seem to force load xocl after a flash now...
# so we just remove everything for good measure:
remote_kmsg("removing_xdma_start")
@ -507,17 +494,15 @@ class EC2InstanceDeployManager(InstanceDeployManager):
# we always clear ALL fpga slots
for slotno in range(self.parent_node.MAX_SIM_SLOTS_ALLOWED):
self.instance_logger("""Clearing FPGA Slot {}.""".format(slotno))
with StreamLogger('stdout'), StreamLogger('stderr'):
remote_kmsg("""about_to_clear_fpga{}""".format(slotno))
run("""sudo fpga-clear-local-image -S {} -A""".format(slotno))
remote_kmsg("""done_clearing_fpga{}""".format(slotno))
remote_kmsg("""about_to_clear_fpga{}""".format(slotno))
run("""sudo fpga-clear-local-image -S {} -A""".format(slotno))
remote_kmsg("""done_clearing_fpga{}""".format(slotno))
for slotno in range(self.parent_node.MAX_SIM_SLOTS_ALLOWED):
self.instance_logger("""Checking for Cleared FPGA Slot {}.""".format(slotno))
with StreamLogger('stdout'), StreamLogger('stderr'):
remote_kmsg("""about_to_check_clear_fpga{}""".format(slotno))
run("""until sudo fpga-describe-local-image -S {} -R -H | grep -q "cleared"; do sleep 1; done""".format(slotno))
remote_kmsg("""done_checking_clear_fpga{}""".format(slotno))
remote_kmsg("""about_to_check_clear_fpga{}""".format(slotno))
run("""until sudo fpga-describe-local-image -S {} -R -H | grep -q "cleared"; do sleep 1; done""".format(slotno))
remote_kmsg("""done_checking_clear_fpga{}""".format(slotno))
def flash_fpgas(self) -> None:
@ -527,9 +512,8 @@ class EC2InstanceDeployManager(InstanceDeployManager):
agfi = firesimservernode.get_agfi()
dummyagfi = agfi
self.instance_logger("""Flashing FPGA Slot: {} with agfi: {}.""".format(slotno, agfi))
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""sudo fpga-load-local-image -S {} -I {} -A""".format(
slotno, agfi))
run("""sudo fpga-load-local-image -S {} -I {} -A""".format(
slotno, agfi))
# We only do this because XDMA hangs if some of the FPGAs on the instance
# are left in the cleared state. So, if you're only using some of the
@ -539,19 +523,16 @@ class EC2InstanceDeployManager(InstanceDeployManager):
# break anything.
for slotno in range(len(self.parent_node.sim_slots), self.parent_node.MAX_SIM_SLOTS_ALLOWED):
self.instance_logger("""Flashing FPGA Slot: {} with dummy agfi: {}.""".format(slotno, dummyagfi))
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""sudo fpga-load-local-image -S {} -I {} -A""".format(
slotno, dummyagfi))
run("""sudo fpga-load-local-image -S {} -I {} -A""".format(
slotno, dummyagfi))
for slotno, firesimservernode in enumerate(self.parent_node.sim_slots):
self.instance_logger("""Checking for Flashed FPGA Slot: {} with agfi: {}.""".format(slotno, agfi))
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""until sudo fpga-describe-local-image -S {} -R -H | grep -q "loaded"; do sleep 1; done""".format(slotno))
run("""until sudo fpga-describe-local-image -S {} -R -H | grep -q "loaded"; do sleep 1; done""".format(slotno))
for slotno in range(len(self.parent_node.sim_slots), self.parent_node.MAX_SIM_SLOTS_ALLOWED):
self.instance_logger("""Checking for Flashed FPGA Slot: {} with agfi: {}.""".format(slotno, dummyagfi))
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""until sudo fpga-describe-local-image -S {} -R -H | grep -q "loaded"; do sleep 1; done""".format(slotno))
run("""until sudo fpga-describe-local-image -S {} -R -H | grep -q "loaded"; do sleep 1; done""".format(slotno))
def load_xdma(self) -> None:
@ -564,25 +545,22 @@ class EC2InstanceDeployManager(InstanceDeployManager):
# now load xdma
self.instance_logger("Loading XDMA Driver Kernel Module.")
# TODO: can make these values automatically be chosen based on link lat
with StreamLogger('stdout'), StreamLogger('stderr'):
run("sudo insmod /home/centos/xdma/linux_kernel_drivers/xdma/xdma.ko poll_mode=1")
run("sudo insmod /home/centos/xdma/linux_kernel_drivers/xdma/xdma.ko poll_mode=1")
def start_ila_server(self) -> None:
""" start the vivado hw_server and virtual jtag on simulation instance. """
if self.instance_assigned_simulations():
self.instance_logger("Starting Vivado hw_server.")
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""screen -S hw_server -d -m bash -c "script -f -c 'hw_server'"; sleep 1""")
run("""screen -S hw_server -d -m bash -c "script -f -c 'hw_server'"; sleep 1""")
self.instance_logger("Starting Vivado virtual JTAG.")
with StreamLogger('stdout'), StreamLogger('stderr'):
run("""screen -S virtual_jtag -d -m bash -c "script -f -c 'sudo fpga-start-virtual-jtag -P 10201 -S 0'"; sleep 1""")
run("""screen -S virtual_jtag -d -m bash -c "script -f -c 'sudo fpga-start-virtual-jtag -P 10201 -S 0'"; sleep 1""")
def kill_ila_server(self) -> None:
""" Kill the vivado hw_server and virtual jtag """
if self.instance_assigned_simulations():
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
run("sudo pkill -SIGKILL hw_server")
with warn_only(), StreamLogger('stdout'), StreamLogger('stderr'):
with warn_only():
run("sudo pkill -SIGKILL fpga-local-cmd")
@ -653,8 +631,7 @@ class VitisInstanceDeployManager(InstanceDeployManager):
card_bdfs = [d["bdf"] for d in json_dict["system"]["host"]["devices"]]
for card_bdf in card_bdfs:
with StreamLogger('stdout'), StreamLogger('stderr'):
run(f"xbutil reset -d {card_bdf} --force")
run(f"xbutil reset -d {card_bdf} --force")
def infrasetup_instance(self) -> None:
""" Handle infrastructure setup for this platform. """

3
deploy/runtools/runtime_config.py
View File

@ -18,7 +18,6 @@ from runtools.firesim_topology_with_passes import FireSimTopologyWithPasses
from runtools.workload import WorkloadConfig
from runtools.run_farm import RunFarm
from runtools.simulation_data_classes import TracerVConfig, AutoCounterConfig, HostDebugConfig, SynthPrintConfig
from util.streamlogger import StreamLogger
from util.inheritors import inheritors
from util.deepmerge import deep_merge
@ -253,8 +252,6 @@ class RuntimeHWConfig:
prefix('export LD_LIBRARY_PATH={}'.format(os.getenv('LD_LIBRARY_PATH', ""))), \
prefix('source ./sourceme-f1-manager.sh'), \
prefix('cd sim/'), \
StreamLogger('stdout'), \
StreamLogger('stderr'), \
prefix('set -o pipefail'):
localcap = None
with settings(warn_only=True):

8
deploy/runtools/switch_model_config.py
View File

@ -8,7 +8,6 @@ import random
import string
import logging
from fabric.api import local # type: ignore
from util.streamlogger import StreamLogger
from typing import TYPE_CHECKING
if TYPE_CHECKING:
@ -151,10 +150,9 @@ class AbstractSwitchToSwitchConfig:
def local_logged(command: str) -> None:
""" Run local command with logging. """
with StreamLogger('stdout'), StreamLogger('stderr'):
localcap = local(command, capture=True)
rootLogger.debug(localcap)
rootLogger.debug(localcap.stderr)
localcap = local(command, capture=True)
rootLogger.debug(localcap)
rootLogger.debug(localcap.stderr)
# make a build dir for this switch
local_logged("mkdir -p " + switchbuilddir)

23
deploy/util/io.py Normal file
View File

@ -0,0 +1,23 @@
import logging
def firesim_input(prompt: object = None) -> str:
"""wrap builtins.input() understanding the idiocyncracies of firesim+fabric+logging
Log the prompt at CRITICAL level so that it will go to the terminal and the log.
Log the entered text as DEBUG so that the log contains it.
Don't pass the prompt to builtins.input() because we don't need StreamLogger to also
be trying to log the prompt.
See 'streamlogger.py' and it's use at the end of 'firesim.py'
"""
rootLogger = logging.getLogger()
if prompt:
rootLogger.critical(prompt)
res = input()
rootLogger.debug("User Provided input():'%s'", res)
return res