replica
/
firesim
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1046 from firesim/what-to-build-v2 

Allow argument passing to bit builder
This commit is contained in:
Abraham Gonzalez 2022-06-14 11:48:32 -07:00 committed by GitHub
parent 1d4f48ba63 e0016cd059
commit 026ef9fae8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 130 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
deploy/awstools/awstools.py
View File

@ -241,6 +241,14 @@ def get_aws_userid() -> str:
else:
assert False, "Unable to obtain accountId from instance metadata"
def get_aws_region() -> str:
""" Get the user's current region to intelligently create a bucket name when doing awsinit(). """
info = get_localhost_instance_info("dynamic/instance-identity/document")
if info is not None:
return json.loads(info)['region'].lower()
else:
assert False, "Unable to obtain region from instance metadata"
def construct_instance_market_options(instancemarket: str, spotinterruptionbehavior: str, spotmaxprice: str) -> Dict[str, Any]:
""" construct the dictionary necessary to configure instance market selection
(on-demand vs spot)
@ -537,7 +545,7 @@ def auto_create_bucket(userbucketname: str) -> None:
s3cli.head_bucket(Bucket=userbucketname)
except s3cli.exceptions.ClientError as exc:
if 'Forbidden' in repr(exc):
rootLogger.critical("You tried to access a bucket that is Forbidden. This probably means that someone else has taken the name already.")
rootLogger.critical(f"You tried to access a bucket {userbucketname} that is Forbidden. This probably means that someone else has taken the name already.")
rootLogger.critical("The full exception is printed below:")
rootLogger.critical("____________________________________________________________")
rootLogger.critical(repr(exc))

20
deploy/bit-builder-recipes/f1.yaml Normal file
View File

@ -0,0 +1,20 @@
# Build-time bitbuilder design configuration for the FireSim Simulation Manager
# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params.
###########
# Schema:
###########
# # Class name of the bitbuilder type.
# # This can be determined from `deploy/buildtools/bitbuilder.py`).
# bitbuilder_type: <TYPE NAME>
# args:
# # Bitbuilder arguments that are passed to the `BitBuilder`
# # object. Determined by looking at `_parse_args` function of class.
# <K/V pairs of args>
bit_builder_type: F1BitBuilder
args:
# REQUIRED: name of s3 bucket
s3_bucket_name: firesim
# REQUIRED: append aws username and current region to s3_bucket_name?
append_userid_region: true

52
deploy/buildtools/bitbuilder.py
View File

@ -13,11 +13,11 @@ from fabric.contrib.console import confirm # type: ignore
from fabric.contrib.project import rsync_project # type: ignore
from awstools.afitools import firesim_tags_to_description, copy_afi_to_all_regions
from awstools.awstools import send_firesim_notification
from awstools.awstools import send_firesim_notification, get_aws_userid, get_aws_region, auto_create_bucket, valid_aws_configure_creds, aws_resource_names, get_snsname_arn
from util.streamlogger import StreamLogger, InfoStreamLogger
# imports needed for python type checking
from typing import Optional, TYPE_CHECKING
from typing import Optional, Dict, Any, TYPE_CHECKING
if TYPE_CHECKING:
from buildtools.buildconfig import BuildConfig
@ -38,15 +38,24 @@ class BitBuilder(metaclass=abc.ABCMeta):
Attributes:
build_config: Build config to build a bitstream for.
args: Args (i.e. options) passed to the bitbuilder.
"""
build_config: BuildConfig
args: Dict[str, Any]
def __init__(self, build_config: BuildConfig) -> None:
def __init__(self, build_config: BuildConfig, args: Dict[str, Any]) -> None:
"""
Args:
build_config: Build config to build a bitstream for.
args: Args (i.e. options) passed to the bitbuilder.
"""
self.build_config = build_config
self.args = args
@abc.abstractmethod
def setup(self) -> None:
"""Any setup needed before `replace_rtl`, `build_driver`, and `build_bitstream` is run."""
raise NotImplementedError
@abc.abstractmethod
def replace_rtl(self) -> None:
@ -69,7 +78,40 @@ class BitBuilder(metaclass=abc.ABCMeta):
raise NotImplementedError
class F1BitBuilder(BitBuilder):
"""Bit builder class that builds a AWS EC2 F1 AGFI (bitstream) from the build config."""
"""Bit builder class that builds a AWS EC2 F1 AGFI (bitstream) from the build config.
Attributes:
s3_bucketname: S3 bucketname for AFI builds.
"""
s3_bucketname: str
def __init__(self, build_config: BuildConfig, args: Dict[str, Any]) -> None:
"""
Args:
args: Args (i.e. options) passed to the build farm.
"""
super().__init__(build_config, args)
self._parse_args()
def _parse_args(self) -> None:
"""Parse bitbuilder arguments."""
self.s3_bucketname = self.args["s3_bucket_name"]
if self.args["append_userid_region"]:
self.s3_bucketname += "-" + get_aws_userid() + "-" + get_aws_region()
if valid_aws_configure_creds():
aws_resource_names_dict = aws_resource_names()
if aws_resource_names_dict['s3bucketname'] is not None:
# in tutorial mode, special s3 bucket name
self.s3_bucketname = aws_resource_names_dict['s3bucketname']
def setup(self) -> None:
auto_create_bucket(self.s3_bucketname)
# check to see email notifications can be subscribed
get_snsname_arn()
def replace_rtl(self) -> None:
"""Generate Verilog from build config."""
rootLogger.info(f"Building Verilog for {self.build_config.get_chisel_triplet()}")
@ -224,7 +266,7 @@ class F1BitBuilder(BitBuilder):
afi = None
agfi = None
s3bucket = self.build_config.s3_bucketname
s3bucket = self.s3_bucketname
afiname = self.build_config.name
# construct the "tags" we store in the AGFI description

40
deploy/buildtools/buildconfig.py
View File

@ -2,10 +2,12 @@ from __future__ import annotations
from time import strftime, gmtime
import pprint
from importlib import import_module
import yaml
from awstools.awstools import valid_aws_configure_creds, aws_resource_names
from buildtools.bitbuilder import BitBuilder
import buildtools
from util.deepmerge import deep_merge
# imports needed for python type checking
from typing import Set, Any, Optional, Dict, TYPE_CHECKING
@ -24,8 +26,8 @@ class BuildConfig:
deploytriplet: Deploy triplet override.
launch_time: Launch time of the manager.
PLATFORM_CONFIG: Platform config to build.
s3_bucketname: S3 bucketname for AFI builds.
post_build_hook: Post build hook script.
bitbuilder: bitstream configuration class.
"""
name: str
build_config_file: BuildConfigFile
@ -35,9 +37,8 @@ class BuildConfig:
deploytriplet: Optional[str]
launch_time: str
PLATFORM_CONFIG: str
s3_bucketname: str
post_build_hook: str
fpga_bit_builder_dispatcher: BitBuilder
bitbuilder: BitBuilder
def __init__(self,
name: str,
@ -62,19 +63,28 @@ class BuildConfig:
# run platform specific options
self.PLATFORM_CONFIG = recipe_config_dict['PLATFORM_CONFIG']
self.s3_bucketname = recipe_config_dict['s3_bucket_name']
if valid_aws_configure_creds():
aws_resource_names_dict = aws_resource_names()
if aws_resource_names_dict['s3bucketname'] is not None:
# in tutorial mode, special s3 bucket name
self.s3_bucketname = aws_resource_names_dict['s3bucketname']
self.post_build_hook = recipe_config_dict['post_build_hook']
fpga_bit_builder_dispatcher_class_name = recipe_config_dict.get('fpga-platform', "F1BitBuilder")
# create run platform dispatcher object using class given and pass args to it
self.fpga_bit_builder_dispatcher = getattr(
import_module("buildtools.bitbuilder"),
fpga_bit_builder_dispatcher_class_name)(self)
# retrieve the bitbuilder section
bitbuilder_conf_dict = None
with open(recipe_config_dict['bit_builder_recipe'], "r") as yaml_file:
bitbuilder_conf_dict = yaml.safe_load(yaml_file)
bitbuilder_type_name = bitbuilder_conf_dict["bit_builder_type"]
bitbuilder_args = bitbuilder_conf_dict["args"]
# add the overrides if it exists
override_args = recipe_config_dict.get('bit_builder_arg_overrides')
if override_args:
bitbuilder_args = deep_merge(bitbuilder_args, override_args)
bitbuilder_dispatch_dict = dict([(x.__name__, x) for x in buildtools.buildconfigfile.inheritors(BitBuilder)])
if not bitbuilder_type_name in bitbuilder_dispatch_dict:
raise Exception(f"Unable to find {bitbuilder_type_name} in available bitbuilder classes: {bitbuilder_dispatch_dict.keys()}")
# create dispatcher object using class given and pass args to it
self.bitbuilder = bitbuilder_dispatch_dict[bitbuilder_type_name](self, bitbuilder_args)
def get_chisel_triplet(self) -> str:
"""Get the unique build-specific '-' deliminated triplet.

12
deploy/buildtools/buildconfigfile.py
View File

@ -8,7 +8,6 @@ import yaml
from runtools.runtime_config import RuntimeHWDB
from buildtools.buildconfig import BuildConfig
from awstools.awstools import auto_create_bucket, get_snsname_arn
from buildtools.buildfarm import BuildFarm
from util.inheritors import inheritors
from util.deepmerge import deep_merge
@ -100,16 +99,15 @@ class BuildConfigFile:
build_farm_dispatch_dict = dict([(x.__name__, x) for x in inheritors(BuildFarm)])
if not build_farm_type_name in build_farm_dispatch_dict:
raise Exception(f"Unable to find {build_farm_type_name} in available build farm classes: {build_farm_dispatch_dict.keys()}")
# create dispatcher object using class given and pass args to it
self.build_farm = build_farm_dispatch_dict[build_farm_type_name](build_farm_args)
def setup(self) -> None:
"""Setup based on the types of build hosts."""
# do bitbuilder setup after all parsing is complete
for build in self.builds_list:
auto_create_bucket(build.s3_bucketname)
# check to see email notifications can be subscribed
get_snsname_arn()
build.bitbuilder.setup()
def request_build_hosts(self) -> None:
"""Launch an instance for the builds. Exits the program if an IP address is reused."""

19
deploy/firesim
View File

@ -137,11 +137,6 @@ def managerinit(args: argparse.Namespace):
rootLogger.debug(m)
rootLogger.debug(m.stderr)
if args.platform == 'f1':
m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.yaml""".format(get_aws_userid(), conf_file), capture=True)
rootLogger.debug(m)
rootLogger.debug(m.stderr)
rootLogger.info("Adding default overrides to config files")
if args.platform == 'f1':
def get_indexes(lines: List[str], keyword_start: str, keyword_end: str)-> Tuple[int, int]:
@ -227,8 +222,8 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None:
# forced to build locally
for build_config in build_config_file.builds_list:
execute(build_config.fpga_bit_builder_dispatcher.replace_rtl, hosts=['localhost'])
execute(build_config.fpga_bit_builder_dispatcher.build_driver, hosts=['localhost'])
execute(build_config.bitbuilder.replace_rtl, hosts=['localhost'])
execute(build_config.bitbuilder.build_driver, hosts=['localhost'])
def release_build_hosts_handler(sig, frame) -> None:
""" Handler that prompts to release build farm hosts if you press ctrl-c. """
@ -243,10 +238,6 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None:
signal.signal(signal.SIGINT, release_build_hosts_handler)
# pre-setup stuff (things that need to be done before build farm hosts are spawned/killed)
# specific to the build farm host
build_config_file.setup()
# local items (replace_rtl) need to be called in a loop, for each config
# remote items will map themselves
build_config_file.request_build_hosts()
@ -264,7 +255,7 @@ def buildbitstream(build_config_file: BuildConfigFile) -> None:
bypass: If true, immediately return and terminate build host. Used for testing purposes.
"""
build_config_file = config_file.get_build_by_ip(env.host_string)
build_config_file.fpga_bit_builder_dispatcher.build_bitstream(bypass)
build_config_file.bitbuilder.build_bitstream(bypass)
# run builds, then terminate instances
execute(parallel_build_helper, build_config_file, hosts=build_config_file.build_ip_set)
@ -285,8 +276,8 @@ def tar2afi(build_config_file: BuildConfigFile) -> None:
"""
for build_config in build_config_file.builds_list:
assert isinstance(build_config.fpga_bit_builder_dispatcher, F1BitBuilder)
execute(build_config.fpga_bit_builder_dispatcher.aws_create_afi, build_config_file, build_config, hosts=['localhost'])
assert isinstance(build_config.bitbuilder, F1BitBuilder)
execute(build_config.bitbuilder.aws_create_afi, build_config_file, build_config, hosts=['localhost'])
@register_task

3
deploy/runtools/runtime_config.py
View File

@ -445,6 +445,9 @@ class InnerRuntimeConfiguration:
run_farm_dispatch_dict = dict([(x.__name__, x) for x in inheritors(RunFarm)])
if not run_farm_type in run_farm_dispatch_dict:
raise Exception(f"Unable to find {run_farm_type} in available run farm classes: {run_farm_dispatch_dict.keys()}")
# create dispatcher object using class given and pass args to it
self.run_farm_dispatcher = run_farm_dispatch_dict[run_farm_type](run_farm_args, self.metasimulation_enabled)

27
deploy/sample-backup-configs/sample_config_build_recipes.yaml
View File

@ -14,9 +14,13 @@
# PLATFORM_CONFIG: Config
# deploy_triplet: null
# post_build_hook: null
# # REQUIRED: TODO: Will move to "run platform" work
# s3_bucket_name: firesim-AWSUSERNAME
# metasim_customruntimeconfig: "path to custom runtime config for metasims"
# bit_builder_recipe:
# # OPTIONAL: overrides for bit builder recipe
# # Arg structure should be identical to the args given
# # in the base_recipe.
# #bit_builder_arg_overrides:
# # <ARG>: <OVERRIDE>
# Quad-core, Rocket-based recipes
firesim_rocket_quadcore_nic_l2_llc4mb_ddr3:
@ -25,8 +29,8 @@ firesim_rocket_quadcore_nic_l2_llc4mb_ddr3:
PLATFORM_CONFIG: WithAutoILA_F90MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# NB: This has a faster host-clock frequency than the NIC-based design, because
# its uncore runs at half rate relative to the tile.
@ -36,8 +40,8 @@ firesim_rocket_quadcore_no_nic_l2_llc4mb_ddr3:
PLATFORM_CONFIG: WithAutoILA_F140MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# Single-core, BOOM-based recipes
firesim_boom_singlecore_nic_l2_llc4mb_ddr3:
@ -46,8 +50,8 @@ firesim_boom_singlecore_nic_l2_llc4mb_ddr3:
PLATFORM_CONFIG: WithAutoILA_F65MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# NB: This has a faster host-clock frequency than the NIC-based design, because
# its uncore runs at half rate relative to the tile.
@ -57,8 +61,8 @@ firesim_boom_singlecore_no_nic_l2_llc4mb_ddr3:
PLATFORM_CONFIG: WithAutoILA_F65MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# Single-core, CVA6-based recipes
firesim_cva6_singlecore_no_nic_l2_llc4mb_ddr3:
@ -67,8 +71,8 @@ firesim_cva6_singlecore_no_nic_l2_llc4mb_ddr3:
PLATFORM_CONFIG: WithAutoILA_F90MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# Single-core, Rocket-based recipes with Gemmini
firesim_rocket_singlecore_gemmini_no_nic_l2_llc4mb_ddr3:
@ -77,8 +81,8 @@ firesim_rocket_singlecore_gemmini_no_nic_l2_llc4mb_ddr3:
PLATFORM_CONFIG: WithAutoILA_F110MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# RAM Optimizations enabled by adding _MCRams PLATFORM_CONFIG string
firesim_boom_singlecore_no_nic_l2_llc4mb_ddr3_ramopts:
@ -87,8 +91,8 @@ firesim_boom_singlecore_no_nic_l2_llc4mb_ddr3_ramopts:
PLATFORM_CONFIG: WithAutoILA_MCRams_F90MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# Supernode configurations -- multiple instances of an SoC in a single simulator
firesim_supernode_rocket_singlecore_nic_l2_lbp:
@ -97,8 +101,8 @@ firesim_supernode_rocket_singlecore_nic_l2_lbp:
PLATFORM_CONFIG: WithAutoILA_F85MHz_BaseF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml
# MIDAS Examples -- BUILD SUPPORT ONLY; Can't launch driver correctly on run farm
midasexamples_gcd:
@ -108,6 +112,5 @@ midasexamples_gcd:
PLATFORM_CONFIG: DefaultF1Config
deploy_triplet: null
post_build_hook: null
s3_bucket_name: firesim-AWSUSERNAME
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/f1.yaml

2
deploy/tests/test_yaml_api.py
View File

@ -232,7 +232,7 @@ class TestConfigBuildAPI:
deploy_triplet: null
PLATFORM_CONFIG: Config
post_build_hook: null
s3_bucket_name: TESTING_BUCKET_NAME
bit_builder_recipe: bit-builder-recipes/f1.yaml
"""))
build_yamls.write()
args = firesim_parse_args(['buildbitstream'] + build_yamls.args)