Merge remote-tracking branch 'origin/main' into metasim-final-1.14

This commit is contained in:
Sagar Karandikar 2022-06-12 03:19:03 +00:00
commit 1dcc7061c1
10 changed files with 154 additions and 137 deletions

View File

@ -0,0 +1,31 @@
# Build-time build farm design configuration for the FireSim Simulation Manager
# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params.
# all fields are required but can be overridden in the `*_runtime.yaml`
###########
# Schema:
###########
# # Class name of the build farm type.
# # This can be determined from `deploy/buildtools/buildfarm.py`).
# build_farm_type: <TYPE NAME>
# args:
# # Build farm arguments that are passed to the `BuildFarmHostDispatcher`
# # object. Determined by looking at `parse_args` function of class.
# <K/V pairs of args>
# Note: For large designs (ones that would fill a EC2 F1.2xlarge/Xilinx VU9P)
# Vivado uses in excess of 32 GiB. Keep this in mind when selecting a
# non-default instance type.
build_farm_type: AWSEC2
args:
# managerinit arg start
# instance type to use per build
instance_type: z1d.2xlarge
# instance market to use per build (ondemand, spot)
build_instance_market: ondemand
# if using spot instances, determine the interrupt behavior (terminate, stop, hibernate)
spot_interruption_behavior: terminate
# if using spot instances, determine the max price
spot_max_price: ondemand
# default location of build directory on build host
default_build_dir: /home/centos/firesim-build

View File

@ -0,0 +1,34 @@
# Build-time build farm design configuration for the FireSim Simulation Manager
# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params.
###########
# Schema:
###########
# # Class name of the build farm type.
# # This can be determined from `deploy/buildtools/buildfarm.py`).
# build_farm_type: <TYPE NAME>
# args:
# # Build farm arguments that are passed to the `BuildFarmHostDispatcher`
# # object. Determined by looking at `parse_args` function of class.
# <K/V pairs of args>
# Unmanaged list of build hosts. Assumed that they are pre-setup to run builds.
build_farm_type: ExternallyProvisioned
args:
# REQUIRED: (replace this) default location of build directory on build host.
default_build_dir: null
# REQUIRED: List of IP addresses (or "localhost"). Each can have an OPTIONAL
# argument, called "override_build_dir", specifying to override the default
# build directory.
#
# Ex:
# build_farm_hosts:
# # use localhost and don't override the default build dir
# - localhost
# # use other IP address (don't override default build dir)
# - "111.111.1.111"
# # use other IP address (override default build dir for this build host)
# - "222.222.2.222":
# override_build_dir: /scratch/specific-build-host-build-dir
build_farm_hosts:
- localhost

View File

@ -11,6 +11,7 @@ from buildtools.buildconfig import BuildConfig
from awstools.awstools import auto_create_bucket, get_snsname_arn
from buildtools.buildfarm import BuildFarm
from util.inheritors import inheritors
from util.deepmerge import deep_merge
# imports needed for python type checking
from typing import Dict, Optional, List, Set, Type, Any, TYPE_CHECKING
@ -83,15 +84,19 @@ class BuildConfigFile:
self.build_ip_set = set()
# retrieve the build host section
build_farm_defaults_file = global_build_config_file["build_farm"]["base_recipe"]
build_farm_config_file = None
with open(args.buildfarmconfigfile, "r") as yaml_file:
with open(build_farm_defaults_file, "r") as yaml_file:
build_farm_config_file = yaml.safe_load(yaml_file)
build_farm_name = global_build_config_file["build_farm"]
build_farm_conf_dict = build_farm_config_file[build_farm_name]
build_farm_type_name = build_farm_config_file["build_farm_type"]
build_farm_args = build_farm_config_file["args"]
build_farm_type_name = build_farm_conf_dict["build_farm_type"]
build_farm_args = build_farm_conf_dict["args"]
# add the overrides if it exists
override_args = global_build_config_file['build_farm'].get('recipe_arg_overrides')
if override_args:
build_farm_args = deep_merge(build_farm_args, override_args)
build_farm_dispatch_dict = dict([(x.__name__, x) for x in inheritors(BuildFarm)])

View File

@ -113,7 +113,7 @@ def managerinit(args: argparse.Namespace):
sys.exit(1)
rootLogger.info("Backing up initial config files, if they exist.")
config_files = ["build", "build_recipes", "build_farm", "hwdb", "runtime"]
config_files = ["build", "build_recipes", "hwdb", "runtime"]
for conf_file in config_files:
with warn_only(), hide('everything'):
m = local("""cp config_{}.yaml sample-backup-configs/backup_config_{}.yaml""".format(conf_file, conf_file), capture=True)
@ -132,15 +132,23 @@ def managerinit(args: argparse.Namespace):
rootLogger.debug(m)
rootLogger.debug(m.stderr)
rootLogger.info("Adding default overrides to default runtime.yaml file")
rootLogger.info("Adding default overrides to config files")
if args.platform == 'f1':
keyword = "managerinit arg start"
runfarm_default_file = "run-farm-recipes/aws_ec2.yaml"
with open(runfarm_default_file, "r") as f:
rf_recipe_lines = f.readlines()
start_lines = [f"base_recipe: {runfarm_default_file}\n"]
start_lines += ["recipe_arg_overrides:\n"]
rf_recipe_lines = [" " + l for l in start_lines] + rf_recipe_lines[5:]
arg_idx = None
for i, l in enumerate(rf_recipe_lines):
if keyword in l:
arg_idx = i + 1
assert arg_idx is not None
rf_recipe_lines = [" " + l for l in start_lines] + rf_recipe_lines[arg_idx:]
file_line_swap(
"config_runtime.yaml",
@ -148,8 +156,29 @@ def managerinit(args: argparse.Namespace):
"managerinit replace start",
"managerinit replace end",
rf_recipe_lines)
buildfarm_default_file = "build-farm-recipes/aws_ec2.yaml"
with open(buildfarm_default_file, "r") as f:
bf_recipe_lines = f.readlines()
start_lines = [f"base_recipe: {buildfarm_default_file}\n"]
start_lines += ["recipe_arg_overrides:\n"]
arg_idx = None
for i, l in enumerate(bf_recipe_lines):
if keyword in l:
arg_idx = i + 1
assert arg_idx is not None
bf_recipe_lines = [" " + l for l in start_lines] + bf_recipe_lines[arg_idx:]
file_line_swap(
"config_build.yaml",
"config_build.yaml",
"managerinit replace start",
"managerinit replace end",
bf_recipe_lines)
else:
rootLogger.info(f"Unknown platform {args.platform} for runtime.yaml setup. Skipping default overrides.")
rootLogger.info(f"Unknown platform {args.platform}. Skipping default config overrides.")
if args.platform == 'f1':
awsinit()
@ -310,9 +339,6 @@ def construct_firesim_argparser() -> argparse.ArgumentParser:
parser.add_argument('-r', '--buildrecipesconfigfile', type=str,
help='Optional custom build recipe config file. Defaults to config_build_recipes.yaml.',
default='config_build_recipes.yaml')
parser.add_argument('-s', '--buildfarmconfigfile', type=str,
help='Optional custom build farm config file. Defaults to config_build_farm.yaml.',
default='config_build_farm.yaml')
parser.add_argument('-a', '--hwdbconfigfile', type=str,
help='Optional custom HW database config file. Defaults to config_hwdb.yaml.',
default='config_hwdb.yaml')

View File

@ -3,15 +3,16 @@
run_farm_type: AWSEC2F1
args:
# managerinit arg start
# tag to apply to run farm hosts
run_farm_tag: mainrunfarm
# TODO
# enable expanding run farm by run_farm_hosts given
always_expand_run_farm: true
# TODO
# minutes to retry attempting to request instances
launch_instances_timeout_minutes: 60
# run farm host market to use (ondemand or spot)
# run farm host market to use (ondemand, spot)
run_instance_market: ondemand
# if using spot instances, determine the interrupt behavior
# if using spot instances, determine the interrupt behavior (terminate, stop, hibernate)
spot_interruption_behavior: terminate
# if using spot instances, determine the max price
spot_max_price: ondemand

View File

@ -11,7 +11,6 @@ import yaml
import os
import sys
from fabric.api import prefix, settings, local # type: ignore
from copy import deepcopy
from awstools.awstools import aws_resource_names
from awstools.afitools import get_firesim_tagval_for_agfi
@ -21,6 +20,7 @@ from runtools.run_farm import RunFarm
from runtools.simulation_data_classes import TracerVConfig, AutoCounterConfig, HostDebugConfig, SynthPrintConfig
from util.streamlogger import StreamLogger
from util.inheritors import inheritors
from util.deepmerge import deep_merge
from typing import Optional, Dict, Any, List, Sequence, Tuple, TYPE_CHECKING
import argparse # this is not within a if TYPE_CHECKING: scope so the `register_task` in FireSim can evaluate it's annotation
@ -419,17 +419,6 @@ class InnerRuntimeConfiguration:
# add the overrides if it exists
# taken from https://gist.github.com/angstwad/bf22d1822c38a92ec0a9
def deep_merge(a: dict, b: dict) -> dict:
result = deepcopy(a)
for bk, bv in b.items():
av = result.get(bk)
if isinstance(av, dict) and isinstance(bv, dict):
result[bk] = deep_merge(av, bv)
else:
result[bk] = deepcopy(bv)
return result
override_args = runtime_dict['run_farm'].get('recipe_arg_overrides')
if override_args:
run_farm_args = deep_merge(run_farm_args, override_args)
@ -448,7 +437,7 @@ class InnerRuntimeConfiguration:
self.tracerv_config = TracerVConfig()
if 'tracing' in runtime_dict:
self.tracerv_config.enable = runtime_dict['tracing'].get('enable') == "yes"
self.tracerv_config.enable = runtime_dict['tracing'].get('enable', False) == True
self.tracerv_config.select = runtime_dict['tracing'].get('selector', "0")
self.tracerv_config.start = runtime_dict['tracing'].get('start', "0")
self.tracerv_config.end = runtime_dict['tracing'].get('end', "-1")
@ -459,13 +448,13 @@ class InnerRuntimeConfiguration:
self.defaulthwconfig = runtime_dict['target_config']['default_hw_config']
self.hostdebug_config = HostDebugConfig()
if 'host_debug' in runtime_dict:
self.hostdebug_config.zero_out_dram = runtime_dict['host_debug'].get('zero_out_dram') == "yes"
self.hostdebug_config.disable_synth_asserts = runtime_dict['host_debug'].get('disable_synth_asserts') == "yes"
self.hostdebug_config.zero_out_dram = runtime_dict['host_debug'].get('zero_out_dram', False) == True
self.hostdebug_config.disable_synth_asserts = runtime_dict['host_debug'].get('disable_synth_asserts', False) == True
self.synthprint_config = SynthPrintConfig()
if 'synth_print' in runtime_dict:
self.synthprint_config.start = runtime_dict['synth_print'].get("start", "0")
self.synthprint_config.end = runtime_dict['synth_print'].get("end", "-1")
self.synthprint_config.cycle_prefix = runtime_dict['synth_print'].get("cycle_prefix", "yes") == "yes"
self.synthprint_config.cycle_prefix = runtime_dict['synth_print'].get("cycle_prefix", True) == True
self.default_plusarg_passthrough = ""
if 'plusarg_passthrough' in runtime_dict['target_config']:
self.default_plusarg_passthrough = runtime_dict['target_config']['plusarg_passthrough']
@ -473,7 +462,7 @@ class InnerRuntimeConfiguration:
self.workload_name = runtime_dict['workload']['workload_name']
# an extra tag to differentiate workloads with the same name in results names
self.suffixtag = runtime_dict['workload']['suffix_tag'] if 'suffix_tag' in runtime_dict['workload'] else None
self.terminateoncompletion = runtime_dict['workload']['terminate_on_completion'] == "yes"
self.terminateoncompletion = runtime_dict['workload']['terminate_on_completion'] == True
def __str__(self) -> str:
return pprint.pformat(vars(self))

View File

@ -2,7 +2,15 @@
# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params.
# this refers to build farms defined in config_build_farm.yaml
build_farm: ec2_build_farm
build_farm:
# managerinit replace start
base_recipe: build-farm-recipes/aws_ec2.yaml
# Uncomment and add args to override defaults.
# Arg structure should be identical to the args given
# in the base_recipe.
#recipe_arg_overrides:
# <ARG>: <OVERRIDE>
# managerinit replace end
builds_to_run:
# this section references builds defined in config_build_recipes.yaml

View File

@ -1,54 +0,0 @@
# Build-time build farm design configuration for the FireSim Simulation Manager
# See https://docs.fires.im/en/stable/Advanced-Usage/Manager/Manager-Configuration-Files.html for documentation of all of these params.
###########
# Schema:
###########
# # Unique name for build farm.
# <NAME>:
# # Class name of the build farm type.
# # This can be determined from `deploy/buildtools/buildfarm.py`).
# build_farm_type: <TYPE NAME>
# args:
# # Build farm arguments that are passed to the `BuildFarmHostDispatcher`
# # object. Determined by looking at `parse_args` function of class.
# <K/V pairs of args>
# Note: For large designs (ones that would fill a EC2.2xlarge/Xilinx VU9P)
# Vivado uses in excess of 32 GiB. Keep this in mind when selecting a
# non-default instance type.
ec2_build_farm:
build_farm_type: AWSEC2
args:
# REQUIRED: instance type to use per build
instance_type: z1d.2xlarge
# REQUIRED: instance market to use per build
build_instance_market: ondemand
# REQUIRED: if using spot instances, determine the interrupt behavior
spot_interruption_behavior: terminate
# REQUIRED: if using spot instances, determine the max price
spot_max_price: ondemand
# REQUIRED: default location of build directory on build host
default_build_dir: /home/centos/firesim-build
# Unmanaged list of build hosts. Assumed that they are pre-setup to run builds.
local_build_farm:
build_farm_type: ExternallyProvisioned
args:
# REQUIRED: default location of build directory on build host
default_build_dir: /home/centos/firesim-build
# REQUIRED: List of IP addresses (or "localhost"). Each can have an OPTIONAL
# argument, called "override_build_dir", specifying to override the default
# build directory.
#
# Ex:
# build_farm_hosts:
# # use localhost and don't override the default build dir
# - localhost
# # use other IP address (don't override default build dir)
# - "111.111.1.111"
# # use other IP address (override default build dir for this build host)
# - "222.222.2.222":
# override_build_dir: /scratch/specific-build-host-build-dir
build_farm_hosts:
- localhost

View File

@ -99,20 +99,17 @@ class BuildTmpYamlSet(TmpYamlSet):
"""
build: TmpYaml
recipes: TmpYaml
farm: TmpYaml
hwdb: TmpYaml
def write(self):
self.build.write()
self.recipes.write()
self.farm.write()
self.hwdb.write()
@property
def args(self):
return ['-b', fspath(self.build.path),
'-r', fspath(self.recipes.path),
'-s', fspath(self.farm.path),
'-a', fspath(self.hwdb.path),
]
@ -157,12 +154,8 @@ def scy_build_recipes(tmp_path: Path, sample_backup_configs: Path) -> TmpYaml:
return TmpYaml(tmp_path, sample_backup_configs / 'sample_config_build_recipes.yaml')
@pytest.fixture()
def scy_build_farm(tmp_path: Path, sample_backup_configs: Path) -> TmpYaml:
return TmpYaml(tmp_path, sample_backup_configs / 'sample_config_build_farm.yaml')
@pytest.fixture()
def build_yamls(scy_build, scy_build_recipes, scy_build_farm, scy_hwdb) -> BuildTmpYamlSet:
return BuildTmpYamlSet(scy_build, scy_build_recipes, scy_build_farm, scy_hwdb)
def build_yamls(scy_build, scy_build_recipes, scy_hwdb) -> BuildTmpYamlSet:
return BuildTmpYamlSet(scy_build, scy_build_recipes, scy_hwdb)
@pytest.fixture()
def scy_hwdb(tmp_path: Path, sample_backup_configs: Path) -> TmpYaml:
@ -198,7 +191,8 @@ class TestConfigBuildAPI:
# at the beginning of the test build_yamls contains the backup-sample-configs
# but we can show exactly what we're doing different from the default by
build_yamls.build.load(dedent("""
build_farm: ec2_build_farm
build_farm:
base_recipe: build-farm-recipes/aws_ec2.yaml
builds_to_run:
@ -215,11 +209,12 @@ class TestConfigBuildAPI:
m['task'].assert_not_called()
m['config'].assert_called_once_with(args)
def test_invalid_buildfarm_type(self, task_mocker, build_yamls, firesim_parse_args):
def test_invalid_buildfarm_recipe(self, task_mocker, build_yamls, firesim_parse_args):
m = task_mocker.patch('buildbitstream', wrap_config=True)
build_yamls.build.load(dedent("""
build_farm: testing_build_farm
build_farm:
base_recipe: INVALID_RECIPE
builds_to_run:
- testing_recipe_name
@ -230,12 +225,6 @@ class TestConfigBuildAPI:
share_with_accounts:
INVALID_NAME: 123456789012
"""))
build_yamls.farm.load(dedent("""
testing_build_farm:
build_farm_type: INVALID_BUILD_FARM_TYPE
args:
DUMMY_ARG: null
"""))
build_yamls.recipes.load(dedent("""
testing_recipe_name:
DESIGN: TopModule
@ -247,45 +236,15 @@ class TestConfigBuildAPI:
"""))
build_yamls.write()
args = firesim_parse_args(['buildbitstream'] + build_yamls.args)
firesim.main.when.called_with(args).should.throw(KeyError, re.compile(r'INVALID_BUILD_FARM_TYPE'))
firesim.main.when.called_with(args).should.throw(FileNotFoundError, re.compile(r'INVALID_RECIPE'))
# the exception should happen while building the config, before the task is actually called
m['config'].assert_called_once_with(args)
m['task'].assert_not_called()
@pytest.mark.parametrize('farm_name',
['ec2_build_farm',
'local_build_farm',
])
def test_invalid_farm_missing_args(self, task_mocker, build_yamls, firesim_parse_args, farm_name):
m = task_mocker.patch('buildbitstream', wrap_config=True)
build_yamls.build.data.should.contain('build_farm')
build_yamls.build.data['build_farm'] = farm_name
build_yamls.farm.data[farm_name].should.contain('args')
build_yamls.farm.data[farm_name]['args'] = None
build_yamls.write()
args = firesim_parse_args(['buildbitstream'] + build_yamls.args)
firesim.main.when.called_with(args).should.throw(TypeError, re.compile(r'object is not subscriptable'))
m['task'].assert_not_called()
def test_invalid_unmanaged_missing_args(self, task_mocker, build_yamls, firesim_parse_args):
m = task_mocker.patch('buildbitstream', wrap_config=True)
build_yamls.build.data['build_farm'] = 'local_build_farm'
build_yamls.farm.data['local_build_farm']['args']['build_farm_hosts'] = None
build_yamls.write()
args = firesim_parse_args(['buildbitstream'] + build_yamls.args)
firesim.main.when.called_with(args).should.throw(TypeError)
m['task'].assert_not_called()
@pytest.mark.parametrize('task_name', [tn for tn in firesim.TASKS if
firesim.TASKS[tn]['config'] is BuildConfigFile])
@pytest.mark.parametrize('opt', ['-b',
'-r',
'-s',
])
def test_config_existence(self, task_mocker, build_yamls, firesim_parse_args, task_name, opt, non_existent_file):
# TODO: Remove after deprecation

18
deploy/util/deepmerge.py Normal file
View File

@ -0,0 +1,18 @@
from __future__ import annotations
from copy import deepcopy
# imports needed for python type checking
from typing import List
# from https://gist.github.com/angstwad/bf22d1822c38a92ec0a9
def deep_merge(a: dict, b: dict) -> dict:
"""Merge two dicts and return a singular dict"""
result = deepcopy(a)
for bk, bv in b.items():
av = result.get(bk)
if isinstance(av, dict) and isinstance(bv, dict):
result[bk] = deep_merge(av, bv)
else:
result[bk] = deepcopy(bv)
return result