diff --git a/deploy/.gitignore b/deploy/.gitignore index 6892aa08..a0b13f50 100644 --- a/deploy/.gitignore +++ b/deploy/.gitignore @@ -3,16 +3,18 @@ switchbuild/switch*.*.*.* switchbuild/switch results-build/* results-workload/* -config_runtime.ini -config_build.ini -config_build_recipes.ini -config_build_hosts.ini -config_hwdb.ini -backup_config_build.ini -backup_config_runtime.ini -backup_config_build_recipes.ini -backup_config_build_hosts.ini -backup_config_hwdb.ini +config_runtime.yaml +config_runfarm.yaml +config_build.yaml +config_build_recipes.yaml +config_build_hosts.yaml +config_hwdb.yaml +backup_config_build.yaml +backup_config_runtime.yaml +backup_config_runfarm.yaml +backup_config_build_recipes.yaml +backup_config_build_hosts.yaml +backup_config_hwdb.yaml generated-topology-diagrams/ logs/*.log built-hwdb-entries/ diff --git a/deploy/awstools/awstools.py b/deploy/awstools/awstools.py index f4c69167..0e7e3656 100755 --- a/deploy/awstools/awstools.py +++ b/deploy/awstools/awstools.py @@ -248,7 +248,7 @@ def launch_run_instances(instancetype, count, fsimclustertag, instancemarket, sp { 'DeviceName': '/dev/sda1', 'Ebs': { - 'VolumeSize': 300, # TODO: make this configurable from .ini? + 'VolumeSize': 300, # TODO: make this configurable from .yaml? 'VolumeType': 'gp2', }, }, diff --git a/deploy/buildtools/buildafi.py b/deploy/buildtools/buildafi.py index 4e296b90..cbbe7dc9 100644 --- a/deploy/buildtools/buildafi.py +++ b/deploy/buildtools/buildafi.py @@ -280,10 +280,10 @@ def aws_create_afi(build_config): message_title = "FireSim FPGA Build Completed" agfi_entry = "[" + afiname + "]\n" - agfi_entry += "afgi=" + agfi + "\n" - agfi_entry += "deploytripletoverride=None\n" - agfi_entry += "customruntimeconfig=None\n" - message_body = "Your AGFI has been created!\nAdd\n\n" + agfi_entry + "\nto your config_hwdb.ini to use this hardware configuration." + agfi_entry += " afgi: " + agfi + "\n" + agfi_entry += " deploy-triplet-override: null\n" + agfi_entry += " custom-runtime-config: null\n" + message_body = "Your AGFI has been created!\nAdd\n\n" + agfi_entry + "\nto your config_hwdb.yaml to use this hardware configuration." send_firesim_notification(message_title, message_body) @@ -292,7 +292,7 @@ def aws_create_afi(build_config): # for convenience when generating a bunch of images. you can just # cat all the files in this directory after your builds finish to get - # all the entries to copy into config_hwdb.ini + # all the entries to copy into config_hwdb.yaml hwdb_entry_file_location = """{}/built-hwdb-entries/""".format(local_deploy_dir) local("mkdir -p " + hwdb_entry_file_location) with open(hwdb_entry_file_location + "/" + afiname, "w") as outputfile: diff --git a/deploy/buildtools/buildconfig.py b/deploy/buildtools/buildconfig.py index 81331cac..1e59d5a6 100644 --- a/deploy/buildtools/buildconfig.py +++ b/deploy/buildtools/buildconfig.py @@ -2,25 +2,33 @@ manager """ from time import strftime, gmtime -import configparser import pprint from importlib import import_module -from runtools.runtime_config import RuntimeHWDB from awstools.awstools import * -from buildtools.buildhostdispatcher import * -from buildtools.build import * +from buildtools.buildfarmhostdispatcher import * + +def inheritors(klass): + subclasses = set() + work = [klass] + while work: + parent = work.pop() + for child in parent.__subclasses__(): + if child not in subclasses: + subclasses.add(child) + work.append(child) + return subclasses class BuildConfig: """ Represents a single build configuration used to build RTL/drivers/AFIs. """ - def __init__(self, name, recipe_config_dict, build_hosts_configfile, global_build_config, launch_time): + def __init__(self, name, recipe_config_dict, build_farm_hosts_configfile, global_build_config, launch_time): """ Initialization function. Parameters: - name (str): Name of config i.e. name of build_recipe.ini section - recipe_config_dict (dict): build_recipe.ini options associated with name - build_hosts_configfile (configparser.ConfigParser): Parsed representation of build_hosts.ini file + name (str): Name of config i.e. name of build_recipe.yaml section + recipe_config_dict (dict): build_recipe.yaml options associated with name + build_farm_hosts_configfile (dict): Parsed representation of build_farm_hosts.yaml file global_build_config (BuildConfigFile): Global build config file launch_time (str): Time manager was launched """ @@ -31,33 +39,36 @@ class BuildConfig: self.TARGET_PROJECT = recipe_config_dict.get('TARGET_PROJECT') self.DESIGN = recipe_config_dict['DESIGN'] self.TARGET_CONFIG = recipe_config_dict['TARGET_CONFIG'] - self.deploytriplet = recipe_config_dict['deploytriplet'] + self.deploytriplet = recipe_config_dict['deploy-triplet'] self.launch_time = launch_time # run platform specific options self.PLATFORM_CONFIG = recipe_config_dict['PLATFORM_CONFIG'] - self.s3_bucketname = recipe_config_dict['s3bucketname'] + self.s3_bucketname = recipe_config_dict['s3-bucket-name'] if valid_aws_configure_creds(): aws_resource_names_dict = aws_resource_names() if aws_resource_names_dict['s3bucketname'] is not None: # in tutorial mode, special s3 bucket name self.s3_bucketname = aws_resource_names_dict['s3bucketname'] - self.post_build_hook = recipe_config_dict['postbuildhook'] + self.post_build_hook = recipe_config_dict['post-build-hook'] # retrieve the build host section - self.build_host = recipe_config_dict.get('buildhost') - if self.build_host == None: - self.build_host = "defaultbuildhost" - build_host_conf_dict = dict(build_hosts_configfile.items(self.build_host)) + self.build_farm_host = recipe_config_dict.get('build-farm', "default-build-farm") + build_farm_host_conf_dict = build_farm_hosts_configfile[self.build_farm_host] + + build_farm_host_type = build_farm_host_conf_dict["build-farm-type"] + build_farm_host_args = build_farm_host_conf_dict["args"] + + build_farm_host_dispatch_dict = dict([(x.NAME, x.__name__) for x in inheritors(BuildHostDispatcher)]) + + build_farm_host_dispatcher_class_name = build_farm_host_dispatch_dict[build_farm_host_type] - self.build_host_dispatcher_class_name = build_host_conf_dict['providerclass'] - del build_host_conf_dict['providerclass'] # create dispatcher object using class given and pass args to it - self.build_host_dispatcher = getattr( + self.build_farm_host_dispatcher = getattr( import_module("buildtools.buildhostdispatcher"), - self.build_host_dispatcher_class_name)(self, build_host_conf_dict) + build_farm_host_dispatcher_class_name)(self, build_farm_host_args) - self.build_host_dispatcher.parse_args() + self.build_farm_host_dispatcher.parse_args() def __repr__(self): """ Print the class. diff --git a/deploy/buildtools/buildconfigfile.py b/deploy/buildtools/buildconfigfile.py index fb0e080c..6cd3f186 100644 --- a/deploy/buildtools/buildconfigfile.py +++ b/deploy/buildtools/buildconfigfile.py @@ -2,19 +2,20 @@ manager """ from time import strftime, gmtime -import ConfigParser import pprint import logging import sys +import yaml +from collections import defaultdict +from importlib import import_module from runtools.runtime_config import RuntimeHWDB -from awstools.awstools import auto_create_bucket, get_snsname_arn from buildtools.buildconfig import BuildConfig rootLogger = logging.getLogger() class BuildConfigFile: - """ Class representing the "global" build config file i.e. sample_config_build.ini. """ + """ Class representing the "global" build config file i.e. sample_config_build.yaml. """ def __init__(self, args): """ Initialize function. @@ -30,72 +31,86 @@ class BuildConfigFile: self.args = args - global_build_configfile = ConfigParser.ConfigParser(allow_no_value=True) - # make option names case sensitive - global_build_configfile.optionxform = str - global_build_configfile.read(args.buildconfigfile) + global_build_configfile = None + with open(args.buildconfigfile, "r") as yaml_file: + global_build_configfile = yaml.safe_load(yaml_file) # aws specific options - self.agfistoshare = [x[0] for x in global_build_configfile.items('agfistoshare')] - self.acctids_to_sharewith = [x[1] for x in global_build_configfile.items('sharewithaccounts')] + self.agfistoshare = global_build_configfile['agfis-to-share'] + self.acctids_to_sharewith = global_build_configfile['share-with-accounts'].values() # this is a list of actual builds to run - builds_to_run_list = map(lambda x: x[0], global_build_configfile.items('builds')) + builds_to_run_list = global_build_configfile['builds'] - build_recipes_configfile = ConfigParser.ConfigParser(allow_no_value=True) - # make option names case sensitive - build_recipes_configfile.optionxform = str - build_recipes_configfile.read(args.buildrecipesconfigfile) + build_recipes_configfile = None + with open(args.buildrecipesconfigfile, "r") as yaml_file: + build_recipes_configfile = yaml.safe_load(yaml_file) - build_hosts_configfile = ConfigParser.ConfigParser(allow_no_value=True) - # make option names case sensitive - build_hosts_configfile.optionxform = str - build_hosts_configfile.read(args.buildhostsconfigfile) + build_farm_hosts_configfile = None + with open(args.buildfarmconfigfile, "r") as yaml_file: + build_farm_hosts_configfile = yaml.safe_load(yaml_file) build_recipes = dict() - for section in build_recipes_configfile.sections(): - build_recipes[section] = BuildConfig( - section, - dict(build_recipes_configfile.items(section)), - build_hosts_configfile, - self, - launch_time) + for section_name, section_dict in build_recipes_configfile.items(): + if section_name in builds_to_run_list: + build_recipes[section_name] = BuildConfig( + section_name, + section_dict, + build_farm_hosts_configfile, + self, + launch_time) self.hwdb = RuntimeHWDB(args.hwdbconfigfile) - self.builds_list = list(map(lambda x: build_recipes[x], builds_to_run_list)) + self.builds_list = build_recipes.values() self.build_ip_set = set() def setup(self): - """ Setup based on the types of buildhosts """ + """ Setup based on the types of buildfarmhosts """ for build in self.builds_list: auto_create_bucket(build.s3_bucketname) - #check to see email notifications can be subscribed + # check to see email notifications can be subscribed get_snsname_arn() - def request_build_hosts(self): + def request_build_farm_hosts(self): """ Launch an instance for the builds. Exits the program if an IP address is reused. """ - # TODO: optimization: batch together items using the same buildhost - for build in self.builds_list: - build.build_host_dispatcher.request_build_host() - num_ips = len(self.build_ip_set) - ip = build.build_host_dispatcher.get_build_host_ip() - self.build_ip_set.add(ip) - if num_ips == len(self.build_ip_set): - rootLogger.critical("ERROR: Duplicate {} IP used when launching instance".format(ip)) - self.release_build_hosts() - sys.exit(1) - def wait_on_build_host_initializations(self): + def categorize(seq): + class_names = list(map(lambda x: x.build_farm_host_dispatcher.__class__.__name__, seq)) + class_builds_zipped = zip(class_names, seq) + + d = defaultdict(list) + for c, b in class_builds_zipped: + d[c].append(b) + + return d + + cat = categorize(self.builds_list) + + for bhd_class, build_farm_hosts in cat.items(): + # batch launching build farm hosts of similar types + getattr(import_module("buildtools.buildfarmhostdispatcher"), + bhd_class).request_build_farm_hosts(list(map(lambda x: x.build_farm_host_dispatcher, build_farm_hosts))) + + for build in build_farm_hosts: + num_ips = len(self.build_ip_set) + ip = build.build_farm_host_dispatcher.get_build_farm_host_ip() + self.build_ip_set.add(ip) + if num_ips == len(self.build_ip_set): + rootLogger.critical("ERROR: Duplicate {} IP used when launching instance".format(ip)) + self.release_build_farm_hosts() + sys.exit(1) + + def wait_on_build_farm_host_initializations(self): """ Block until all build instances are launched """ for build in self.builds_list: - build.build_host_dispatcher.wait_on_build_host_initialization() + build.build_farm_host_dispatcher.wait_on_build_farm_host_initialization() - def release_build_hosts(self): + def release_build_farm_hosts(self): """ Terminate all build instances that are launched """ for build in self.builds_list: - build.build_host_dispatcher.release_build_host() + build.build_farm_host_dispatcher.release_build_farm_host() def get_build_by_ip(self, nodeip): """ For a particular IP (aka instance), return the build config it is running. @@ -107,11 +122,11 @@ class BuildConfigFile: """ for build in self.builds_list: - if build.build_host_dispatcher.get_build_host_ip() == nodeip: + if build.build_farm_host_dispatcher.get_build_farm_host_ip() == nodeip: return build return None - def get_build_host_ips(self): + def get_build_farm_host_ips(self): """ Get all the build instance IPs (later passed to fabric as hosts). Returns: @@ -135,4 +150,3 @@ class BuildConfigFile: (str): String representation of the class """ return pprint.pformat(vars(self)) - diff --git a/deploy/buildtools/buildfarmhostdispatcher.py b/deploy/buildtools/buildfarmhostdispatcher.py new file mode 100644 index 00000000..e709b1c1 --- /dev/null +++ b/deploy/buildtools/buildfarmhostdispatcher.py @@ -0,0 +1,247 @@ +import logging +import sys + +from awstools.awstools import * + +rootLogger = logging.getLogger() + +class BuildFarmHostDispatcher(object): + """ Abstract class to manage how to handle a single build farm host (request, wait, release, etc). """ + + NAME = "" + + def __init__(self, build_config, args): + """ Initialization function. + + Parameters: + build_config (BuildConfig): Build config associated with this dispatcher + args (list/dict): List/Dict of args (i.e. options) passed to the dispatcher + """ + + self.build_config = build_config + self.args = args + + # used if dispatcher refers to local build farm + self.is_local = False + + self.override_remote_build_dir = None + + def parse_args(self): + """ Parse default build farm arguments. Can be overridden by child classes. """ + raise NotImplementedError + + def request_build_farm_host(self): + """ Request build farm host to use for build. """ + raise NotImplementedError + + # technically not inherited, but put here to make things clear + @staticmethod + def request_build_farm_hosts(build_farm_host_dispatchers): + """ Request multiple build farm hosts """ + return + + def wait_on_build_farm_host_initialization(self): + """ Ensure build farm host is launched and ready to be used. """ + raise NotImplementedError + + def get_build_farm_host_ip(self): + """ Get IP address associated with this dispatched build farm host. """ + raise NotImplementedError + + def release_build_farm_host(self): + """ Release the build farm host. """ + raise NotImplementedError + +class IPAddrBuildFarmHostDispatcher(BuildFarmHostDispatcher): + """ Dispatcher class that uses the IP address given as the build farm host. """ + + NAME = "unmanaged" + + dispatch_counter = 0 + + def __init__(self, build_config, args): + """ Initialization function. Sets IP address and determines if it is localhost. + + Parameters: + build_config (BuildConfig): Build config associated with this dispatcher + args (list): List of args (i.e. options) passed to the dispatcher + """ + + BuildFarmHostDispatcher.__init__(self, build_config, args) + + # ip addrs associated with build farm host + self.ip_addr = None + + self.dispatch_id = IPAddrBuildFarmHostDispatcher.dispatch_counter + IPAddrBuildFarmHostDispatcher.dispatch_counter += 1 + + def parse_args(self): + """ Parse build farm host arguments. """ + build_farm_hosts_list = self.args["build-farm-hosts"] + if len(build_farm_hosts_list) > self.dispatch_id: + default_build_dir = self.args.get("default-build-dir") + + build_farm_host = build_farm_hosts_list[self.dispatch_id] + + if type(build_farm_host) is dict: + # add element { ip-addr: { arg1: val1, arg2: val2, ... } } + assert(len(build_farm_host.keys()) == 1) + + self.ip_addr = build_farm_host.keys()[0] + ip_args = build_farm_host.values()[0] + + self.override_remote_build_dir = ip_args.get("override-build-dir", default_build_dir) + elif type(build_farm_host) is str: + # add element w/ defaults + + self.ip_addr = build_farm_host + self.override_remote_build_dir = default_build_dir + else: + raise Exception("Unknown build farm host type") + + if self.ip_addr == "localhost": + self.is_local = True + + rootLogger.info("Using host {} for {} with IP address: {}".format(self.build_config.build_farm_host, self.build_config.get_chisel_triplet(), self.ip_addr)) + else: + rootLogger.critical("ERROR: Less IPs available than builds. Add more IPs.") + raise Exception("ERROR: Less IPs available than builds. Add more IPs.") + + def request_build_farm_host(self): + """ In this case, nothing happens since IP address should be pre-setup. """ + return + + @staticmethod + def request_build_farm_hosts(build_farm_host_dispatchers): + return + + def wait_on_build_farm_host_initialization(self): + """ In this case, nothing happens since IP address should be pre-setup. """ + return + + def get_build_farm_host_ip(self): + """ Get IP address associated with this dispatched build farm host. + + Returns: + (str): IP address given as the dispatcher arg + """ + + return self.ip_addr + + def release_build_farm_host(self): + """ In this case, nothing happens. Up to the IP address to cleanup after itself. """ + return + +class EC2BuildFarmHostDispatcher(BuildFarmHostDispatcher): + """ Dispatcher class to manage an AWS EC2 instance as the build farm host. """ + + NAME = "aws-ec2" + + def __init__(self, build_config, args): + """ Initialization function. Setup AWS instance variables. + + Parameters: + build_config (BuildConfig): Build config associated with this dispatcher + args (dict): Dict of args (i.e. options) passed to the dispatcher + """ + + BuildFarmHostDispatcher.__init__(self, build_config, args) + + # instance object associated with the build farm host + self.launched_instance_object = None + + # aws specific options + self.instance_type = None + self.build_instance_market = None + self.spot_interruption_behavior = None + self.spot_max_price = None + + # ec2 hosts will never be local + self.is_local = False + + def parse_args(self): + """ Parse build farm host arguments. """ + # get aws specific args + self.instance_type = self.args['instance-type'] + self.build_instance_market = self.args['build-instance-market'] + self.spot_interruption_behavior = self.args['spot-interruption-behavior'] + self.spot_max_price = self.args['spot-max-price'] + + self.override_remote_build_dir = self.args.get("build-dir") + + def request_build_farm_host(self): + """ Launch an AWS EC2 instance for the build config. """ + + build_instance_market = self.build_instance_market + spot_interruption_behavior = self.spot_interruption_behavior + spot_max_price = self.spot_max_price + + num_instances = 1 + self.launched_instance_object = EC2BuildFarmHostDispatcher.ec2_launch_instances( + self.instance_type, + num_instances, + build_instance_market, + spot_interruption_behavior, + spot_max_price)[0] + + @staticmethod + def request_build_farm_hosts(build_farm_host_dispatchers): + amt_requested = len(build_farm_host_dispatchers) + + launched_objs = EC2BuildFarmHostDispatcher.ec2_launch_instances( + instance_type, + amt_requested, + build_instance_market, + spot_interruption_behavior, + spot_max_price) + + for bh, lo in zip(build_farm_host_dispatchers, launched_objs): + bh.launched_instance_object = lo + + @staticmethod + def ec2_launch_instances(inst_type, num_insts, build_inst_market, spot_int_behav, spot_max_price): + # get access to the runfarmprefix, which we will apply to build + # instances too now. + aws_resource_names_dict = aws_resource_names() + # just duplicate the runfarmprefix for now. This can be None, + # in which case we give an empty build farm prefix + build_farm_prefix = aws_resource_names_dict['runfarmprefix'] + + buildfarmprefix = '' if build_farm_prefix is None else build_farm_prefix + + return launch_instances( + inst_type, + num_insts, + build_inst_market, + spot_int_behav, + spot_max_price, + blockdevices=[ + { + 'DeviceName': '/dev/sda1', + 'Ebs': { + 'VolumeSize': 200, + 'VolumeType': 'gp2', + }, + }, + ], + tags={ 'fsimbuildcluster': buildfarmprefix }, + randomsubnet=True) + + def wait_on_build_farm_host_initialization(self): + """ Wait for EC2 instance launch. """ + wait_on_build_farm_host_initializationes([self.launched_instance_object]) + + def get_build_farm_host_ip(self): + """ Get IP address associated with this dispatched instance. + + Returns: + (str): IP address of EC2 build farm host + """ + + return self.launched_instance_object.private_ip_address + + def release_build_farm_host(self): + """ Terminate the EC2 instance running this build. """ + instance_ids = get_instance_ids_for_instances([self.launched_instance_object]) + rootLogger.info("Terminating build instances {}. Please confirm in your AWS Management Console".format(instance_ids)) + terminate_instances(instance_ids, dryrun=False) diff --git a/deploy/buildtools/buildhostdispatcher.py b/deploy/buildtools/buildhostdispatcher.py deleted file mode 100644 index 7abc7c24..00000000 --- a/deploy/buildtools/buildhostdispatcher.py +++ /dev/null @@ -1,199 +0,0 @@ -import logging -import sys - -from awstools.awstools import * - -rootLogger = logging.getLogger() - -class BuildHostDispatcher: - """ Abstract class to manage how to handle build hosts (request, wait, release, etc). """ - - def __init__(self, build_config, arg_dict): - """ Initialization function. - - Parameters: - build_config (BuildConfig): Build config associated with this dispatcher - arg_dict (dict): Dict of args (i.e. options) passed to the dispatcher - """ - - self.build_config = build_config - self.arg_dict = arg_dict - - # used if dispatcher refers to local build host - self.is_local = False - - # used to override where to do the fpga build (if done remotely) - self.override_remote_build_dir = None - - def parse_args(self): - """ Parse default build host arguments. Can be overridden by child classes. """ - self.override_remote_build_dir = self.get_arg("remotebuilddir") - - def get_arg(self, arg_wanted): - """ Retrieve argument from arg dict and error if not found. - - Parameters: - arg_wanted (str): Argument to get value of - Returns: - (str or None): Value of argument wanted - """ - if not arg_dict.has_key(arg_wanted): - rootLogger.critical("ERROR: Unable to find arg {} for {}".format(arg_wanted, self.__name__)) - sys.exit(1) - return arg_dict.get(arg_wanted) - - def request_build_host(self): - """ Request build host to use for build. """ - raise NotImplementedError - - def wait_on_build_host_initialization(self): - """ Ensure build host is launched and ready to be used. """ - raise NotImplementedError - - def get_build_host_ip(self): - """ Get IP address associated with this dispatched build host. """ - raise NotImplementedError - - def release_build_host(self): - """ Release the build host. """ - raise NotImplementedError - -class IPAddrBuildHostDispatcher(BuildHostDispatcher): - """ Dispatcher class that uses the IP address given as the build host. """ - - def __init__(self, build_config, arg_dict): - """ Initialization function. Sets IP address and determines if it is localhost. - - Parameters: - build_config (BuildConfig): Build config associated with this dispatcher - arg_dict (dict): Dict of args (i.e. options) passed to the dispatcher - """ - - BuildHostDispatcher.__init__(self, build_config, arg_dict) - - # ip addr associated with build host - self.ip_addr = None - - rootLogger.info("Using host {} for {}".format(self.build_config.build_host, self.build_config.get_chisel_triplet())) - - def parse_args(self): - """ Parse build host arguments. """ - # get default arguments - BuildHostDispatcher.parse_args(self) - - # ip address arg - self.ip_addr = self.get_arg("ipaddr") - if self.ip_addr == "localhost": - self.is_local = True - - def request_build_host(self): - """ In this case, nothing happens since IP address should be pre-setup. """ - return - - def wait_on_build_host_initialization(self): - """ In this case, nothing happens since IP address should be pre-setup. """ - return - - def get_build_host_ip(self): - """ Get IP address associated with this dispatched build host. - - Returns: - (str): IP address given as the dispatcher arg - """ - - return self.ip_addr - - def release_build_host(self): - """ In this case, nothing happens. Up to the IP address to cleanup after itself. """ - return - -class EC2BuildHostDispatcher(BuildHostDispatcher): - """ Dispatcher class to manage an AWS EC2 instance as the build host. """ - - def __init__(self, build_config, arg_dict): - """ Initialization function. Setup AWS instance variables. - - Parameters: - build_config (BuildConfig): Build config associated with this dispatcher - arg_dict (dict): Dict of args (i.e. options) passed to the dispatcher - """ - - BuildHostDispatcher.__init__(self, build_config, arg_dict) - - # instance object associated with the build host - self.launched_instance_object = None - - # aws specific options - self.instance_type = None - self.build_instance_market = None - self.spot_interruption_behavior = None - self.spot_max_price = None - - # ec2 hosts will never be local - self.is_local = False - - def parse_args(self): - """ Parse build host arguments. """ - # get default arguments - BuildHostDispatcher.parse_args(self) - - # get aws specific args - self.instance_type = self.get_arg('instancetype') - self.build_instance_market = self.get_arg('buildinstancemarket') - self.spot_interruption_behavior = self.get_arg('spotinterruptionbehavior') - self.spot_max_price = self.get_arg('spotmaxprice') - - def request_build_host(self): - """ Launch an AWS EC2 instance for the build config. """ - - buildconf = self.build_config - - # get access to the runfarmprefix, which we will apply to build - # instances too now. - aws_resource_names_dict = aws_resource_names() - # just duplicate the runfarmprefix for now. This can be None, - # in which case we give an empty build farm prefix - build_farm_prefix = aws_resource_names_dict['runfarmprefix'] - - build_instance_market = self.build_instance_market - spot_interruption_behavior = self.spot_interruption_behavior - spot_max_price = self.spot_max_price - - buildfarmprefix = '' if build_farm_prefix is None else build_farm_prefix - num_instances = 1 - self.launched_instance_object = launch_instances( - self.instance_type, - num_instances, - build_instance_market, - spot_interruption_behavior, - spot_max_price, - blockdevices=[ - { - 'DeviceName': '/dev/sda1', - 'Ebs': { - 'VolumeSize': 200, - 'VolumeType': 'gp2', - }, - }, - ], - tags={ 'fsimbuildcluster': buildfarmprefix }, - randomsubnet=True)[0] - - def wait_on_build_host_initialization(self): - """ Wait for EC2 instance launch. """ - wait_on_build_host_initializationes([self.launched_instance_object]) - - def get_build_host_ip(self): - """ Get IP address associated with this dispatched instance. - - Returns: - (str): IP address of EC2 build host - """ - - return self.launched_instance_object.private_ip_address - - def release_build_host(self): - """ Terminate the EC2 instance running this build. """ - instance_ids = get_instance_ids_for_instances([self.launched_instance_object]) - rootLogger.info("Terminating build instances {}. Please confirm in your AWS Management Console".format(instance_ids)) - terminate_instances(instance_ids, dryrun=False) diff --git a/deploy/firesim b/deploy/firesim index 68113fd5..79b88f5e 100755 --- a/deploy/firesim +++ b/deploy/firesim @@ -9,6 +9,7 @@ import argparse from time import sleep, strftime, gmtime import logging import random +import string import argcomplete from fabric.api import * @@ -30,26 +31,7 @@ from util.streamlogger import StreamLogger ## to add a task, add it here and to the choices array def managerinit(): - """ Setup local FireSim manager components. """ - - def valid_aws_configure_creds(): - """ See if aws configure has been run. Returns False if aws configure - needs to be run, else True. - - This DOES NOT perform any deeper validation. - """ - import botocore.session - session = botocore.session.get_session() - creds = session.get_credentials() - if creds is None: - return False - if session.get_credentials().access_key == '': - return False - if session.get_credentials().secret_key == '': - return False - if session.get_config_variable('region') == '': - return False - return True + """ Setup FireSim manager components. """ valid_creds = valid_aws_configure_creds() while not valid_creds: @@ -66,28 +48,29 @@ def managerinit(): rootLogger.info("Invalid AWS credentials. Try again.") rootLogger.info("Backing up initial config files, if they exist.") - config_files = ["build", "build_recipes", "build_hosts", "hwdb", "runtime"] + config_files = ["build", "build_recipes", "build_farm", "hwdb", "runtime"] for conf_file in config_files: with warn_only(), hide('everything'): - m = local("""cp config_{}.ini sample-backup-configs/backup_config_{}.ini""".format(conf_file, conf_file), capture=True) + m = local("""cp config_{}.yaml sample-backup-configs/backup_config_{}.yaml""".format(conf_file, conf_file), capture=True) rootLogger.debug(m) rootLogger.debug(m.stderr) rootLogger.info("Creating initial config files from examples.") with hide('everything'): for conf_file in config_files: - m = local("""cp sample-backup-configs/sample_config_{}.ini config_{}.ini""".format(conf_file, conf_file), capture=True) + m = local("""cp sample-backup-configs/sample_config_{}.yaml config_{}.yaml""".format(conf_file, conf_file), capture=True) rootLogger.debug(m) rootLogger.debug(m.stderr) - m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.ini""".format(get_aws_userid(), conf_file), capture=True) + m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.yaml""".format("DUMMY", conf_file), capture=True) rootLogger.debug(m) rootLogger.debug(m.stderr) - useremail = input("If you are a new user, supply your email address [abc@xyz.abc] for email notifications (leave blank if you do not want email notifications): ") + useremail = input("If you are a new user, supply your email address [abc@xyz.abc] for AWS email notifications (leave blank if you do not want email notifications): ") if useremail != "": subscribe_to_firesim_topic(useremail) else: rootLogger.info("You did not supply an email address. No notifications will be sent.") + rootLogger.info("FireSim Manager setup completed.") def infrasetup(runtime_conf): @@ -110,36 +93,37 @@ def buildafi(globalbuildconf): """ Starting from local Chisel, build an AFI for all of the specified hardware configs. """ - for buildconf in globalbuildconf.get_builds_list(): - execute(replace_rtl, buildconf, hosts=['localhost']) - execute(build_driver, buildconf, hosts=['localhost']) + # forced to build locally + for build_config in globalbuildconf.get_builds_list(): + execute(replace_rtl, hosts=['localhost']) + execute(build_driver, hosts=['localhost']) - def release_build_hosts_handler(sig, frame): - """ Handler that prompts to release build hosts if you press ctrl-c. """ + def release_build_farm_hosts_handler(sig, frame): + """ Handler that prompts to release build farm hosts if you press ctrl-c. """ rootLogger.info("You pressed ctrl-c, so builds have been killed.") - userconfirm = input("Do you also want to release your build hosts? Type 'yes' to do so.\n") + userconfirm = input("Do you also want to terminate your build instances? Type 'yes' to do so.\n") if userconfirm == "yes": - globalbuildconf.release_build_hosts() - rootLogger.info("Build hosts released.") + globalbuildconf.release_build_farm_hosts() + rootLogger.info("Build farm hosts released.") else: - rootLogger.info("Termination skipped. There may still be build hosts running.") + rootLogger.info("Termination skipped. There may still be build farm hosts running.") sys.exit(0) - signal.signal(signal.SIGINT, release_build_hosts_handler) + signal.signal(signal.SIGINT, release_build_farm_hosts_handler) - # pre-setup stuff (things that need to be done before build hosts are spawned/killed) - # specific to the buildhost + # pre-setup stuff (things that need to be done before build farm hosts are spawned/killed) + # specific to the build farm host globalbuildconf.setup() # local items (replace_rtl) need to be called in a loop, for each config # remote items will map themselves - globalbuildconf.request_build_hosts() + globalbuildconf.request_build_farm_hosts() - # confirm that build hosts have finished booting - globalbuildconf.wait_on_build_host_initializations() + # confirm that build farm hosts have finished booting + globalbuildconf.wait_on_build_farm_host_initializations() - # run builds, then terminate build hosts - execute(aws_build, globalbuildconf, hosts=globalbuildconf.get_build_host_ips()) + # run builds, then terminate instances + execute(aws_build, globalbuildconf, hosts=globalbuildconf.get_build_farm_host_ips()) def tar2afi(globalbuildconf): @@ -177,12 +161,12 @@ def launchrunfarm(runtime_conf): def terminaterunfarm(runtime_conf, terminatesomef1_16, terminatesomef1_4, terminatesomef1_2, terminatesomem4_16, forceterminate): - """ Terminate instances in the runfarm. + """ Terminate instances in the run farm. This works in 2 modes: 1) If you pass no --terminatesomeINSTANCETYPE flags, it will terminate all - instances with the specified runfarm tag. + instances with the specified run farm tag. 2) If you pass ANY --terminatesomeINSTANCETYPE flag, it will terminate only that many instances of the specified types and leave all others @@ -222,20 +206,20 @@ def construct_firesim_argparser(): 'tar2afi' ]) parser.add_argument('-c', '--runtimeconfigfile', type=str, - help='Optional custom runtime/workload config file. Defaults to config_runtime.ini.', - default='config_runtime.ini') + help='Optional custom runtime/workload config file. Defaults to config_runtime.yaml.', + default='config_runtime.yaml') parser.add_argument('-b', '--buildconfigfile', type=str, - help='Optional custom build config file. Defaults to config_build.ini.', - default='config_build.ini') + help='Optional custom build config file. Defaults to config_build.yaml.', + default='config_build.yaml') parser.add_argument('-r', '--buildrecipesconfigfile', type=str, - help='Optional custom build recipe config file. Defaults to config_build_recipes.ini.', - default='config_build_recipes.ini') - parser.add_argument('-s', '--buildhostsconfigfile', type=str, - help='Optional custom build host config file.', - default='config_build_hosts.ini') + help='Optional custom build recipe config file. Defaults to config_build_recipes.yaml.', + default='config_build_recipes.yaml') + parser.add_argument('-s', '--buildfarmconfigfile', type=str, + help='Optional custom build farm config file. Defaults to config_build_farm.yaml.', + default='config_build_farm.yaml') parser.add_argument('-a', '--hwdbconfigfile', type=str, - help='Optional custom HW database config file. Defaults to config_hwdb.ini.', - default='config_hwdb.ini') + help='Optional custom HW database config file. Defaults to config_hwdb.yaml.', + default='config_hwdb.yaml') parser.add_argument('-x', '--overrideconfigdata', type=str, help='Override a single value from one of the the RUNTIME e.g.: --overrideconfigdata "targetconfig linklatency 6405".', default="") diff --git a/deploy/regression.sh b/deploy/regression.sh index b3de7bb8..a40c2ccd 100755 --- a/deploy/regression.sh +++ b/deploy/regression.sh @@ -5,22 +5,22 @@ # NB: The onus is still un the invoker to check the uartlogs. -echo "Diffing config_hwdb.ini against sample_config_hwdb.ini:" -diff config_hwdb.ini sample-backup-configs/sample_config_hwdb.ini +echo "Diffing config_hwdb.yaml against sample_config_hwdb.yaml:" +diff config_hwdb.yaml sample-backup-configs/sample_config_hwdb.yaml local_diff_rc=$? -echo "Diffing sample_config_hwdb.ini against origin/dev:" -git diff --exit-code origin/dev -- sample-backup-configs/sample_config_hwdb.ini +echo "Diffing sample_config_hwdb.yaml against origin/dev:" +git diff --exit-code origin/dev -- sample-backup-configs/sample_config_hwdb.yaml remote_diff_rc=$? if [[ $local_diff_rc = 0 && $remote_diff_rc = 0 ]]; then echo "Local HWDB does not differ from origin/dev." - echo "Did you update config_hwdb.ini with new AGFIs?" + echo "Did you update config_hwdb.yaml with new AGFIs?" exit 1 fi # Run linux-poweroff on unnetworked targets -./workloads/run-workload.sh workloads/linux-poweroff-all-no-nic.ini --withlaunch +./workloads/run-workload.sh workloads/linux-poweroff-all-no-nic.yaml --withlaunch # Run linux-poweroff on the networked target running on a f1.16xlarge -./workloads/run-workload.sh workloads/linux-poweroff-nic.ini --withlaunch +./workloads/run-workload.sh workloads/linux-poweroff-nic.yaml --withlaunch diff --git a/deploy/runtools/runtime_config.py b/deploy/runtools/runtime_config.py index 4badc624..986b8c57 100644 --- a/deploy/runtools/runtime_config.py +++ b/deploy/runtools/runtime_config.py @@ -4,7 +4,6 @@ simulation tasks. """ from __future__ import print_function from time import strftime, gmtime -import configparser import pprint import logging @@ -25,16 +24,16 @@ CUSTOM_RUNTIMECONFS_BASE = "../sim/custom-runtime-configs/" rootLogger = logging.getLogger() class RuntimeHWConfig: - """ A pythonic version of the entires in config_hwdb.ini """ + """ A pythonic version of the entires in config_hwdb.yaml """ def __init__(self, name, hwconfig_dict): self.name = name self.agfi = hwconfig_dict['agfi'] - self.deploytriplet = hwconfig_dict['deploytripletoverride'] + self.deploytriplet = hwconfig_dict['deploy-triplet-override'] self.deploytriplet = self.deploytriplet if self.deploytriplet != "None" else None if self.deploytriplet is not None: - rootLogger.warning("Your config_hwdb.ini file is overriding a deploytriplet. Make sure you understand why!") - self.customruntimeconfig = hwconfig_dict['customruntimeconfig'] + rootLogger.warning("{} is overriding a deploy triplet in your config_hwdb.yaml file. Make sure you understand why!".format(name)) + self.customruntimeconfig = hwconfig_dict['custom-runtime-config'] self.customruntimeconfig = self.customruntimeconfig if self.customruntimeconfig != "None" else None # note whether we've built a copy of the simulation driver for this hwconf self.driver_built = False @@ -223,9 +222,12 @@ class RuntimeHWDB: as endpoints on the simulation. """ def __init__(self, hardwaredbconfigfile): - agfidb_configfile = configparser.ConfigParser(allow_no_value=True) - agfidb_configfile.read(hardwaredbconfigfile) - agfidb_dict = {s:dict(agfidb_configfile.items(s)) for s in agfidb_configfile.sections()} + + agfidb_configfile = None + with open(hardwaredbconfigfile, "r") as yaml_file: + agfidb_configfile = yaml.safe_load(yaml_file) + + agfidb_dict = agfidb_configfile self.hwconf_dict = {s: RuntimeHWConfig(s, v) for s, v in agfidb_dict.items()} @@ -237,12 +239,15 @@ class RuntimeHWDB: class InnerRuntimeConfiguration: - """ Pythonic version of config_runtime.ini """ + """ Pythonic version of config_runtime.yaml """ def __init__(self, runtimeconfigfile, configoverridedata): - runtime_configfile = configparser.ConfigParser(allow_no_value=True) - runtime_configfile.read(runtimeconfigfile) - runtime_dict = {s:dict(runtime_configfile.items(s)) for s in runtime_configfile.sections()} + + runtime_configfile = None + with open(runtimeconfigfile, "r") as yaml_file: + runtime_configfile = yaml.safe_load(yaml_file) + + runtime_dict = runtime_configfile # override parts of the runtime conf if specified configoverrideval = configoverridedata @@ -261,28 +266,28 @@ class InnerRuntimeConfiguration: if runfarmtagprefix != "": runfarmtagprefix += "-" - self.runfarmtag = runfarmtagprefix + runtime_dict['runfarm']['runfarmtag'] + self.runfarmtag = runfarmtagprefix + runtime_dict['run-farm']['run-farm-tag'] aws_resource_names_dict = aws_resource_names() if aws_resource_names_dict['runfarmprefix'] is not None: # if specified, further prefix runfarmtag self.runfarmtag = aws_resource_names_dict['runfarmprefix'] + "-" + self.runfarmtag - self.f1_16xlarges_requested = int(runtime_dict['runfarm']['f1_16xlarges']) if 'f1_16xlarges' in runtime_dict['runfarm'] else 0 - self.f1_4xlarges_requested = int(runtime_dict['runfarm']['f1_4xlarges']) if 'f1_4xlarges' in runtime_dict['runfarm'] else 0 - self.m4_16xlarges_requested = int(runtime_dict['runfarm']['m4_16xlarges']) if 'm4_16xlarges' in runtime_dict['runfarm'] else 0 - self.f1_2xlarges_requested = int(runtime_dict['runfarm']['f1_2xlarges']) if 'f1_2xlarges' in runtime_dict['runfarm'] else 0 + self.f1_16xlarges_requested = int(runtime_dict['run-farm']['f1_16xlarges']) if 'f1_16xlarges' in runtime_dict['run-farm'] else 0 + self.f1_4xlarges_requested = int(runtime_dict['run-farm']['f1_4xlarges']) if 'f1_4xlarges' in runtime_dict['run-farm'] else 0 + self.m4_16xlarges_requested = int(runtime_dict['run-farm']['m4_16xlarges']) if 'm4_16xlarges' in runtime_dict['run-farm'] else 0 + self.f1_2xlarges_requested = int(runtime_dict['run-farm']['f1_2xlarges']) if 'f1_2xlarges' in runtime_dict['run-farm'] else 0 - self.run_instance_market = runtime_dict['runfarm']['runinstancemarket'] - self.spot_interruption_behavior = runtime_dict['runfarm']['spotinterruptionbehavior'] - self.spot_max_price = runtime_dict['runfarm']['spotmaxprice'] + self.run_instance_market = runtime_dict['run-farm']['run-instance-market'] + self.spot_interruption_behavior = runtime_dict['run-farm']['spot-interruption-behavior'] + self.spot_max_price = runtime_dict['run-farm']['spot-max-price'] - self.topology = runtime_dict['targetconfig']['topology'] - self.no_net_num_nodes = int(runtime_dict['targetconfig']['no_net_num_nodes']) - self.linklatency = int(runtime_dict['targetconfig']['linklatency']) - self.switchinglatency = int(runtime_dict['targetconfig']['switchinglatency']) - self.netbandwidth = int(runtime_dict['targetconfig']['netbandwidth']) - self.profileinterval = int(runtime_dict['targetconfig']['profileinterval']) + self.topology = runtime_dict['target-config']['topology'] + self.no_net_num_nodes = int(runtime_dict['target-config']['no-net-num-nodes']) + self.linklatency = int(runtime_dict['target-config']['link-latency']) + self.switchinglatency = int(runtime_dict['target-config']['switching-latency']) + self.netbandwidth = int(runtime_dict['target-config']['net-bandwidth']) + self.profileinterval = int(runtime_dict['target-config']['profile-interval']) # Default values self.trace_enable = False self.trace_select = "0" @@ -301,22 +306,22 @@ class InnerRuntimeConfiguration: self.trace_select = runtime_dict['tracing'].get('selector', "0") self.trace_start = runtime_dict['tracing'].get('start', "0") self.trace_end = runtime_dict['tracing'].get('end', "-1") - self.trace_output_format = runtime_dict['tracing'].get('output_format', "0") + self.trace_output_format = runtime_dict['tracing'].get('output-format', "0") if 'autocounter' in runtime_dict: - self.autocounter_readrate = int(runtime_dict['autocounter'].get('readrate', "0")) - self.defaulthwconfig = runtime_dict['targetconfig']['defaulthwconfig'] - if 'hostdebug' in runtime_dict: - self.zerooutdram = runtime_dict['hostdebug'].get('zerooutdram') == "yes" - self.disable_asserts = runtime_dict['hostdebug'].get('disable_synth_asserts') == "yes" - if 'synthprint' in runtime_dict: - self.print_start = runtime_dict['synthprint'].get("start", "0") - self.print_end = runtime_dict['synthprint'].get("end", "-1") - self.print_cycle_prefix = runtime_dict['synthprint'].get("cycleprefix", "yes") == "yes" + self.autocounter_readrate = int(runtime_dict['autocounter'].get('read-rate', "0")) + self.defaulthwconfig = runtime_dict['target-config']['default-hw-config'] + if 'host-debug' in runtime_dict: + self.zerooutdram = runtime_dict['host-debug'].get('zero-out-dram') == "yes" + self.disable_asserts = runtime_dict['host-debug'].get('disable-synth-asserts') == "yes" + if 'synth-print' in runtime_dict: + self.print_start = runtime_dict['synth-print'].get("start", "0") + self.print_end = runtime_dict['synth-print'].get("end", "-1") + self.print_cycle_prefix = runtime_dict['synth-print'].get("cycle-prefix", "yes") == "yes" - self.workload_name = runtime_dict['workload']['workloadname'] + self.workload_name = runtime_dict['workload']['workload-name'] # an extra tag to differentiate workloads with the same name in results names - self.suffixtag = runtime_dict['workload']['suffixtag'] if 'suffixtag' in runtime_dict['workload'] else "" - self.terminateoncompletion = runtime_dict['workload']['terminateoncompletion'] == "yes" + self.suffixtag = runtime_dict['workload']['suffix-tag'] if 'suffix-tag' in runtime_dict['workload'] else "" + self.terminateoncompletion = runtime_dict['workload']['terminate-on-completion'] == "yes" def __str__(self): return pprint.pformat(vars(self)) diff --git a/deploy/sample-backup-configs/sample_config_build.ini b/deploy/sample-backup-configs/sample_config_build.ini deleted file mode 100644 index 04130360..00000000 --- a/deploy/sample-backup-configs/sample_config_build.ini +++ /dev/null @@ -1,43 +0,0 @@ -# BUILDTIME/AGFI management configuration for the FireSim Simulation Manager -# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. - -[builds] -# this section references builds defined in config_build_recipes.ini -# if you add a build here, it will be built when you run buildafi - -# Unnetworked designs use a three-domain configuration -# Tiles: 1600 MHz -# -# Uncore: 800 MHz -# -# DRAM : 1000 MHz -firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3 -firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3 - -# All NIC-based designs use the legacy FireSim frequency selection, with the -# tiles and uncore running at 3.2 GHz to sustain 200Gb theoretical NIC BW -firesim-supernode-rocket-singlecore-nic-l2-lbp -firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 -firesim-boom-singlecore-nic-l2-llc4mb-ddr3 - -# SHA3 configs for tutorial -# firesim-singlecore-sha3-no-nic-l2-llc4mb-ddr3 -# firesim-singlecore-sha3-print-no-nic-l2-llc4mb-ddr3 - -[agfistoshare] -firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 -firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3 -firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3 -firesim-boom-singlecore-nic-l2-llc4mb-ddr3 - -firesim-supernode-rocket-singlecore-nic-l2-lbp - -# SHA3 configs for tutorial -# firesim-singlecore-sha3-no-nic-l2-llc4mb-ddr3 -# firesim-singlecore-sha3-print-no-nic-l2-llc4mb-ddr3 - -[sharewithaccounts] -# To share with a specific user: -somebodysname=123456789012 -# To share publicly: -#public=public diff --git a/deploy/sample-backup-configs/sample_config_build.yaml b/deploy/sample-backup-configs/sample_config_build.yaml new file mode 100644 index 00000000..f981c6e6 --- /dev/null +++ b/deploy/sample-backup-configs/sample_config_build.yaml @@ -0,0 +1,43 @@ +# BUILDTIME/AGFI management configuration for the FireSim Simulation Manager +# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. + +builds: + # this section references builds defined in config_build_recipes.yaml + # if you add a build here, it will be built when you run buildafi + + # Unnetworked designs use a three-domain configuration + # Tiles: 1600 MHz + # + # Uncore: 800 MHz + # + # DRAM : 1000 MHz + - firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3 + - firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3 + + # All NIC-based designs use the legacy FireSim frequency selection, with the + # tiles and uncore running at 3.2 GHz to sustain 200Gb theoretical NIC BW + - firesim-supernode-rocket-singlecore-nic-l2-lbp + - firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + - firesim-boom-singlecore-nic-l2-llc4mb-ddr3 + + # SHA3 configs for tutorial + # firesim-singlecore-sha3-no-nic-l2-llc4mb-ddr3 + # firesim-singlecore-sha3-print-no-nic-l2-llc4mb-ddr3 + +agfis-to-share: + - firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + - firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3 + - firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3 + - firesim-boom-singlecore-nic-l2-llc4mb-ddr3 + + - firesim-supernode-rocket-singlecore-nic-l2-lbp + + # SHA3 configs for tutorial + #- firesim-singlecore-sha3-no-nic-l2-llc4mb-ddr3 + #- firesim-singlecore-sha3-print-no-nic-l2-llc4mb-ddr3 + +share-with-accounts: + # To share with a specific user: + somebodysname: 123456789012 + # To share publicly: + #public: public diff --git a/deploy/sample-backup-configs/sample_config_build_farm.yaml b/deploy/sample-backup-configs/sample_config_build_farm.yaml new file mode 100644 index 00000000..5a744016 --- /dev/null +++ b/deploy/sample-backup-configs/sample_config_build_farm.yaml @@ -0,0 +1,25 @@ +### Build Farm ### + +# THIS WILL BE USED IF BUILDFARM NOT SPECIFIED IN RECIPE +# Note: For large designs (ones that would fill a EC2.2xlarge/Xilinx VU9P) +# Vivado uses in excess of 32 GiB. Keep this in mind when selecting a +# non-default instance type. +default-build-farm: + build-farm-type: aws-ec2 + args: + instance-type: z1d.2xlarge + build-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + # change location of build on build host + build-dir: null + +local-build-farm: + build-farm-type: unmanaged + args: + default-build-dir: null + build-farm-hosts: + - localhost + - "192.168.1.254": + # change location of build on build host (abs. path) + override-build-dir: null diff --git a/deploy/sample-backup-configs/sample_config_build_hosts.ini b/deploy/sample-backup-configs/sample_config_build_hosts.ini deleted file mode 100644 index 22b11ef1..00000000 --- a/deploy/sample-backup-configs/sample_config_build_hosts.ini +++ /dev/null @@ -1,19 +0,0 @@ -### Build Hosts ### - -# THIS WILL BE USED IF BUILDHOST NOT SPECIFIED IN RECIPE -# Note: For large designs (ones that would fill a EC2.2xlarge/Xilinx VU9P) -# Vivado uses in excess of 32 GiB. Keep this in mind when selecting a -# non-default instancetype. -[defaultbuildhost] -providerclass=EC2BuildHostDispatcher -instancetype=z1d.2xlarge -buildinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand -# change builddir w/ remote machines (empty defaults to no change) -remotebuilddir= - -[localbuildhost] -providerclass=IPAddrBuildHostDispatcher -remotebuilddir= -ipaddr=localhost diff --git a/deploy/sample-backup-configs/sample_config_build_recipes.ini b/deploy/sample-backup-configs/sample_config_build_recipes.ini deleted file mode 100644 index 5871ba8b..00000000 --- a/deploy/sample-backup-configs/sample_config_build_recipes.ini +++ /dev/null @@ -1,108 +0,0 @@ -# Build-time design configuration for the FireSim Simulation Manager -# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. - -# this file contains sections that describe hardware designs that /can/ be built. -# edit config_build.ini to actually "turn on" a config to be built when you run -# buildafi - -# Example Build Recipe -# [example-recipe-name] -# # DESIGN SPECIFIC -# DESIGN=TopModule -# TARGET_CONFIG=Config -# deploytriplet=None -# # RUN PLATFORM SPECIFIC -# PLATFORM_CONFIG=Config -# postbuildhook= -# s3bucketname=firesim-AWSUSERNAME -# # BUILD PLATFORM SPECIFIC (if unspecified, use the defaultbuildhost) -# buildhost=defaultbuildhost - -# Quad-core, Rocket-based recipes -[firesim-rocket-quadcore-nic-l2-llc4mb-ddr3] -DESIGN=FireSim -TARGET_CONFIG=WithNIC_DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimHighPerfConfigTweaks_chipyard.QuadRocketConfig -deploytriplet=None -PLATFORM_CONFIG=F90MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - -# NB: This has a faster host-clock frequency than the NIC-based design, because -# its uncore runs at half rate relative to the tile. -[firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3] -DESIGN=FireSim -TARGET_CONFIG=DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimTestChipConfigTweaks_chipyard.QuadRocketConfig -deploytriplet=None -PLATFORM_CONFIG=F140MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - - -# Single-core, BOOM-based recipes -[firesim-boom-singlecore-nic-l2-llc4mb-ddr3] -DESIGN=FireSim -TARGET_CONFIG=WithNIC_DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimHighPerfConfigTweaks_chipyard.LargeBoomConfig -deploytriplet=None -PLATFORM_CONFIG=F65MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - -# NB: This has a faster host-clock frequency than the NIC-based design, because -# its uncore runs at half rate relative to the tile. -[firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3] -DESIGN=FireSim -TARGET_CONFIG=DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimTestChipConfigTweaks_chipyard.LargeBoomConfig -deploytriplet=None -PLATFORM_CONFIG=F75MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - - -# Single-core, CVA6-based recipes -[firesim-cva6-singlecore-no-nic-l2-llc4mb-ddr3] -DESIGN=FireSim -TARGET_CONFIG=DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimConfigTweaks_chipyard.CVA6Config -deploytriplet=None -PLATFORM_CONFIG=F90MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - - -# Single-core, Rocket-based recipes with Gemmini -[firesim-rocket-singlecore-gemmini-no-nic-l2-llc4mb-ddr3] -DESIGN=FireSim -TARGET_CONFIG=DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimConfigTweaks_chipyard.GemminiRocketConfig -deploytriplet=None -PLATFORM_CONFIG=F110MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - - -# RAM Optimizations enabled by adding _MCRams PLATFORM_CONFIG string -[firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3-ramopts] -DESIGN=FireSim -TARGET_CONFIG=DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimTestChipConfigTweaks_chipyard.LargeBoomConfig -deploytriplet=None -PLATFORM_CONFIG=MCRams_F90MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - - -# Supernode configurations -- multiple instances of an SoC in a single simulator -[firesim-supernode-rocket-singlecore-nic-l2-lbp] -DESIGN=FireSim -TARGET_CONFIG=WithNIC_SupernodeFireSimRocketConfig -deploytriplet=None -PLATFORM_CONFIG=F85MHz_BaseF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME - -# MIDAS Examples -- BUILD SUPPORT ONLY; Can't launch driver correctly on runfarm -[midasexamples-gcd] -TARGET_PROJECT=midasexamples -DESIGN=GCD -TARGET_CONFIG=NoConfig -deploytriplet=None -PLATFORM_CONFIG=DefaultF1Config -postbuildhook= -s3bucketname=firesim-AWSUSERNAME diff --git a/deploy/sample-backup-configs/sample_config_build_recipes.yaml b/deploy/sample-backup-configs/sample_config_build_recipes.yaml new file mode 100644 index 00000000..901a08ec --- /dev/null +++ b/deploy/sample-backup-configs/sample_config_build_recipes.yaml @@ -0,0 +1,112 @@ +# Build-time design configuration for the FireSim Simulation Manager +# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. + +# this file contains sections that describe hardware designs that /can/ be built. +# edit config_build.yaml to actually "turn on" a config to be built when you run +# buildafi + +# Example Build Recipe +# example-recipe-name: +# # DESIGN SPECIFIC +# DESIGN: TopModule +# TARGET_CONFIG: Config +# deploy-triplet: null +# # RUN PLATFORM SPECIFIC +# PLATFORM_CONFIG: Config +# post-build-hook: null +# s3-bucket-name: firesim-DUMMY +# # BUILD PLATFORM SPECIFIC +# build-farm: default-build-farm + +# Quad-core, Rocket-based recipes +firesim-rocket-quadcore-nic-l2-llc4mb-ddr3: + DESIGN: FireSim + TARGET_CONFIG: WithNIC_DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimHighPerfConfigTweaks_chipyard.QuadRocketConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F90MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# NB: This has a faster host-clock frequency than the NIC-based design, because +# its uncore runs at half rate relative to the tile. +firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3: + DESIGN: FireSim + TARGET_CONFIG: DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimTestChipConfigTweaks_chipyard.QuadRocketConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F140MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# Single-core, BOOM-based recipes +firesim-boom-singlecore-nic-l2-llc4mb-ddr3: + DESIGN: FireSim + TARGET_CONFIG: WithNIC_DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimHighPerfConfigTweaks_chipyard.LargeBoomConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F65MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# NB: This has a faster host-clock frequency than the NIC-based design, because +# its uncore runs at half rate relative to the tile. +firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3: + DESIGN: FireSim + TARGET_CONFIG: DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimTestChipConfigTweaks_chipyard.LargeBoomConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F75MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# Single-core, CVA6-based recipes +firesim-cva6-singlecore-no-nic-l2-llc4mb-ddr3: + DESIGN: FireSim + TARGET_CONFIG: DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimConfigTweaks_chipyard.CVA6Config + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F90MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# Single-core, Rocket-based recipes with Gemmini +firesim-rocket-singlecore-gemmini-no-nic-l2-llc4mb-ddr3: + DESIGN: FireSim + TARGET_CONFIG: DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimConfigTweaks_chipyard.GemminiRocketConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F110MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# RAM Optimizations enabled by adding _MCRams PLATFORM_CONFIG string +firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3-ramopts: + DESIGN: FireSim + TARGET_CONFIG: DDR3FRFCFSLLC4MB_WithDefaultFireSimBridges_WithFireSimTestChipConfigTweaks_chipyard.LargeBoomConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_MCRams_F90MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# Supernode configurations -- multiple instances of an SoC in a single simulator +firesim-supernode-rocket-singlecore-nic-l2-lbp: + DESIGN: FireSim + TARGET_CONFIG: WithNIC_SupernodeFireSimRocketConfig + deploy-triplet: null + PLATFORM_CONFIG: WithAutoILA_F85MHz_BaseF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm + +# MIDAS Examples -- BUILD SUPPORT ONLY; Can't launch driver correctly on run farm +midasexamples-gcd: + TARGET_PROJECT: midasexamples + DESIGN: GCD + TARGET_CONFIG: NoConfig + deploy-triplet: null + PLATFORM_CONFIG: DefaultF1Config + post-build-hook: null + s3-bucket-name: firesim-DUMMY + build-farm: default-build-farm diff --git a/deploy/sample-backup-configs/sample_config_hwdb.ini b/deploy/sample-backup-configs/sample_config_hwdb.ini deleted file mode 100644 index b91f4d58..00000000 --- a/deploy/sample-backup-configs/sample_config_hwdb.ini +++ /dev/null @@ -1,36 +0,0 @@ -# Hardware config database for FireSim Simulation Manager -# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. - -# Hardware configs represent a combination of an agfi, a deploytriplet override -# (if needed), and a custom runtime config (if needed) - -# The AGFIs provided below are public and available to all users. -# Only AGFIs for the latest release of FireSim are guaranteed to be available. -# If you are using an older version of FireSim, you will need to generate your -# own images. - -[firesim-boom-singlecore-nic-l2-llc4mb-ddr3] -agfi=agfi-011237cb56f4e09d0 -deploytripletoverride=None -customruntimeconfig=None - -[firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3] -agfi=agfi-0527595dd6245d41a -deploytripletoverride=None -customruntimeconfig=None - -[firesim-rocket-quadcore-nic-l2-llc4mb-ddr3] -agfi=agfi-027d8c3d242d7c024 -deploytripletoverride=None -customruntimeconfig=None - -[firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3] -agfi=agfi-09f7ad49e6da339d5 -deploytripletoverride=None -customruntimeconfig=None - -[firesim-supernode-rocket-singlecore-nic-l2-lbp] -agfi=agfi-0968f0d2b39fc2428 -deploytripletoverride=None -customruntimeconfig=None - diff --git a/deploy/sample-backup-configs/sample_config_hwdb.yaml b/deploy/sample-backup-configs/sample_config_hwdb.yaml new file mode 100644 index 00000000..81e9e321 --- /dev/null +++ b/deploy/sample-backup-configs/sample_config_hwdb.yaml @@ -0,0 +1,36 @@ +# Hardware config database for FireSim Simulation Manager +# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. + +# Hardware configs represent a combination of an agfi, a deploytriplet override +# (if needed), and a custom runtime config (if needed) + +# The AGFIs provided below are public and available to all users. +# Only AGFIs for the latest release of FireSim are guaranteed to be available. +# If you are using an older version of FireSim, you will need to generate your +# own images. + +firesim-boom-singlecore-nic-l2-llc4mb-ddr3: + agfi: agfi-0aa96f4214e125fa7 + deploy-triplet-override: null + custom-runtime-config: null + +firesim-boom-singlecore-no-nic-l2-llc4mb-ddr3: + agfi: agfi-05696fc3823b9c6db + deploy-triplet-override: null + custom-runtime-config: null + +firesim-rocket-quadcore-nic-l2-llc4mb-ddr3: + agfi: agfi-0e817cc7ccaf9fd0a + deploy-triplet-override: null + custom-runtime-config: null + +firesim-rocket-quadcore-no-nic-l2-llc4mb-ddr3: + agfi: agfi-0833e931ad5f2fa38 + deploy-triplet-override: null + custom-runtime-config: null + +firesim-supernode-rocket-singlecore-nic-l2-lbp: + agfi: agfi-02059611104e23100 + deploy-triplet-override: null + custom-runtime-config: null + diff --git a/deploy/sample-backup-configs/sample_config_runtime.ini b/deploy/sample-backup-configs/sample_config_runtime.ini deleted file mode 100644 index 16069819..00000000 --- a/deploy/sample-backup-configs/sample_config_runtime.ini +++ /dev/null @@ -1,73 +0,0 @@ -# RUNTIME configuration for the FireSim Simulation Manager -# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for -# documentation of all of these params. - -[runfarm] -runfarmtag=mainrunfarm - -f1_16xlarges=1 -m4_16xlarges=0 -f1_4xlarges=0 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -#Set topology=no_net_config to run without a network simulation -topology=example_8config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -# This references a section from config_build_recipes.ini -# In homogeneous configurations, use this to set the hardware config deployed -# for all simulators -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no - -# Trace output formats. Only enabled if "enable" is set to "yes" above -# 0 = human readable; 1 = binary (compressed raw data); 2 = flamegraph (stack -# unwinding -> Flame Graph) -output_format=0 - -# Trigger selector. -# 0 = no trigger; 1 = cycle count trigger; 2 = program counter trigger; 3 = -# instruction trigger -selector=1 -start=0 -end=-1 - -[autocounter] -readrate=0 - -[workload] -workloadname=linux-uniform.json -terminateoncompletion=no -suffixtag= - -[hostdebug] -# When enabled (=yes), Zeros-out FPGA-attached DRAM before simulations -# begin (takes 2-5 minutes). -# In general, this is not required to produce deterministic simulations on -# target machines running linux. Enable if you observe simulation non-determinism. -zerooutdram=no -# If disable_synth_asserts=no, simulation will print assertion message and -# terminate simulation if synthesized assertion fires. -# If disable_synth_asserts=yes, simulation ignores assertion firing and -# continues simulation. -disable_synth_asserts=no - -[synthprint] -# Start and end cycles for outputting synthesized prints. -# They are given in terms of the base clock and will be converted -# for each clock domain. -start=0 -end=-1 -# When enabled (=yes), prefix print output with the target cycle at which the print was triggered -cycleprefix=yes diff --git a/deploy/sample-backup-configs/sample_config_runtime.yaml b/deploy/sample-backup-configs/sample_config_runtime.yaml new file mode 100644 index 00000000..4ba16e76 --- /dev/null +++ b/deploy/sample-backup-configs/sample_config_runtime.yaml @@ -0,0 +1,71 @@ +# RUNTIME configuration for the FireSim Simulation Manager +# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for +# documentation of all of these params. + +run-farm: + run-farm-tag: mainrunfarm + f1_16xlarges: 1 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + # Set topology: no_net_config to run without a network simulation + topology: no_net_config + no-net-num-nodes: 1 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + + # This references a section from config_build_recipes.yaml + # In homogeneous configurations, use this to set the hardware config deployed + # for all simulators + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + + # Trace output formats. Only enabled if "enable" is set to "yes" above + # 0 : human readable; 1 : binary (compressed raw data); 2 : flamegraph (stack + # unwinding -> Flame Graph) + output-format: 0 + + # Trigger selector. + # 0 : no trigger; 1 : cycle count trigger; 2 : program counter trigger; 3 : + # instruction trigger + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: linux-uniform.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + # When enabled (: yes), Zeros-out FPGA-attached DRAM before simulations + # begin (takes 2-5 minutes). + # In general, this is not required to produce deterministic simulations on + # target machines running linux. Enable if you observe simulation non-determinism. + zero-out-dram: no + # If disable-synth-asserts: no, simulation will print assertion message and + # terminate simulation if synthesized assertion fires. + # If disable-synth-asserts: yes, simulation ignores assertion firing and + # continues simulation. + disable-synth-asserts: no + +synth-print: + # Start and end cycles for outputting synthesized prints. + # They are given in terms of the base clock and will be converted + # for each clock domain. + start: 0 + end: -1 + # When enabled (: yes), prefix print output with the target cycle at which the print was triggered + cycle-prefix: yes diff --git a/deploy/workloads/bw-test-config.ini b/deploy/workloads/bw-test-config.ini deleted file mode 100644 index 42de97e9..00000000 --- a/deploy/workloads/bw-test-config.ini +++ /dev/null @@ -1,24 +0,0 @@ -[runfarm] -runfarmtag=bwtest-mainrunfarm - -f1_16xlarges=2 -m4_16xlarges=1 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=example_16config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[workload] -workloadname=bw-test-two-instances.json -terminateoncompletion=no diff --git a/deploy/workloads/bw-test-config.yaml b/deploy/workloads/bw-test-config.yaml new file mode 100644 index 00000000..16f2114f --- /dev/null +++ b/deploy/workloads/bw-test-config.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: bwtest-mainrun-farm + f1_16xlarges: 2 + m4_16xlarges: 1 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: example_16config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: bw-test-two-instances.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/ccbench-cache-sweep.ini b/deploy/workloads/ccbench-cache-sweep.ini deleted file mode 100644 index 7f7986b7..00000000 --- a/deploy/workloads/ccbench-cache-sweep.ini +++ /dev/null @@ -1,31 +0,0 @@ - -[runfarm] -runfarmtag=ccbench-cache-sweep - -f1_16xlarges=0 -m4_16xlarges=0 -f1_2xlarges=1 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=no_net_config -no_net_num_nodes=1 -# These are unused -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-quadcore-no-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no -startcycle=0 -endcycle=-1 - -[workload] -workloadname=ccbench-cache-sweep.json -terminateoncompletion=no diff --git a/deploy/workloads/ccbench-cache-sweep.yaml b/deploy/workloads/ccbench-cache-sweep.yaml new file mode 100644 index 00000000..5153153e --- /dev/null +++ b/deploy/workloads/ccbench-cache-sweep.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: ccbench-cache-sweep + f1_16xlarges: 0 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 1 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: no_net_config + no-net-num-nodes: 1 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-quadcore-no-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: ccbench-cache-sweep.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/gapbs.ini b/deploy/workloads/gapbs.ini deleted file mode 100644 index fae81bcf..00000000 --- a/deploy/workloads/gapbs.ini +++ /dev/null @@ -1,31 +0,0 @@ -[runfarm] -runfarmtag=gapbs-runfarm - -f1_16xlarges=0 -m4_16xlarges=0 -f1_2xlarges=6 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=no_net_config -no_net_num_nodes=6 -# These are unused -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -# Need not be single core. -defaulthwconfig=firesim-quadcore-no-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no -startcyle=0 -endcycle=-1 - -[workload] -workloadname=gapbs.json -terminateoncompletion=yes diff --git a/deploy/workloads/gapbs.yaml b/deploy/workloads/gapbs.yaml new file mode 100644 index 00000000..8c60a0bd --- /dev/null +++ b/deploy/workloads/gapbs.yaml @@ -0,0 +1,44 @@ +run-farm: + run-farm-tag: gapbs-runfarm + f1_16xlarges: 0 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 6 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: no_net_config + no-net-num-nodes: 6 + # These are unused + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + # Need not be single core. + default-hw-config: firesim-quadcore-no-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: gapbs.json + terminate-on-completion: yes + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/linux-poweroff-all-no-nic.ini b/deploy/workloads/linux-poweroff-all-no-nic.ini deleted file mode 100644 index 9a833876..00000000 --- a/deploy/workloads/linux-poweroff-all-no-nic.ini +++ /dev/null @@ -1,36 +0,0 @@ -# RUNTIME configuration for the FireSim Simulation Manager -# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for documentation of all of these params. - -[runfarm] -runfarmtag=linuxpoweroffallnonictargets - -f1_16xlarges=0 -m4_16xlarges=0 -f1_4xlarges=0 -f1_2xlarges=2 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=all_no_net_targets_config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -# This references a section from config_hwconfigs.ini -# In homogeneous configurations, use this to set the hardware config deployed -# for all simulators -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no -startcycle=0 -endcycle=-1 - -[workload] -workloadname=linux-poweroff-uniform.json -terminateoncompletion=no diff --git a/deploy/workloads/linux-poweroff-all-no-nic.yaml b/deploy/workloads/linux-poweroff-all-no-nic.yaml new file mode 100644 index 00000000..230d9356 --- /dev/null +++ b/deploy/workloads/linux-poweroff-all-no-nic.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: linuxpoweroffallnonictargets + f1_16xlarges: 0 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 2 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: all_no_net_targets_config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: linux-poweroff-uniform.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/linux-poweroff-nic.ini b/deploy/workloads/linux-poweroff-nic.ini deleted file mode 100644 index 71d4cbd1..00000000 --- a/deploy/workloads/linux-poweroff-nic.ini +++ /dev/null @@ -1,73 +0,0 @@ -# RUNTIME configuration for the FireSim Simulation Manager -# See docs/Advanced-Usage/Manager/Manager-Configuration-Files.rst for -# documentation of all of these params. - -[runfarm] -runfarmtag=linuxpoweroffnic - -f1_16xlarges=1 -m4_16xlarges=0 -f1_4xlarges=0 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -#Set topology=no_net_config to run without a network simulation -topology=example_8config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -# This references a section from config_hwconfigs.ini -# In homogeneous configurations, use this to set the hardware config deployed -# for all simulators -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no - -# Trace output formats. Only enabled if "enable" is set to "yes" above -# 0 = human readable; 1 = binary (compressed raw data); 2 = flamegraph (stack -# unwinding -> Flame Graph) -output_format=0 - -# Trigger selector. -# 0 = no trigger; 1 = cycle count trigger; 2 = program counter trigger; 3 = -# instruction trigger -selector=1 -start=0 -end=-1 - -[autocounter] -readrate=0 - -[workload] -workloadname=linux-poweroff-uniform.json -terminateoncompletion=no -suffixtag= - -[hostdebug] -# When enabled (=yes), Zeros-out FPGA-attached DRAM before simulations -# begin (takes 2-5 minutes). -# In general, this is not required to produce deterministic simulations on -# target machines running linux. Enable if you observe simulation non-determinism. -zerooutdram=no -# If disable_synth_asserts=no, simulation will print assertion message and -# terminate simulation if synthesized assertion fires. -# If disable_synth_asserts=yes, simulation ignores assertion firing and -# continues simulation. -disable_synth_asserts=no - -[synthprint] -# Start and end cycles for outputting synthesized prints. -# They are given in terms of the base clock and will be converted -# for each clock domain. -start=0 -end=-1 -# When enabled (=yes), prefix print output with the target cycle at which the print was triggered -cycleprefix=yes diff --git a/deploy/workloads/linux-poweroff-nic.yaml b/deploy/workloads/linux-poweroff-nic.yaml new file mode 100644 index 00000000..b7d5ee3f --- /dev/null +++ b/deploy/workloads/linux-poweroff-nic.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: linuxpoweroffnic + f1_16xlarges: 1 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: example_8config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: linux-poweroff-uniform.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/memcached-thread-imbalance-config.ini b/deploy/workloads/memcached-thread-imbalance-config.ini deleted file mode 100644 index 45904e1c..00000000 --- a/deploy/workloads/memcached-thread-imbalance-config.ini +++ /dev/null @@ -1,24 +0,0 @@ -[runfarm] -runfarmtag=memcached-mainrunfarm - -f1_16xlarges=3 -m4_16xlarges=0 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=triple_example_8config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[workload] -workloadname=memcached-thread-imbalance.json -terminateoncompletion=no diff --git a/deploy/workloads/memcached-thread-imbalance-config.yaml b/deploy/workloads/memcached-thread-imbalance-config.yaml new file mode 100644 index 00000000..f020ba6a --- /dev/null +++ b/deploy/workloads/memcached-thread-imbalance-config.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: memcached-mainrunfarm + f1_16xlarges: 3 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: triple_example_8config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: memcached-thread-imbalance.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/ping-latency-config.ini b/deploy/workloads/ping-latency-config.ini deleted file mode 100644 index 54ad4f19..00000000 --- a/deploy/workloads/ping-latency-config.ini +++ /dev/null @@ -1,24 +0,0 @@ -[runfarm] -runfarmtag=pinglatency-mainrunfarm - -f1_16xlarges=1 -m4_16xlarges=0 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=example_8config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[workload] -workloadname=ping-latency.json -terminateoncompletion=no diff --git a/deploy/workloads/ping-latency-config.yaml b/deploy/workloads/ping-latency-config.yaml new file mode 100644 index 00000000..f942fc8d --- /dev/null +++ b/deploy/workloads/ping-latency-config.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: pinglatency-mainrunfarm + f1_16xlarges: 1 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: example_8config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: ping-latency.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/replace-agfi-used.sh b/deploy/workloads/replace-agfi-used.sh index efd2d7c7..10c6267c 100755 --- a/deploy/workloads/replace-agfi-used.sh +++ b/deploy/workloads/replace-agfi-used.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash -inis=(bw-test-config.ini memcached-thread-imbalance-config.ini ping-latency-config.ini simperf-test-latency-config.ini simperf-test-scale-config-continue.ini simperf-test-scale-config.ini) +yamls=(bw-test-config.yaml memcached-thread-imbalance-config.yaml ping-latency-config.yaml simperf-test-latency-config.yaml simperf-test-scale-config-continue.yaml simperf-test-scale-config.yaml) OLDAGFI="defaultserver=quadcore-nic-ddr3-llc4M" NEWAGFI="defaultserver=firesim-quadcore-nic-ddr3-llc4mb" -for i in "${inis[@]}" +for i in "${yamls[@]}" do sed -i "s/$OLDAGFI/$NEWAGFI/g" $i done diff --git a/deploy/workloads/run-bw-test.sh b/deploy/workloads/run-bw-test.sh index 99d56d1e..7899eaeb 100755 --- a/deploy/workloads/run-bw-test.sh +++ b/deploy/workloads/run-bw-test.sh @@ -10,7 +10,7 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim launchrunfarm -c workloads/bw-test-config.ini + firesim launchrunfarm -c workloads/bw-test-config.yaml fi ORIGDIR=$(pwd) @@ -32,8 +32,8 @@ sleep 2 for i in "${bandwidths[@]}" do - firesim infrasetup -c workloads/bw-test-config.ini - firesim runworkload -c workloads/bw-test-config.ini --overrideconfigdata "targetconfig netbandwidth $i" + firesim infrasetup -c workloads/bw-test-config.yaml + firesim runworkload -c workloads/bw-test-config.yaml --overrideconfigdata "targetconfig netbandwidth $i" # rename the output directory with the net bandwidth files=(*bw-test*) originalfilename=${files[-1]} @@ -46,5 +46,5 @@ cd $ORIGDIR cd ../../ git apply -R $ORIGDIR/bw-test-two-instances/switchpatch.patch -firesim terminaterunfarm -c workloads/bw-test-config.ini --forceterminate +firesim terminaterunfarm -c workloads/bw-test-config.yaml --forceterminate diff --git a/deploy/workloads/run-ccbench-cache-sweep.sh b/deploy/workloads/run-ccbench-cache-sweep.sh index 206652b2..df5e564d 100755 --- a/deploy/workloads/run-ccbench-cache-sweep.sh +++ b/deploy/workloads/run-ccbench-cache-sweep.sh @@ -7,10 +7,10 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim -c workloads/ccbench-cache-sweep.ini launchrunfarm + firesim -c workloads/ccbench-cache-sweep.yaml launchrunfarm fi -firesim -c workloads/ccbench-cache-sweep.ini infrasetup -firesim -c workloads/ccbench-cache-sweep.ini runworkload -firesim -c workloads/ccbench-cache-sweep.ini terminaterunfarm --forceterminate +firesim -c workloads/ccbench-cache-sweep.yaml infrasetup +firesim -c workloads/ccbench-cache-sweep.yaml runworkload +firesim -c workloads/ccbench-cache-sweep.yaml terminaterunfarm --forceterminate diff --git a/deploy/workloads/run-memcached-thread-imbalance.sh b/deploy/workloads/run-memcached-thread-imbalance.sh index 14aa45da..764aa13c 100755 --- a/deploy/workloads/run-memcached-thread-imbalance.sh +++ b/deploy/workloads/run-memcached-thread-imbalance.sh @@ -10,11 +10,11 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim launchrunfarm -c workloads/memcached-thread-imbalance-config.ini + firesim launchrunfarm -c workloads/memcached-thread-imbalance-config.yaml fi -firesim infrasetup -c workloads/memcached-thread-imbalance-config.ini -firesim runworkload -c workloads/memcached-thread-imbalance-config.ini -firesim terminaterunfarm -c workloads/memcached-thread-imbalance-config.ini --forceterminate +firesim infrasetup -c workloads/memcached-thread-imbalance-config.yaml +firesim runworkload -c workloads/memcached-thread-imbalance-config.yaml +firesim terminaterunfarm -c workloads/memcached-thread-imbalance-config.yaml --forceterminate diff --git a/deploy/workloads/run-ping-latency.sh b/deploy/workloads/run-ping-latency.sh index 9d92a869..90ae21d1 100755 --- a/deploy/workloads/run-ping-latency.sh +++ b/deploy/workloads/run-ping-latency.sh @@ -10,7 +10,7 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim launchrunfarm -c workloads/ping-latency-config.ini + firesim launchrunfarm -c workloads/ping-latency-config.yaml fi ORIGDIR=$(pwd) @@ -31,8 +31,8 @@ sleep 2 for i in "${latencies[@]}" do - firesim infrasetup -c workloads/ping-latency-config.ini - firesim runworkload -c workloads/ping-latency-config.ini --overrideconfigdata "targetconfig linklatency $i" + firesim infrasetup -c workloads/ping-latency-config.yaml + firesim runworkload -c workloads/ping-latency-config.yaml --overrideconfigdata "targetconfig linklatency $i" # rename the output directory with the ping latency files=(*ping-latency*) originalfilename=${files[-1]} @@ -41,5 +41,5 @@ done python3 $ORIGDIR/ping-latency/ping-latency-graph.py $(pwd)/$resultsdir -firesim terminaterunfarm -c workloads/ping-latency-config.ini --forceterminate +firesim terminaterunfarm -c workloads/ping-latency-config.yaml --forceterminate diff --git a/deploy/workloads/run-simperf-test-latency.sh b/deploy/workloads/run-simperf-test-latency.sh index 7a499aa5..10f83e3b 100755 --- a/deploy/workloads/run-simperf-test-latency.sh +++ b/deploy/workloads/run-simperf-test-latency.sh @@ -10,7 +10,7 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim launchrunfarm -c workloads/simperf-test-latency-config.ini + firesim launchrunfarm -c workloads/simperf-test-latency-config.yaml fi ORIGDIR=$(pwd) @@ -31,8 +31,8 @@ sleep 2 for i in "${latencies[@]}" do - firesim infrasetup -c workloads/simperf-test-latency-config.ini - firesim runworkload -c workloads/simperf-test-latency-config.ini --overrideconfigdata "targetconfig linklatency $i" + firesim infrasetup -c workloads/simperf-test-latency-config.yaml + firesim runworkload -c workloads/simperf-test-latency-config.yaml --overrideconfigdata "targetconfig linklatency $i" # rename the output directory with the ping latency files=(*simperf-test-latency*) originalfilename=${files[-1]} @@ -40,6 +40,6 @@ do done python3 $ORIGDIR/simperf-test-latency/simperf-test-results.py $(pwd)/$resultsdir -firesim terminaterunfarm -c workloads/simperf-test-latency-config.ini --forceterminate +firesim terminaterunfarm -c workloads/simperf-test-latency-config.yaml --forceterminate diff --git a/deploy/workloads/run-simperf-test-scale-supernode.sh b/deploy/workloads/run-simperf-test-scale-supernode.sh index 18c9c9f8..45de9255 100755 --- a/deploy/workloads/run-simperf-test-scale-supernode.sh +++ b/deploy/workloads/run-simperf-test-scale-supernode.sh @@ -23,7 +23,7 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim launchrunfarm -c workloads/simperf-test-scale-supernode-config.ini + firesim launchrunfarm -c workloads/simperf-test-scale-supernode-config.yaml fi ORIGDIR=$(pwd) @@ -43,14 +43,14 @@ loopfunc () { # arg 1 is num nodes # arg 2 is num f116xlarges to kill AFTERWARDS # arg 3 is num m416xlarges to kill AFTERWARDS - firesim infrasetup -c workloads/simperf-test-scale-supernode-config.ini --overrideconfigdata "targetconfig topology supernode_example_$1config" - firesim runworkload -c workloads/simperf-test-scale-supernode-config.ini --overrideconfigdata "targetconfig topology supernode_example_$1config" + firesim infrasetup -c workloads/simperf-test-scale-supernode-config.yaml --overrideconfigdata "targetconfig topology supernode_example_$1config" + firesim runworkload -c workloads/simperf-test-scale-supernode-config.yaml --overrideconfigdata "targetconfig topology supernode_example_$1config" # rename the output directory with the ping latency files=(*simperf-test-scale*) originalfilename=${files[-1]} mv $originalfilename $resultsdir/$1 - firesim terminaterunfarm -c workloads/simperf-test-scale-supernode-config.ini --terminatesomef116 $2 --terminatesomem416 $3 --forceterminate + firesim terminaterunfarm -c workloads/simperf-test-scale-supernode-config.yaml --terminatesomef116 $2 --terminatesomem416 $3 --forceterminate } diff --git a/deploy/workloads/run-simperf-test-scale.sh b/deploy/workloads/run-simperf-test-scale.sh index f85ce246..2feed796 100755 --- a/deploy/workloads/run-simperf-test-scale.sh +++ b/deploy/workloads/run-simperf-test-scale.sh @@ -23,7 +23,7 @@ set -e set -o pipefail if [ "$1" == "withlaunch" ]; then - firesim launchrunfarm -c workloads/simperf-test-scale-config.ini + firesim launchrunfarm -c workloads/simperf-test-scale-config.yaml fi ORIGDIR=$(pwd) @@ -43,14 +43,14 @@ loopfunc () { # arg 1 is num nodes # arg 2 is num f116xlarges to kill AFTERWARDS # arg 3 is num m416xlarges to kill AFTERWARDS - firesim infrasetup -c workloads/simperf-test-scale-config.ini --overrideconfigdata "targetconfig topology example_$1config" - firesim runworkload -c workloads/simperf-test-scale-config.ini --overrideconfigdata "targetconfig topology example_$1config" + firesim infrasetup -c workloads/simperf-test-scale-config.yaml --overrideconfigdata "targetconfig topology example_$1config" + firesim runworkload -c workloads/simperf-test-scale-config.yaml --overrideconfigdata "targetconfig topology example_$1config" # rename the output directory with the ping latency files=(*simperf-test-scale*) originalfilename=${files[-1]} mv $originalfilename $resultsdir/$1 - firesim terminaterunfarm -c workloads/simperf-test-scale-config.ini --terminatesomef116 $2 --terminatesomem416 $3 --forceterminate + firesim terminaterunfarm -c workloads/simperf-test-scale-config.yaml --terminatesomef116 $2 --terminatesomem416 $3 --forceterminate } diff --git a/deploy/workloads/run-workload.sh b/deploy/workloads/run-workload.sh index 6061a225..d1614007 100755 --- a/deploy/workloads/run-workload.sh +++ b/deploy/workloads/run-workload.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # This is some sugar around: -# ./firesim -c {launchrunfarm && infrasetup && runworkload && terminaterunfarm} +# ./firesim -c {launchrunfarm && infrasetup && runworkload && terminaterunfarm} # And thus will only work for workloads that do not need run other applications # between firesim calls @@ -11,10 +11,10 @@ terminate=1 function usage { - echo "usage: run-workload.sh [-H | -h | --help] [--noterminate] [--withlaunch]" - echo " workload.ini: the firesim-relative path to the workload you'd like to run" - echo " e.g. workloads/gapbs.ini" - echo " --withlaunch: (Optional) will spin up a runfarm based on the ini" + echo "usage: run-workload.sh [-H | -h | --help] [--noterminate] [--withlaunch]" + echo " workload.yaml: the firesim-relative path to the workload you'd like to run" + echo " e.g. workloads/gapbs.yaml" + echo " --withlaunch: (Optional) will spin up a runfarm based on the yaml" echo " --noterminate: (Optional) will not forcibly terminate runfarm instances after runworkload" } @@ -23,7 +23,7 @@ if [ $# -eq 0 -o "$1" == "--help" -o "$1" == "-h" -o "$1" == "-H" ]; then exit 3 fi -ini=$1 +yaml=$1 shift while test $# -gt 0 @@ -56,12 +56,12 @@ set -e set -o pipefail if [ "$withlaunch" -ne "0" ]; then - firesim -c $ini launchrunfarm + firesim -c $yaml launchrunfarm fi -firesim -c $ini infrasetup -firesim -c $ini runworkload +firesim -c $yaml infrasetup +firesim -c $yaml runworkload if [ "$terminate" -eq "1" ]; then - firesim -c $ini terminaterunfarm --forceterminate + firesim -c $yaml terminaterunfarm --forceterminate fi diff --git a/deploy/workloads/simperf-test-latency-config.ini b/deploy/workloads/simperf-test-latency-config.ini deleted file mode 100644 index a6a42073..00000000 --- a/deploy/workloads/simperf-test-latency-config.ini +++ /dev/null @@ -1,24 +0,0 @@ -[runfarm] -runfarmtag=simperftestlatency-mainrunfarm - -f1_16xlarges=1 -m4_16xlarges=0 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=example_8config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[workload] -workloadname=simperf-test-latency.json -terminateoncompletion=no diff --git a/deploy/workloads/simperf-test-latency-config.yaml b/deploy/workloads/simperf-test-latency-config.yaml new file mode 100644 index 00000000..924f3edb --- /dev/null +++ b/deploy/workloads/simperf-test-latency-config.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: simperftestlatency-mainrunfarm + f1_16xlarges: 1 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: example_8config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: simperf-test-latency.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/simperf-test-scale-config.ini b/deploy/workloads/simperf-test-scale-config.ini deleted file mode 100644 index c4a23ad8..00000000 --- a/deploy/workloads/simperf-test-scale-config.ini +++ /dev/null @@ -1,24 +0,0 @@ -[runfarm] -runfarmtag=simperftestscale-mainrunfarm - -f1_16xlarges=32 -m4_16xlarges=5 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=example_256config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 - -[workload] -workloadname=simperf-test-scale.json -terminateoncompletion=no diff --git a/deploy/workloads/simperf-test-scale-config.yaml b/deploy/workloads/simperf-test-scale-config.yaml new file mode 100644 index 00000000..63563920 --- /dev/null +++ b/deploy/workloads/simperf-test-scale-config.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: simperftestscale-mainrunfarm + f1_16xlarges: 32 + m4_16xlarges: 5 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: example_256config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-rocket-quadcore-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: simperf-test-scale.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/simperf-test-scale-supernode-config.ini b/deploy/workloads/simperf-test-scale-supernode-config.ini deleted file mode 100644 index 7f79bb1f..00000000 --- a/deploy/workloads/simperf-test-scale-supernode-config.ini +++ /dev/null @@ -1,24 +0,0 @@ -[runfarm] -runfarmtag=simperftestscalesupernode-mainrunfarm - -f1_16xlarges=32 -m4_16xlarges=5 -f1_2xlarges=0 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=supernode_example_1024config -no_net_num_nodes=2 -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-supernode-singlecore-nic-l2-lbp - -[workload] -workloadname=simperf-test-scale.json -terminateoncompletion=no diff --git a/deploy/workloads/simperf-test-scale-supernode-config.yaml b/deploy/workloads/simperf-test-scale-supernode-config.yaml new file mode 100644 index 00000000..be28095e --- /dev/null +++ b/deploy/workloads/simperf-test-scale-supernode-config.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: simperftestscalesupernode-mainrunfarm + f1_16xlarges: 32 + m4_16xlarges: 5 + f1_4xlarges: 0 + f1_2xlarges: 0 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: supernode_example_1024config + no-net-num-nodes: 2 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-supernode-singlecore-nic-l2-lbp + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: simperf-test-scale.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/spec17-intrate.ini b/deploy/workloads/spec17-intrate.ini deleted file mode 100644 index 903c1493..00000000 --- a/deploy/workloads/spec17-intrate.ini +++ /dev/null @@ -1,30 +0,0 @@ -[runfarm] -runfarmtag=intrate-runfarm - -f1_16xlarges=0 -m4_16xlarges=0 -f1_2xlarges=10 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=no_net_config -no_net_num_nodes=10 -# These are unused -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -defaulthwconfig=firesim-quadcore-no-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no -startcycle=0 -endcycle=-1 - -[workload] -workloadname=spec17-intrate.json -terminateoncompletion=yes diff --git a/deploy/workloads/spec17-intrate.yaml b/deploy/workloads/spec17-intrate.yaml new file mode 100644 index 00000000..86489e57 --- /dev/null +++ b/deploy/workloads/spec17-intrate.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: intrate-runfarm + f1_16xlarges: 0 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 10 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: no_net_config + no-net-num-nodes: 10 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-quadcore-no-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: spec17-intrate.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/spec17-intspeed.ini b/deploy/workloads/spec17-intspeed.ini deleted file mode 100644 index 07a079ac..00000000 --- a/deploy/workloads/spec17-intspeed.ini +++ /dev/null @@ -1,31 +0,0 @@ -[runfarm] -runfarmtag=intspeed-runfarm - -f1_16xlarges=0 -m4_16xlarges=0 -f1_2xlarges=11 - -runinstancemarket=ondemand -spotinterruptionbehavior=terminate -spotmaxprice=ondemand - -[targetconfig] -topology=no_net_config -no_net_num_nodes=11 -# These are unused -linklatency=6405 -switchinglatency=10 -netbandwidth=200 -profileinterval=-1 - -# Need not be single core. -defaulthwconfig=firesim-quadcore-no-nic-l2-llc4mb-ddr3 - -[tracing] -enable=no -startcyle=0 -endcycle=-1 - -[workload] -workloadname=spec17-intspeed.json -terminateoncompletion=yes diff --git a/deploy/workloads/spec17-intspeed.yaml b/deploy/workloads/spec17-intspeed.yaml new file mode 100644 index 00000000..b8611399 --- /dev/null +++ b/deploy/workloads/spec17-intspeed.yaml @@ -0,0 +1,42 @@ +run-farm: + run-farm-tag: intspeed-runfarm + f1_16xlarges: 0 + m4_16xlarges: 0 + f1_4xlarges: 0 + f1_2xlarges: 11 + run-instance-market: ondemand + spot-interruption-behavior: terminate + spot-max-price: ondemand + +target-config: + topology: no_net_config + no-net-num-nodes: 11 + link-latency: 6405 + switching-latency: 10 + net-bandwidth: 200 + profile-interval: -1 + default-hw-config: firesim-quadcore-no-nic-l2-llc4mb-ddr3 + +tracing: + enable: no + output-format: 0 + selector: 1 + start: 0 + end: -1 + +autocounter: + read-rate: 0 + +workload: + workload-name: spec17-intspeed.json + terminate-on-completion: no + suffix-tag: null + +host-debug: + zero-out-dram: no + disable-synth-asserts: no + +synth-print: + start: 0 + end: -1 + cycle-prefix: yes diff --git a/deploy/workloads/unittest/run-flash-stress.sh b/deploy/workloads/unittest/run-flash-stress.sh index 6492a48d..24d52d47 100755 --- a/deploy/workloads/unittest/run-flash-stress.sh +++ b/deploy/workloads/unittest/run-flash-stress.sh @@ -10,7 +10,7 @@ set -e set -o pipefail #if [ "$1" == "withlaunch" ]; then -# firesim launchrunfarm -c workloads/unittest/flash-stress-config.ini +# firesim launchrunfarm -c workloads/unittest/flash-stress-config.yaml #fi COUNTER=1 @@ -19,10 +19,10 @@ echo "start at" >> STRESSRUNS date >> STRESSRUNS while [ $COUNTER -gt 0 ]; do - firesim launchrunfarm -c workloads/unittest/flash-stress-config.ini - firesim infrasetup -c workloads/unittest/flash-stress-config.ini - firesim runworkload -c workloads/unittest/flash-stress-config.ini - firesim terminaterunfarm -c workloads/unittest/flash-stress-config.ini --forceterminate + firesim launchrunfarm -c workloads/unittest/flash-stress-config.yaml + firesim infrasetup -c workloads/unittest/flash-stress-config.yaml + firesim runworkload -c workloads/unittest/flash-stress-config.yaml + firesim terminaterunfarm -c workloads/unittest/flash-stress-config.yaml --forceterminate echo "done $COUNTER" echo "done $COUNTER" >> STRESSRUNS date >> STRESSRUNS