Fix errors up until AFI starts building

deploy/awstools/awstools.py

@@ -22,6 +22,25 @@ rootLogger = logging.getLogger()
 # https://console.aws.amazon.com/ec2/v2/home?region=us-east-1#Images:visibility=public-images;search=FPGA%20Developer;sort=name
 f1_ami_name = "FPGA Developer AMI - 1.11.0-40257ab5-6688-4c95-97d1-e251a40fd1fc"
 
+def valid_aws_configure_creds():
+    """ See if aws configure has been run. Returns False if aws configure
+    needs to be run, else True.
+
+    This DOES NOT perform any deeper validation.
+    """
+    import botocore.session
+    session = botocore.session.get_session()
+    creds = session.get_credentials()
+    if creds is None:
+        return False
+    if session.get_credentials().access_key == '':
+        return False
+    if session.get_credentials().secret_key == '':
+        return False
+    if session.get_config_variable('region') == '':
+        return False
+    return True
+
 def aws_resource_names():
     """ Get names for various aws resources the manager relies on. For example:
     vpcname, securitygroupname, keyname, etc.

deploy/buildtools/buildafi.py

@@ -59,7 +59,7 @@ def build_driver(build_config):
          prefix('cd sim/'), \
          InfoStreamLogger('stdout'), \
          InfoStreamLogger('stderr'):
-        run(buildconfig.make_recipe("PLATFORM=f1 driver"))
+        run(build_config.make_recipe("PLATFORM=f1 driver"))
 
 def remote_setup(build_config):
     """ Setup CL_DIR on remote machine

deploy/buildtools/buildconfig.py

@@ -59,13 +59,13 @@ class BuildConfig:
         build_farm_host_type = build_farm_host_conf_dict["build-farm-type"]
         build_farm_host_args = build_farm_host_conf_dict["args"]
 
-        build_farm_host_dispatch_dict = dict([(x.NAME, x.__name__) for x in inheritors(BuildHostDispatcher)])
+        build_farm_host_dispatch_dict = dict([(x.NAME, x.__name__) for x in inheritors(BuildFarmHostDispatcher)])
 
         build_farm_host_dispatcher_class_name = build_farm_host_dispatch_dict[build_farm_host_type]
 
         # create dispatcher object using class given and pass args to it
         self.build_farm_host_dispatcher = getattr(
-            import_module("buildtools.buildhostdispatcher"),
+            import_module("buildtools.buildfarmhostdispatcher"),
             build_farm_host_dispatcher_class_name)(self, build_farm_host_args)
 
         self.build_farm_host_dispatcher.parse_args()

deploy/buildtools/buildconfigfile.py

@@ -11,6 +11,7 @@ from importlib import import_module
 
 from runtools.runtime_config import RuntimeHWDB
 from buildtools.buildconfig import BuildConfig
+from awstools.awstools import auto_create_bucket, get_snsname_arn
 
 rootLogger = logging.getLogger()
 
@@ -103,12 +104,14 @@ class BuildConfigFile:
                     sys.exit(1)
 
     def wait_on_build_farm_host_initializations(self):
-        """ Block until all build instances are launched """
+        """ Block until all build instances are initialized """
+        # TODO: batch optimize
         for build in self.builds_list:
             build.build_farm_host_dispatcher.wait_on_build_farm_host_initialization()
 
     def release_build_farm_hosts(self):
         """ Terminate all build instances that are launched """
+        # TODO: batch optimize
         for build in self.builds_list:
             build.build_farm_host_dispatcher.release_build_farm_host()
 

deploy/buildtools/buildfarmhostdispatcher.py

@@ -186,17 +186,32 @@ class EC2BuildFarmHostDispatcher(BuildFarmHostDispatcher):
 
     @staticmethod
     def request_build_farm_hosts(build_farm_host_dispatchers):
-        amt_requested = len(build_farm_host_dispatchers)
+        # TODO: this can be further optimized (spawn-like build farm hosts)
+        # double-check that build farms are the same
+        it_len = len(set([x.instance_type for x in build_farm_host_dispatchers]))
+        bim_len = len(set([x.build_instance_market for x in build_farm_host_dispatchers]))
+        sib_len = len(set([x.spot_interruption_behavior for x in build_farm_host_dispatchers]))
+        smp_len = len(set([x.spot_max_price for x in build_farm_host_dispatchers]))
+        if it_len == 1 and bim_len == 1 and sib_len == 1 and smp_len == 1: 
+            amt_requested = len(build_farm_host_dispatchers)
+            instance_type = build_farm_host_dispatchers[0].instance_type
+            build_instance_market = build_farm_host_dispatchers[0].build_instance_market
+            spot_interruption_behavior = build_farm_host_dispatchers[0].spot_interruption_behavior
+            spot_max_price = build_farm_host_dispatchers[0].spot_max_price
 
-        launched_objs = EC2BuildFarmHostDispatcher.ec2_launch_instances(
-            instance_type,
-            amt_requested,
-            build_instance_market,
-            spot_interruption_behavior,
-            spot_max_price)
+            launched_objs = EC2BuildFarmHostDispatcher.ec2_launch_instances(
+                instance_type,
+                amt_requested,
+                build_instance_market,
+                spot_interruption_behavior,
+                spot_max_price)
 
-        for bh, lo in zip(build_farm_host_dispatchers, launched_objs):
-            bh.launched_instance_object = lo
+            for bh, lo in zip(build_farm_host_dispatchers, launched_objs):
+                bh.launched_instance_object = lo
+        else:
+            # default to the original request function
+            for build_farm_host_dispatcher in build_farm_host_dispatchers:
+                 build_farm_host_dispatchers.request_build_farm_host()
 
     @staticmethod
     def ec2_launch_instances(inst_type, num_insts, build_inst_market, spot_int_behav, spot_max_price):
@@ -229,7 +244,7 @@ class EC2BuildFarmHostDispatcher(BuildFarmHostDispatcher):
 
     def wait_on_build_farm_host_initialization(self):
         """ Wait for EC2 instance launch. """
-        wait_on_build_farm_host_initializationes([self.launched_instance_object])
+        wait_on_instance_launches([self.launched_instance_object])
 
     def get_build_farm_host_ip(self):
         """ Get IP address associated with this dispatched instance.

deploy/firesim

@@ -59,7 +59,7 @@ def managerinit():
             m = local("""cp sample-backup-configs/sample_config_{}.yaml config_{}.yaml""".format(conf_file, conf_file), capture=True)
             rootLogger.debug(m)
             rootLogger.debug(m.stderr)
-            m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.yaml""".format("DUMMY", conf_file), capture=True)
+            m = local("""sed -i 's/AWSUSERNAME/{}/g' config_{}.yaml""".format(get_aws_userid(), conf_file), capture=True)
             rootLogger.debug(m)
             rootLogger.debug(m.stderr)
 
@@ -93,8 +93,8 @@ def buildafi(globalbuildconf):
 
     # forced to build locally
     for build_config in globalbuildconf.get_builds_list():
-        execute(replace_rtl, hosts=['localhost'])
-        execute(build_driver, hosts=['localhost'])
+        execute(replace_rtl, build_config, hosts=['localhost'])
+        execute(build_driver, build_config, hosts=['localhost'])
 
     def release_build_farm_hosts_handler(sig, frame):
         """ Handler that prompts to release build farm hosts if you press ctrl-c. """

deploy/runtools/runtime_config.py

@@ -6,6 +6,7 @@ from __future__ import print_function
 from time import strftime, gmtime
 import pprint
 import logging
+import yaml
 
 from fabric.api import *
 from awstools.awstools import *

deploy/sample-backup-configs/sample_config_build_farm.yaml

@@ -19,6 +19,6 @@ local-build-farm:
     args:
         default-build-dir: null
         build-farm-hosts:
-            - localhost
+            - localhost:
                 # change location of build on build host (abs. path)
                 override-build-dir: null