Add support for local CI | Re-add sudo | Other misc. fixes

.github/scripts/ci_variables.py

@@ -1,13 +1,14 @@
 import os
+from local_flags import RUN_LOCAL, local_fsim_dir
 
 # This package contains utilities that rely on environment variable
 # definitions present only on the CI container instance.
 
 # CI instance environment variables
 # This is used as a unique tag for all instances launched in a workflow
-ci_workflow_run_id = os.environ['GITHUB_RUN_ID']
-ci_commit_sha1 = os.environ['GITHUB_SHA']
+ci_workflow_run_id = os.environ['GITHUB_RUN_ID'] if not RUN_LOCAL else 0
+ci_commit_sha1 = os.environ['GITHUB_SHA'] if not RUN_LOCAL else 0
 # expanduser to replace the ~ present in the default, for portability
-ci_workdir = os.path.expanduser(os.environ['GITHUB_WORKSPACE'])
-ci_api_token = os.environ['GITHUB_TOKEN']
-ci_personal_api_token = os.environ['PERSONAL_ACCESS_TOKEN']
+ci_workdir = os.path.expanduser(os.environ['GITHUB_WORKSPACE']) if not RUN_LOCAL else local_fsim_dir
+ci_api_token = os.environ['GITHUB_TOKEN'] if not RUN_LOCAL else 0
+ci_personal_api_token = os.environ['PERSONAL_ACCESS_TOKEN'] if not RUN_LOCAL else 0

.github/scripts/common.py

@@ -3,16 +3,16 @@ import boto3
 import os
 from fabric.api import *
 import requests
+from local_flags import RUN_LOCAL, local_fsim_dir
 
 # Reuse manager utilities
-script_dir = os.path.dirname(os.path.realpath(__file__)) + "/../.."
-sys.path.append(script_dir + "/deploy/awstools")
+sys.path.append(local_fsim_dir + "/deploy/awstools")
 from awstools import get_instances_with_filter
 
 # Remote paths
 manager_home_dir = "/home/centos"
 manager_fsim_pem = manager_home_dir + "/firesim.pem"
-manager_fsim_dir = manager_home_dir + "/firesim"
+manager_fsim_dir = manager_home_dir + "/firesim" if not RUN_LOCAL else local_fsim_dir
 manager_marshal_dir = manager_fsim_dir + "/target-design/chipyard/software/firemarshal"
 manager_ci_dir = manager_fsim_dir + "/.github/scripts"
 

.github/scripts/local_flags.py

@@ -0,0 +1,4 @@
+import os
+
+RUN_LOCAL = False # change to True to run CI scripts locally
+local_fsim_dir = os.path.dirname(os.path.realpath(__file__)) + "/../.."

.github/scripts/run-linux-poweroff-externally-provisioned.py

@@ -26,7 +26,7 @@ def run_linux_poweroff_externally_provisioned():
             workload_full = workload_path + "/" + workload
             log_tail_length = 100
             # rename runfarm tag with a unique tag based on the ci workflow
-            with prefix(f"export FIRESIM_RUNFARM_PREFIX={ci_workflow_run_id}-2"):
+            with prefix(f"export FIRESIM_RUNFARM_PREFIX={ci_workflow_run_id}-ep"):
                 rc = 0
                 with settings(warn_only=True):
                     # do the following:
@@ -43,14 +43,14 @@ def run_linux_poweroff_externally_provisioned():
                             {'Name': 'tag:fsimcluster', 'Values': [f'{ci_workflow_run_id}-2*']},
                             ]
                     instances = get_instances_with_filter(instances_filter, allowed_states=["running"])
-                    instance_ips = get_private_ips_for_instances(instances)
+                    instance_ips = [instance['PrivateIpAddress'] for instance in instances]
 
                     start_lines = [f"  defaults: sample-run-farm-recipes/externally_provisioned.yaml\n"]
-                    start_lines += ["   override_args:\n"]
-                    start_lines += ["     default_num_fpgas: 1\n"]
-                    start_lines += ["     run_farm_hosts:\n"]
+                    start_lines += ["  override_args:\n"]
+                    start_lines += ["    default_num_fpgas: 1\n"]
+                    start_lines += ["    run_farm_hosts:\n"]
                     for ip in instance_ips:
-                        start_lines += ["""       - "centos@{ip}"\n"""]
+                        start_lines += [f"""      - "centos@{ip}"\n"""]
 
                     file_line_swap(
                             workload_full,
@@ -73,8 +73,8 @@ def run_linux_poweroff_externally_provisioned():
                     print(f"Printing last {log_tail_length} lines of all output files. See results-workload for more info.")
                     run(f"""cd deploy/results-workload/ && LAST_DIR=$(ls | tail -n1) && if [ -d "$LAST_DIR" ]; then tail -n{log_tail_length} $LAST_DIR/*/*; fi""")
 
-                # no matter what terminate using aws ec2 runfarm
-                run(f"firesim terminaterunfarm -q -c {workload_full}")
+                    # no matter what terminate using aws ec2 runfarm
+                    run(f"firesim terminaterunfarm -q -c {workload_full}")
 
                 if rc != 0:
                     print(f"Workload {workload} failed.")

.github/scripts/run-linux-poweroff.py

@@ -19,7 +19,7 @@ def run_linux_poweroff():
             """
             log_tail_length = 100
             # rename runfarm tag with a unique tag based on the ci workflow
-            with prefix(f"export FIRESIM_RUNFARM_PREFIX={ci_workflow_run_id}"):
+            with prefix(f"export FIRESIM_RUNFARM_PREFIX={ci_workflow_run_id}-default"):
                 rc = 0
                 with settings(warn_only=True):
                     # avoid logging excessive amounts to prevent GH-A masking secrets (which slows down log output)

deploy/runtools/runtime_config.py

@@ -194,7 +194,7 @@ class RuntimeHWConfig:
     def get_kill_simulation_command(self) -> str:
         driver = self.get_local_driver_binaryname()
         # Note that pkill only works for names <=15 characters
-        return """pkill -SIGKILL {driver}""".format(driver=driver[:15])
+        return """sudo pkill -SIGKILL {driver}""".format(driver=driver[:15])
 
     def build_fpga_driver(self) -> None:
         """ Build FPGA driver for running simulation """