Fix CI bugs and small cleanup

.github/scripts/run-linux-poweroff-externally-provisioned.py

@@ -6,9 +6,13 @@ from fabric.api import *
 
 from common import manager_fsim_dir, set_fabric_firesim_pem
 from ci_variables import ci_workflow_run_id
+sys.path.append(ci_workdir + "/deploy/awstools")
+from awstools import get_instances_with_filter, get_private_ips_for_instances
+sys.path.append(ci_workdir + "/deploy/util")
+from util.filelineswap import file_line_swap
 
 def run_linux_poweroff_externally_provisioned():
-    """ Runs Linux poweroff workloads """
+    """ Runs Linux poweroff workloads using externally provisioned AWS run farm """
 
     with prefix(f"cd {manager_fsim_dir} && source sourceme-f1-manager.sh"):
 
@@ -41,30 +45,19 @@ def run_linux_poweroff_externally_provisioned():
                     instances = get_instances_with_filter(instances_filter, allowed_states=["running"])
                     instance_ips = get_private_ips_for_instances(instances)
 
-                    with open(f"{workload_full}", "r") as f:
-                        og_lines = f.readlines()
-
-                    runfarm_default_file = "sample-run-farm-recipes/externally_provisioned.yaml"
-                    start_lines = [f"  defaults: {runfarm_default_file}"]
-                    start_lines += "   override_args:"
-                    start_lines += "     default_num_fpgas: 1"
-                    start_lines += "     run_farm_hosts:"
+                    start_lines = [f"  defaults: sample-run-farm-recipes/externally_provisioned.yaml\n"]
+                    start_lines += ["   override_args:\n"]
+                    start_lines += ["     default_num_fpgas: 1\n"]
+                    start_lines += ["     run_farm_hosts:\n"]
                     for ip in instance_ips:
-                        start_lines += """       - "centos@{instance_ips}" """
+                        start_lines += ["""       - "centos@{ip}"\n"""]
 
-                    with open("/tmp/{workload}", "w") as f:
-                        write_og = True
-                        for og_line in og_lines:
-                            if "ci replace start" in og_line:
-                                write_og = False
-
-                            if write_og:
-                                f.write(og_line)
-                            else:
-                                f.writelines(rf_recipe_lines)
-
-                            if "ci replace end" in og_line:
-                                write_og = True
+                    file_line_swap(
+                            workload_full,
+                            f"/tmp/{workload}",
+                            "ci replace start",
+                            "ci replace end",
+                            start_lines)
 
                     # avoid logging excessive amounts to prevent GH-A masking secrets (which slows down log output)
                     # pty=False needed to avoid issues with screen -ls stalling in fabric

deploy/firesim

@@ -25,6 +25,7 @@ from buildtools.buildconfigfile import BuildConfigFile
 from buildtools.bitbuilder import F1BitBuilder
 
 from util.streamlogger import StreamLogger
+from util.filelineswap import file_line_swap
 
 from typing import Dict, Callable, Type, Optional, TypedDict, get_type_hints
 
@@ -133,29 +134,20 @@ def managerinit(args: argparse.Namespace):
 
     rootLogger.info("Adding default overrides to default runtime.yaml file")
     if args.platform == 'f1':
-        with open("config_runtime.yaml", "r") as f:
-            og_lines = f.readlines()
 
         runfarm_default_file = "sample-run-farm-recipes/aws_ec2.yaml"
         with open(runfarm_default_file, "r") as f:
             rf_recipe_lines = f.readlines()
-        start_lines = [f"defaults: {runfarm_default_file}"]
-        start_lines += "override_args:"
-        rf_recipe_lines = ["  " + l for l in start_lines] + ["    " + l for l in rf_recipe_lines]
+        start_lines = [f"defaults: {runfarm_default_file}\n"]
+        start_lines += ["override_args:\n"]
+        rf_recipe_lines = ["  " + l for l in start_lines] + rf_recipe_lines[2:]
 
-        with open("config_runtime.yaml", "w") as f:
-            write_og = True
-            for og_line in og_lines:
-                if "managerinit replace start" in og_line:
-                    write_og = False
-
-                if write_og:
-                    f.write(og_line)
-                else:
-                    f.writelines(rf_recipe_lines)
-
-                if "managerinit replace end" in og_line:
-                    write_og = True
+        file_line_swap(
+            "config_runtime.yaml",
+            "config_runtime.yaml",
+            "managerinit replace start",
+            "managerinit replace end",
+            rf_recipe_lines)
     else:
         rootLogger.info(f"Unknown platform {args.platform} for runtime.yaml setup. Skipping default overrides.")
 

deploy/util/filelineswap.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+# imports needed for python type checking
+from typing import List
+
+def file_line_swap(in_file: str, out_file: str, start_marker: str, end_marker: str, inject_lines: List[str]):
+    """Inject a set of lines into a file given two markers. Output into a new file.
+
+    Ex.
+        In:
+            Temp
+            # in mark
+            Temp2
+            # out mark
+            Temp3
+
+        Run w/ start_marker = "in" and out_marker = "out" and inject_lines = [Hi\n, I'm, Jeremy]
+
+        Out:
+            Temp
+            Hi
+            I'mJeremy
+            Temp3
+
+    Args:
+        in_file: input file to use as a base
+        out_file: output file to write into
+        start_marker: use this to determine start of replace area
+        end_marker: use this to determine end of replace area
+        inject_lines: lines to inject in the replacement area
+    """
+    with open(in_file, "r") as f:
+        og_lines = f.readlines()
+
+    with open(out_file, "w") as f:
+        write_og = True
+        written = False
+        for og_line in og_lines:
+            if start_marker in og_line:
+                write_og = False
+
+            if write_og:
+                f.write(og_line)
+            else:
+                if not written:
+                    f.writelines(inject_lines)
+                    written = True
+
+            if end_marker in og_line:
+                write_og = True