Cleanup workflow.yaml | Build all xclbins

.github/scripts/run-xclbin-buildbitstream.py

@@ -51,94 +51,108 @@ def run_xclbin_buildbitstream():
 
     # repo should already be checked out
 
-    relative_hwdb_path = f"deploy/sample-backup-configs/sample_config_hwdb.yaml"
-
     manager_fsim_dir = ci_env['REMOTE_WORK_DIR']
     with prefix(f"cd {manager_fsim_dir}"):
         with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
-            # modify config_build.yaml (uncomment only a single vitis bitstream)
-            build_yaml = f"{manager_fsim_dir}/deploy/config_build.yaml"
-            build_yaml_lines = open(build_yaml).read().split("\n")
-            with open(build_yaml, "w") as byf:
-                for line in build_yaml_lines:
-                    if "- firesim" in line:
-                        # comment out AWS specific lines
-                        byf.write("# " + line + '\n')
-                    elif "- vitis_firesim_rocket" in line:
-                        # remove comment on vitis line
-                        byf.write(line.replace("# ", '') + '\n')
-                    elif 'default_build_dir:' in line:
-                        byf.write(line.replace('null', f"{manager_fsim_dir}/tmpbuildarea") + '\n')
-                    else:
-                        byf.write(line + '\n')
 
-            print(f"Printing {build_yaml}...")
+            # return a copy of config_build.yaml w/ hwdb entry uncommented + new build dir
-            run(f"cat {build_yaml}")
+            def modify_config_build(hwdb_entry_to_gen: str) -> str:
+                build_yaml = f"{manager_fsim_dir}/deploy/config_build.yaml"
+                copy_build_yaml = f"{manager_fsim_dir}/deploy/config_build_{hwdb_entry_to_gen}.yaml"
+                build_yaml_lines = open(build_yaml).read().split("\n")
+                with open(copy_build_yaml, "w") as byf:
+                    for line in build_yaml_lines:
+                        if "- firesim" in line:
+                            # comment out AWS specific lines
+                            byf.write("# " + line + '\n')
+                        elif "- {hwdb_entry_to_gen}" in line:
+                            # remove comment
+                            byf.write(line.replace("# ", '') + '\n')
+                        elif 'default_build_dir:' in line:
+                            byf.write(line.replace('null', f"{manager_fsim_dir}/tmp_build_area") + '\n')
+                        else:
+                            byf.write(line + '\n')
+                return copy_build_yaml
 
-            rc = 0
+            def build_upload(build_yaml: str, hwdb_entry_name: str) -> str:
-            with settings(warn_only=True):
+
-                # pty=False needed to avoid issues with screen -ls stalling in fabric
+                print(f"Printing {build_yaml}...")
-                build_result = run("timeout 10h firesim buildbitstream --forceterminate", pty=False)
+                run(f"cat {build_yaml}")
-                rc = build_result.return_code
+
+                rc = 0
+                with settings(warn_only=True):
+                    # pty=False needed to avoid issues with screen -ls stalling in fabric
+                    build_result = run(f"timeout 10h firesim buildbitstream -b {build_yaml} --forceterminate", pty=False)
+                    rc = build_result.return_code
+
+                if rc != 0:
+                    log_lines = 200
+                    print(f"Buildbitstream failed. Printing {log_lines} of last log file:")
+                    run(f"""LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""")
+                    sys.exit(rc)
 
-            if rc != 0:
-                log_lines = 200
-                print(f"Buildbitstream failed. Printing {log_lines} of last log file:")
-                run(f"""LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""")
-                sys.exit(rc)
-            else:
                 hwdb_entry_dir = f"{manager_fsim_dir}/deploy/built-hwdb-entries"
-                built_hwdb_entries = [x for x in os.listdir(hwdb_entry_dir) if os.path.isfile(os.path.join(hwdb_entry_dir, x))]
+                hwdb_entry = f"{hwdb_entry_dir}/{hwdb_entry_name}"
 
-                hwdb_to_link = {}
+                print(f"Printing {hwdb_entry}...")
-                for hwdb in built_hwdb_entries:
+                run(f"cat {hwdb_entry}")
-                    print(f"Printing {hwdb}")
-                    run(f"cat {hwdb_entry_dir}/{hwdb}")
 
-                    with open(f"{hwdb_entry_dir}/{hwdb}") as hwdbef:
+                with open(hwdb_entry, 'r') as hwdbef:
-                        lines = hwdbef.readlines()
+                    lines = hwdbef.readlines()
-                        for line in lines:
+                    for line in lines:
-                            if "xclbin:" in line:
+                        if "xclbin:" in line:
-                                file_path = Path(line.strip().split(' ')[1]) # 2nd element
+                            file_path = Path(line.strip().split(' ')[1]) # 2nd element (i.e. the path)
-                                file_name = f"vitis/{hwdb}.xclbin"
+                            file_name = f"vitis/{hwdb_entry_name}.xclbin"
-                                sha = upload_file(file_path, file_name)
+                            sha = upload_file(file_path, file_name)
-                                link = f"{URL_PREFIX}/{sha}/{file_name}"
+                            link = f"{URL_PREFIX}/{sha}/{file_name}"
-                                print(f"Uploaded xclbin for {hwdb} to {link}")
+                            print(f"Uploaded xclbin for {hwdb_entry_name} to {link}")
-                                hwdb_to_link[hwdb] = link
+                            return link
 
-                # parse the output yamls, replace the sample hwdb's xclbin line only
+                sys.exit(":ERROR: Something went wrong. Should have uploaded by now and returned a link.")
-                sample_hwdb_filename = f"{manager_fsim_dir}/{relative_hwdb_path}"
-                for hwdb in built_hwdb_entries:
-                    sample_hwdb_lines = open(sample_hwdb_filename).read().split('\n')
 
-                    with open(sample_hwdb_filename, "w") as sample_hwdb_file:
+            relative_hwdb_path = "deploy/sample-backup-configs/sample_config_hwdb.yaml"
-                        match_xclbin = False
+            sample_hwdb_filename = f"{manager_fsim_dir}/{relative_hwdb_path}"
-                        for line in sample_hwdb_lines:
-                            if hwdb in line.strip().split(' ')[0].replace(':', ''):
-                                # hwdb entry matches key name
-                                match_xclbin = True
-                                sample_hwdb_file.write(line + '\n')
-                            elif match_xclbin == True and ("xclbin:" in line.strip().split(' ')[0]):
-                                # only replace this xclbin
-                                match_xclbin = False
 
-                                new_xclbin_line = f"    xclbin: {hwdb_to_link[hwdb]}"
+            def replace_in_hwdb(hwdb_entry_name: str, link: str) -> None:
-                                print(f"Replacing {line.strip()} with {new_xclbin_line}")
+                # replace the sample hwdb's xclbin line only
+                sample_hwdb_lines = open(sample_hwdb_filename).read().split('\n')
 
-                                # print out the xclbin line
+                with open(sample_hwdb_filename, "w") as sample_hwdb_file:
-                                sample_hwdb_file.write(new_xclbin_line + '\n')
+                    match_xclbin = False
-                            else:
+                    for line in sample_hwdb_lines:
-                                # if no match print other lines
+                        if hwdb_entry_name in line.strip().split(' ')[0].replace(':', ''):
-                                sample_hwdb_file.write(line + '\n')
+                            # hwdb entry matches key name
+                            match_xclbin = True
+                            sample_hwdb_file.write(line + '\n')
+                        elif match_xclbin == True and ("xclbin:" in line.strip().split(' ')[0]):
+                            # only replace this xclbin
+                            match_xclbin = False
 
-                        if match_xclbin == True:
+                            new_xclbin_line = f"    xclbin: {link}"
-                            sys.exit("::ERROR:: Unable to find matching xclbin key for HWDB entry")
+                            print(f"Replacing {line.strip()} with {new_xclbin_line}")
 
-                print(f"Printing {sample_hwdb_filename}...")
+                            # print out the xclbin line
-                run(f"cat {sample_hwdb_filename}")
+                            sample_hwdb_file.write(new_xclbin_line + '\n')
+                        else:
+                            # if no match print other lines
+                            sample_hwdb_file.write(line + '\n')
 
-                # copy back to workspace area so you can PR it
+                    if match_xclbin == True:
-                run(f"cp -f {sample_hwdb_filename} {ci_env['GITHUB_WORKSPACE']}/{relative_hwdb_path}")
+                        sys.exit(f"::ERROR:: Unable to replace URL for {hwdb_entry_name} in {sample_hwdb_filename}")
+
+            hwdb_entry_name = "vitis_firesim_rocket_singlecore_no_nic"
+            copy_build_yaml = modify_config_build(hwdb_entry_name)
+            replace_in_hwdb(hwdb_entry_name, build_upload(copy_build_yaml, hwdb_entry_name))
+
+            hwdb_entry_name = "vitis_firesim_gemmini_rocket_singlecore_no_nic"
+            copy_build_yaml = modify_config_build(hwdb_entry_name)
+            replace_in_hwdb(hwdb_entry_name, build_upload(copy_build_yaml, hwdb_entry_name))
+
+            print(f"Printing {sample_hwdb_filename}...")
+            run(f"cat {sample_hwdb_filename}")
+
+            # copy back to workspace area so you can PR it
+            run(f"cp -f {sample_hwdb_filename} {manager_fsim_dir}/{relative_hwdb_path}")
 
 if __name__ == "__main__":
     execute(run_xclbin_buildbitstream, hosts=["localhost"])

.github/workflows/firesim-run-tests.yml

@@ -30,6 +30,7 @@ env:
   CI_LABEL_DEBUG: ${{ contains(github.event.pull_request.labels.*.name, 'ci:debug') }}
   CI_LABEL_PERSIST: ${{ contains(github.event.pull_request.labels.*.name, 'ci:persist-prior-workflows') }}
   REMOTE_WORK_DIR: /scratch/buildbot/fs-shared/fs-${{ github.sha }}
+  TERM: xterm-256-color
 
 jobs:
   cancel-prior-workflows:
@@ -156,8 +157,6 @@ jobs:
   #  name: build-default-workloads
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Build default workloads (FireMarshal and paper workloads)
@@ -167,8 +166,6 @@ jobs:
   #  name: run-manager-pytests
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run pytests
@@ -178,8 +175,6 @@ jobs:
   #  name: run-manager-python-typecheck
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run manager Python type checker
@@ -189,8 +184,6 @@ jobs:
   #    name: run-ci-python-typecheck
   #    needs: [setup-manager]
   #    runs-on: aws-${{ github.run_id }}
-  #    env:
-  #      TERM: xterm-256-color
   #    steps:
   #    - uses: actions/checkout@v3
   #    - name: Run CI Python type checker
@@ -200,8 +193,6 @@ jobs:
   #  name: run-scalafmt-check
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run Scalafmt on FireSim Scala main sources
@@ -211,8 +202,6 @@ jobs:
   #  name: run-check-docs-generated-components
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Check docs components that require manual re-generation (e.g. config_runtime.yaml example, help output)
@@ -222,8 +211,6 @@ jobs:
   #  name: build-f1-driver
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Runs compilation of F1 driver for the make-default tuple
@@ -233,8 +220,6 @@ jobs:
   #  name: run-targetutils-scala-tests
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run targetutils scala tests
@@ -244,8 +229,6 @@ jobs:
   #  name: run-midas-scala-tests
   #  needs: [setup-manager]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run targetutils scala tests
@@ -255,8 +238,6 @@ jobs:
   #  name: run-test-groupA
   #  needs: [build-f1-driver]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run CIGroupA Scala tests
@@ -268,8 +249,6 @@ jobs:
   #  name: run-test-groupB
   #  needs: [run-test-groupA]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run CIGroupB Scala tests
@@ -281,8 +260,6 @@ jobs:
   #  name: run-test-firesim-lib
   #  needs: [run-test-groupB]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run firesim-lib Scala tests
@@ -296,8 +273,6 @@ jobs:
   #  name: run-chipyard-tests
   #  needs: [run-test-firesim-lib]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run other (CITests) Scala tests
@@ -313,8 +288,6 @@ jobs:
   #  # this behind the scalatest train. Remove once we're off SBT.
   #  needs: [run-chipyard-tests]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run parallel Verilator metasimulation tests (deploy on localhost and on AWS instances)
@@ -327,8 +300,6 @@ jobs:
   #  # this behind the scalatest train. Remove once we're off SBT.
   #  needs: [build-default-workloads, run-chipyard-tests, run-parallel-verilator-metasims]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run linux-poweroff test w/ AWS EC2 run farm
@@ -339,8 +310,6 @@ jobs:
   #  name: run-basic-linux-poweroff-externally-provisioned
   #  needs: [build-default-workloads]
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run linux-poweroff test w/ externally provisioned (AWS EC2) run farm
@@ -351,8 +320,6 @@ jobs:
   #  name: run-agfi-buildbitstream
   #  needs: [build-f1-driver] # delay until known working scala compile
   #  runs-on: aws-${{ github.run_id }}
-  #  env:
-  #    TERM: xterm-256-color
   #  steps:
   #    - uses: actions/checkout@v3
   #    - name: Run buildbitstream command and update sample AGFIs
@@ -361,7 +328,7 @@ jobs:
   #      with:
   #        add-paths: |
   #          'deploy/sample-backup-configs/sample_config_hwdb.yaml'
-  #        commit-message: "Update AGFI(s)"
+  #        commit-message: "Update AGFI(s) [ci skip]"
   #        branch-suffix: short-commit-hash
   #        title: "Update AGFI(s) for ${{ github.ref_name }}"
 
@@ -525,7 +492,7 @@ jobs:
         with:
           add-paths: |
             'deploy/sample-backup-configs/sample_config_hwdb.yaml'
-          commit-message: "Update xclbin(s)"
+          commit-message: "Update xclbin(s) [ci skip]"
           branch-suffix: short-commit-hash
           title: "Update xclbin(s) for ${{ github.ref_name }}"