Cleanup workflow.yaml | Build all xclbins
This commit is contained in:
parent
ccb4cb3e4c
commit
dfdc1aba9f
|
@ -51,26 +51,30 @@ def run_xclbin_buildbitstream():
|
|||
|
||||
# repo should already be checked out
|
||||
|
||||
relative_hwdb_path = f"deploy/sample-backup-configs/sample_config_hwdb.yaml"
|
||||
|
||||
manager_fsim_dir = ci_env['REMOTE_WORK_DIR']
|
||||
with prefix(f"cd {manager_fsim_dir}"):
|
||||
with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
|
||||
# modify config_build.yaml (uncomment only a single vitis bitstream)
|
||||
|
||||
# return a copy of config_build.yaml w/ hwdb entry uncommented + new build dir
|
||||
def modify_config_build(hwdb_entry_to_gen: str) -> str:
|
||||
build_yaml = f"{manager_fsim_dir}/deploy/config_build.yaml"
|
||||
copy_build_yaml = f"{manager_fsim_dir}/deploy/config_build_{hwdb_entry_to_gen}.yaml"
|
||||
build_yaml_lines = open(build_yaml).read().split("\n")
|
||||
with open(build_yaml, "w") as byf:
|
||||
with open(copy_build_yaml, "w") as byf:
|
||||
for line in build_yaml_lines:
|
||||
if "- firesim" in line:
|
||||
# comment out AWS specific lines
|
||||
byf.write("# " + line + '\n')
|
||||
elif "- vitis_firesim_rocket" in line:
|
||||
# remove comment on vitis line
|
||||
elif "- {hwdb_entry_to_gen}" in line:
|
||||
# remove comment
|
||||
byf.write(line.replace("# ", '') + '\n')
|
||||
elif 'default_build_dir:' in line:
|
||||
byf.write(line.replace('null', f"{manager_fsim_dir}/tmpbuildarea") + '\n')
|
||||
byf.write(line.replace('null', f"{manager_fsim_dir}/tmp_build_area") + '\n')
|
||||
else:
|
||||
byf.write(line + '\n')
|
||||
return copy_build_yaml
|
||||
|
||||
def build_upload(build_yaml: str, hwdb_entry_name: str) -> str:
|
||||
|
||||
print(f"Printing {build_yaml}...")
|
||||
run(f"cat {build_yaml}")
|
||||
|
@ -78,7 +82,7 @@ def run_xclbin_buildbitstream():
|
|||
rc = 0
|
||||
with settings(warn_only=True):
|
||||
# pty=False needed to avoid issues with screen -ls stalling in fabric
|
||||
build_result = run("timeout 10h firesim buildbitstream --forceterminate", pty=False)
|
||||
build_result = run(f"timeout 10h firesim buildbitstream -b {build_yaml} --forceterminate", pty=False)
|
||||
rc = build_result.return_code
|
||||
|
||||
if rc != 0:
|
||||
|
@ -86,35 +90,37 @@ def run_xclbin_buildbitstream():
|
|||
print(f"Buildbitstream failed. Printing {log_lines} of last log file:")
|
||||
run(f"""LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""")
|
||||
sys.exit(rc)
|
||||
else:
|
||||
|
||||
hwdb_entry_dir = f"{manager_fsim_dir}/deploy/built-hwdb-entries"
|
||||
built_hwdb_entries = [x for x in os.listdir(hwdb_entry_dir) if os.path.isfile(os.path.join(hwdb_entry_dir, x))]
|
||||
hwdb_entry = f"{hwdb_entry_dir}/{hwdb_entry_name}"
|
||||
|
||||
hwdb_to_link = {}
|
||||
for hwdb in built_hwdb_entries:
|
||||
print(f"Printing {hwdb}")
|
||||
run(f"cat {hwdb_entry_dir}/{hwdb}")
|
||||
print(f"Printing {hwdb_entry}...")
|
||||
run(f"cat {hwdb_entry}")
|
||||
|
||||
with open(f"{hwdb_entry_dir}/{hwdb}") as hwdbef:
|
||||
with open(hwdb_entry, 'r') as hwdbef:
|
||||
lines = hwdbef.readlines()
|
||||
for line in lines:
|
||||
if "xclbin:" in line:
|
||||
file_path = Path(line.strip().split(' ')[1]) # 2nd element
|
||||
file_name = f"vitis/{hwdb}.xclbin"
|
||||
file_path = Path(line.strip().split(' ')[1]) # 2nd element (i.e. the path)
|
||||
file_name = f"vitis/{hwdb_entry_name}.xclbin"
|
||||
sha = upload_file(file_path, file_name)
|
||||
link = f"{URL_PREFIX}/{sha}/{file_name}"
|
||||
print(f"Uploaded xclbin for {hwdb} to {link}")
|
||||
hwdb_to_link[hwdb] = link
|
||||
print(f"Uploaded xclbin for {hwdb_entry_name} to {link}")
|
||||
return link
|
||||
|
||||
# parse the output yamls, replace the sample hwdb's xclbin line only
|
||||
sys.exit(":ERROR: Something went wrong. Should have uploaded by now and returned a link.")
|
||||
|
||||
relative_hwdb_path = "deploy/sample-backup-configs/sample_config_hwdb.yaml"
|
||||
sample_hwdb_filename = f"{manager_fsim_dir}/{relative_hwdb_path}"
|
||||
for hwdb in built_hwdb_entries:
|
||||
|
||||
def replace_in_hwdb(hwdb_entry_name: str, link: str) -> None:
|
||||
# replace the sample hwdb's xclbin line only
|
||||
sample_hwdb_lines = open(sample_hwdb_filename).read().split('\n')
|
||||
|
||||
with open(sample_hwdb_filename, "w") as sample_hwdb_file:
|
||||
match_xclbin = False
|
||||
for line in sample_hwdb_lines:
|
||||
if hwdb in line.strip().split(' ')[0].replace(':', ''):
|
||||
if hwdb_entry_name in line.strip().split(' ')[0].replace(':', ''):
|
||||
# hwdb entry matches key name
|
||||
match_xclbin = True
|
||||
sample_hwdb_file.write(line + '\n')
|
||||
|
@ -122,7 +128,7 @@ def run_xclbin_buildbitstream():
|
|||
# only replace this xclbin
|
||||
match_xclbin = False
|
||||
|
||||
new_xclbin_line = f" xclbin: {hwdb_to_link[hwdb]}"
|
||||
new_xclbin_line = f" xclbin: {link}"
|
||||
print(f"Replacing {line.strip()} with {new_xclbin_line}")
|
||||
|
||||
# print out the xclbin line
|
||||
|
@ -132,13 +138,21 @@ def run_xclbin_buildbitstream():
|
|||
sample_hwdb_file.write(line + '\n')
|
||||
|
||||
if match_xclbin == True:
|
||||
sys.exit("::ERROR:: Unable to find matching xclbin key for HWDB entry")
|
||||
sys.exit(f"::ERROR:: Unable to replace URL for {hwdb_entry_name} in {sample_hwdb_filename}")
|
||||
|
||||
hwdb_entry_name = "vitis_firesim_rocket_singlecore_no_nic"
|
||||
copy_build_yaml = modify_config_build(hwdb_entry_name)
|
||||
replace_in_hwdb(hwdb_entry_name, build_upload(copy_build_yaml, hwdb_entry_name))
|
||||
|
||||
hwdb_entry_name = "vitis_firesim_gemmini_rocket_singlecore_no_nic"
|
||||
copy_build_yaml = modify_config_build(hwdb_entry_name)
|
||||
replace_in_hwdb(hwdb_entry_name, build_upload(copy_build_yaml, hwdb_entry_name))
|
||||
|
||||
print(f"Printing {sample_hwdb_filename}...")
|
||||
run(f"cat {sample_hwdb_filename}")
|
||||
|
||||
# copy back to workspace area so you can PR it
|
||||
run(f"cp -f {sample_hwdb_filename} {ci_env['GITHUB_WORKSPACE']}/{relative_hwdb_path}")
|
||||
run(f"cp -f {sample_hwdb_filename} {manager_fsim_dir}/{relative_hwdb_path}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
execute(run_xclbin_buildbitstream, hosts=["localhost"])
|
||||
|
|
|
@ -30,6 +30,7 @@ env:
|
|||
CI_LABEL_DEBUG: ${{ contains(github.event.pull_request.labels.*.name, 'ci:debug') }}
|
||||
CI_LABEL_PERSIST: ${{ contains(github.event.pull_request.labels.*.name, 'ci:persist-prior-workflows') }}
|
||||
REMOTE_WORK_DIR: /scratch/buildbot/fs-shared/fs-${{ github.sha }}
|
||||
TERM: xterm-256-color
|
||||
|
||||
jobs:
|
||||
cancel-prior-workflows:
|
||||
|
@ -156,8 +157,6 @@ jobs:
|
|||
# name: build-default-workloads
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Build default workloads (FireMarshal and paper workloads)
|
||||
|
@ -167,8 +166,6 @@ jobs:
|
|||
# name: run-manager-pytests
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run pytests
|
||||
|
@ -178,8 +175,6 @@ jobs:
|
|||
# name: run-manager-python-typecheck
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run manager Python type checker
|
||||
|
@ -189,8 +184,6 @@ jobs:
|
|||
# name: run-ci-python-typecheck
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run CI Python type checker
|
||||
|
@ -200,8 +193,6 @@ jobs:
|
|||
# name: run-scalafmt-check
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run Scalafmt on FireSim Scala main sources
|
||||
|
@ -211,8 +202,6 @@ jobs:
|
|||
# name: run-check-docs-generated-components
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Check docs components that require manual re-generation (e.g. config_runtime.yaml example, help output)
|
||||
|
@ -222,8 +211,6 @@ jobs:
|
|||
# name: build-f1-driver
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Runs compilation of F1 driver for the make-default tuple
|
||||
|
@ -233,8 +220,6 @@ jobs:
|
|||
# name: run-targetutils-scala-tests
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run targetutils scala tests
|
||||
|
@ -244,8 +229,6 @@ jobs:
|
|||
# name: run-midas-scala-tests
|
||||
# needs: [setup-manager]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run targetutils scala tests
|
||||
|
@ -255,8 +238,6 @@ jobs:
|
|||
# name: run-test-groupA
|
||||
# needs: [build-f1-driver]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run CIGroupA Scala tests
|
||||
|
@ -268,8 +249,6 @@ jobs:
|
|||
# name: run-test-groupB
|
||||
# needs: [run-test-groupA]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run CIGroupB Scala tests
|
||||
|
@ -281,8 +260,6 @@ jobs:
|
|||
# name: run-test-firesim-lib
|
||||
# needs: [run-test-groupB]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run firesim-lib Scala tests
|
||||
|
@ -296,8 +273,6 @@ jobs:
|
|||
# name: run-chipyard-tests
|
||||
# needs: [run-test-firesim-lib]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run other (CITests) Scala tests
|
||||
|
@ -313,8 +288,6 @@ jobs:
|
|||
# # this behind the scalatest train. Remove once we're off SBT.
|
||||
# needs: [run-chipyard-tests]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run parallel Verilator metasimulation tests (deploy on localhost and on AWS instances)
|
||||
|
@ -327,8 +300,6 @@ jobs:
|
|||
# # this behind the scalatest train. Remove once we're off SBT.
|
||||
# needs: [build-default-workloads, run-chipyard-tests, run-parallel-verilator-metasims]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run linux-poweroff test w/ AWS EC2 run farm
|
||||
|
@ -339,8 +310,6 @@ jobs:
|
|||
# name: run-basic-linux-poweroff-externally-provisioned
|
||||
# needs: [build-default-workloads]
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run linux-poweroff test w/ externally provisioned (AWS EC2) run farm
|
||||
|
@ -351,8 +320,6 @@ jobs:
|
|||
# name: run-agfi-buildbitstream
|
||||
# needs: [build-f1-driver] # delay until known working scala compile
|
||||
# runs-on: aws-${{ github.run_id }}
|
||||
# env:
|
||||
# TERM: xterm-256-color
|
||||
# steps:
|
||||
# - uses: actions/checkout@v3
|
||||
# - name: Run buildbitstream command and update sample AGFIs
|
||||
|
@ -361,7 +328,7 @@ jobs:
|
|||
# with:
|
||||
# add-paths: |
|
||||
# 'deploy/sample-backup-configs/sample_config_hwdb.yaml'
|
||||
# commit-message: "Update AGFI(s)"
|
||||
# commit-message: "Update AGFI(s) [ci skip]"
|
||||
# branch-suffix: short-commit-hash
|
||||
# title: "Update AGFI(s) for ${{ github.ref_name }}"
|
||||
|
||||
|
@ -525,7 +492,7 @@ jobs:
|
|||
with:
|
||||
add-paths: |
|
||||
'deploy/sample-backup-configs/sample_config_hwdb.yaml'
|
||||
commit-message: "Update xclbin(s)"
|
||||
commit-message: "Update xclbin(s) [ci skip]"
|
||||
branch-suffix: short-commit-hash
|
||||
title: "Update xclbin(s) for ${{ github.ref_name }}"
|
||||
|
||||
|
|
Loading…
Reference in New Issue