replica
/
firesim
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'og/main' into ntnu-extended

This commit is contained in:
abejgonzalez 2023-05-15 14:47:18 -07:00
parent 185bdeac9b 3fe38fe0bc
commit 2decc7a220
13 changed files with 979 additions and 789 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/scripts/build-vitis-driver.py vendored
View File

@ -11,7 +11,7 @@ def build_vitis_driver():
# - the firesim repo is already setup in a prior script
# - machine-launch-script requirements are already installed
with prefix(f"cd {ci_env['GITHUB_WORKSPACE']}"):
with prefix(f"cd {ci_env['REMOTE_WORK_DIR']}"):
with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
with prefix("cd ./sim"):
run("make PLATFORM=vitis vitis")

4
.github/scripts/check-vitis-docs-generated-components.py vendored
View File

@ -15,10 +15,8 @@ def run_docs_generated_components_check():
# repo should already be checked out
with prefix(f"cd {ci_env['GITHUB_WORKSPACE']}"):
run("./build-setup.sh --skip-validate")
with prefix(f"cd {ci_env['REMOTE_WORK_DIR']}"):
with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
run("firesim managerinit --platform vitis")
with prefix("cd deploy"):
run("cat config_runtime.yaml")
path = "docs/Getting-Started-Guides/On-Premises-FPGA-Tutorial/Running-Simulations/DOCS_EXAMPLE_config_runtime.yaml"

6
.github/scripts/ci_variables.py vendored
View File

@ -47,7 +47,6 @@ class CIEnvironment(TypedDict):
AWS_ACCESS_KEY_ID: str
AWS_SECRET_ACCESS_KEY: str
AWS_DEFAULT_REGION: str
AWS_BUCKET_NAME: str
AZURE_CLIENT_ID: str
AZURE_CLIENT_SECRET: str
AZURE_TENANT_ID: str
@ -60,6 +59,9 @@ class CIEnvironment(TypedDict):
FIRESIM_PEM: str
FIRESIM_PEM_PUBLIC: str
# FireSim repo used on local CI machine to run tests from (cached across all workflow CI jobs)
REMOTE_WORK_DIR: str
RUN_LOCAL = os.environ.get('GITHUB_ACTIONS', 'false') == 'false'
# When running locally (not in a CI pipeline) run commands out of the clone hosting this file.
local_fsim_dir = os.path.normpath((os.path.realpath(__file__)) + "/../../..")
@ -85,7 +87,6 @@ ci_env: CIEnvironment = {
'AWS_ACCESS_KEY_ID': get_ci_value('AWS_ACCESS_KEY_ID'),
'AWS_SECRET_ACCESS_KEY': get_ci_value('AWS_SECRET_ACCESS_KEY'),
'AWS_DEFAULT_REGION': get_ci_value('AWS_DEFAULT_REGION'),
'AWS_BUCKET_NAME': get_ci_value('AWS_BUCKET_NAME'),
'AZURE_CLIENT_ID': get_ci_value('AZURE_CLIENT_ID'),
'AZURE_CLIENT_SECRET': get_ci_value('AZURE_CLIENT_SECRET'),
'AZURE_TENANT_ID': get_ci_value('AZURE_TENANT_ID'),
@ -96,4 +97,5 @@ ci_env: CIEnvironment = {
'AZURE_CI_NSG_ID': get_ci_value('AZURE_CI_NSG_ID'),
'FIRESIM_PEM': get_ci_value('FIRESIM_PEM'),
'FIRESIM_PEM_PUBLIC': get_ci_value('FIRESIM_PEM_PUBLIC'),
'REMOTE_WORK_DIR': get_ci_value('REMOTE_WORK_DIR'),
}

20
.github/scripts/run-agfi-buildbitstream.py vendored
View File

@ -12,6 +12,9 @@ from ci_variables import ci_env
def run_agfi_buildbitstream():
""" Runs AGFI buildbitstream"""
relative_hwdb_path = f"deploy/sample-backup-configs/sample_config_hwdb.yaml"
relative_build_path = f"deploy/sample-backup-configs/sample_config_build.yaml"
with prefix(f'cd {manager_fsim_dir} && source sourceme-f1-manager.sh'):
rc = 0
@ -20,18 +23,24 @@ def run_agfi_buildbitstream():
with prefix(f"export FIRESIM_BUILDFARM_PREFIX={ci_env['GITHUB_RUN_ID']}-{Path(__file__).stem}"):
with settings(warn_only=True):
# pty=False needed to avoid issues with screen -ls stalling in fabric
rc = run("timeout 10h firesim buildbitstream --forceterminate", pty=False).return_code
build_result = run("timeout 10h firesim buildbitstream --forceterminate", pty=False)
rc = build_result.return_code
if rc != 0:
print("Buildbitstream failed")
log_lines = 200
print(f"Buildbitstream failed. Printing {log_lines} of last log file:")
run(f"""LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""")
sys.exit(rc)
else:
# parse the output yamls, replace the sample hwdb's agfi line only
sample_hwdb_filename = f"{manager_fsim_dir}/deploy/sample-backup-configs/sample_config_hwdb.yaml"
sample_hwdb_filename = f"{manager_fsim_dir}/{relative_hwdb_path}"
hwdb_entry_dir = f"{manager_fsim_dir}/deploy/built-hwdb-entries"
built_hwdb_entries = [x for x in os.listdir(hwdb_entry_dir) if os.path.isfile(os.path.join(hwdb_entry_dir, x))]
for hwdb in built_hwdb_entries:
print(f"Printing {hwdb}")
run(f"cat {hwdb_entry_dir}/{hwdb}")
sample_hwdb_lines = open(sample_hwdb_filename).read().split('\n')
with open(sample_hwdb_filename, "w") as sample_hwdb_file:
@ -61,7 +70,7 @@ def run_agfi_buildbitstream():
run(f"cat {sample_hwdb_filename}")
# share agfis
sample_build_filename = f"{manager_fsim_dir}/deploy/sample-backup-configs/sample_config_build.yaml"
sample_build_filename = f"{manager_fsim_dir}/{relative_build_path}"
sample_build_lines = open(sample_build_filename).read().split('\n')
with open(sample_build_filename, "w") as sample_build_file:
for line in sample_build_lines:
@ -78,6 +87,9 @@ def run_agfi_buildbitstream():
run(f"firesim shareagfi -a {sample_hwdb_filename} -b {sample_build_filename}")
# copy back to workspace area so you can PR it
run(f"cp -f {sample_hwdb_filename} {ci_env['GITHUB_WORKSPACE']}/{relative_hwdb_path}")
if __name__ == "__main__":
set_fabric_firesim_pem()
execute(run_agfi_buildbitstream, hosts=["localhost"])

14
.github/scripts/run-linux-poweroff-vitis.py vendored
View File

@ -15,24 +15,14 @@ def run_linux_poweroff_vitis():
# repo should already be checked out
with prefix(f"cd {ci_env['GITHUB_WORKSPACE']}"):
with prefix(f"cd {ci_env['REMOTE_WORK_DIR']}"):
run("./build-setup.sh --skip-validate")
with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
# avoid logging excessive amounts to prevent GH-A masking secrets (which slows down log output)
with prefix('cd sw/firesim-software'):
run("./init-submodules.sh")
# build outputs.yaml (use this workload since firemarshal can guestmount)
with settings(warn_only=True):
rc = run("./marshal -v build test/outputs.yaml &> outputs.full.log").return_code
if rc != 0:
run("cat outputs.full.log")
raise Exception("Building test/outputs.yaml failed to run")
run("./marshal -v build test/outputs.yaml")
run("./marshal -v install test/outputs.yaml")
run("firesim managerinit --platform vitis")
def run_w_timeout(workload_path, workload, timeout, num_passes):
log_tail_length = 300
rc = 0

17
.github/scripts/run-parallel-vcs-metasims.py vendored
View File

@ -16,24 +16,13 @@ def run_parallel_metasim():
# repo should already be checked out
with prefix(f"cd {ci_env['GITHUB_WORKSPACE']}"):
run("./build-setup.sh --skip-validate")
with prefix(f"cd {ci_env['REMOTE_WORK_DIR']}"):
with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
# avoid logging excessive amounts to prevent GH-A masking secrets (which slows down log output)
with prefix('cd sw/firesim-software'):
run("./init-submodules.sh")
# build hello world baremetal test
with settings(warn_only=True):
rc = run("./marshal -v build test/bare.yaml &> bare.full.log").return_code
if rc != 0:
run("cat bare.full.log")
raise Exception("Building test/bare.yaml failed to run")
run("./marshal -v build test/bare.yaml")
run("./marshal -v install test/bare.yaml")
run("firesim managerinit --platform vitis")
def run_w_timeout(workload: str, timeout: str):
""" Run workload with a specific timeout
@ -69,7 +58,7 @@ def run_parallel_metasim():
else:
print(f"Workload {workload} successful.")
run_w_timeout(f"{ci_env['GITHUB_WORKSPACE']}/deploy/workloads/ci/hello-world-localhost-vcs-metasim.yaml", "45m")
run_w_timeout(f"{ci_env['REMOTE_WORK_DIR']}/deploy/workloads/ci/hello-world-localhost-vcs-metasim.yaml", "45m")
if __name__ == "__main__":
execute(run_parallel_metasim, hosts=["localhost"])

234
.github/scripts/run-xclbin-buildbitstream.py vendored
View File

@ -4,33 +4,67 @@ import sys
from pathlib import Path
from fabric.api import prefix, run, settings, execute # type: ignore
import os
from botocore.exceptions import ClientError
import boto3
from github import Github
import base64
import time
from ci_variables import ci_env
# taken from: https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html
def upload_file(file_name, bucket, object_name=None):
"""Upload a file to an S3 bucket
GH_REPO = 'firesim-public-bitstreams'
GH_ORG = 'firesim'
URL_PREFIX = f"https://raw.githubusercontent.com/{GH_ORG}/{GH_REPO}"
:param file_name: File to upload
:param bucket: Bucket to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# taken from https://stackoverflow.com/questions/63427607/python-upload-files-directly-to-github-using-pygithub
def upload_binary_file(local_file_path, gh_file_path):
print(f":DEBUG: Attempting to upload {local_file_path} to {gh_file_path}")
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = os.path.basename(file_name)
g = Github(ci_env['PERSONAL_ACCESS_TOKEN'])
# Upload the file
s3_client = boto3.client('s3')
try:
response = s3_client.upload_file(file_name, bucket, object_name)
except ClientError as e:
print(e)
return False
return True
repo = g.get_repo(f'{GH_ORG}/{GH_REPO}')
all_files = []
contents = repo.get_contents("")
while contents:
file_content = contents.pop(0)
if file_content.type == "dir":
contents.extend(repo.get_contents(file_content.path))
else:
file = file_content
all_files.append(str(file).replace('ContentFile(path="','').replace('")',''))
with open(local_file_path, 'rb') as file:
content = base64.b64encode(file.read()).decode("utf-8")
tries = 10
delay = 15
msg = f"Committing files from {ci_env['GITHUB_SHA']}"
upload_branch = 'main'
r = None
# Upload to github
git_file = gh_file_path
if git_file in all_files:
contents = repo.get_contents(git_file)
for n in range(tries):
try:
r = repo.update_file(contents.path, msg, content, contents.sha, branch=upload_branch)
break
except Exception as e:
print(f"Got exception: {e}")
time.sleep(delay)
assert r is not None, f"Unable to poll 'update_file' API {tries} times"
print(f"Updated: {git_file}")
else:
for n in range(tries):
try:
r = repo.create_file(git_file, msg, content, branch=upload_branch)
break
except Exception as e:
print(f"Got exception: {e}")
time.sleep(delay)
assert r is not None, f"Unable to poll 'create_file' API {tries} times"
print(f"Created: {git_file}")
return r['commit'].sha
def run_xclbin_buildbitstream():
""" Runs Xclbin buildbitstream"""
@ -41,86 +75,114 @@ def run_xclbin_buildbitstream():
# repo should already be checked out
manager_fsim_dir = ci_env['GITHUB_WORKSPACE']
#print("DEBUG: " + str(upload_binary_file("/scratch/buildbot/prebuilt-xclbins/firesim.xclbin", "vitis/temp2.xclbin")))
#sys.exit(0)
manager_fsim_dir = ci_env['REMOTE_WORK_DIR']
with prefix(f"cd {manager_fsim_dir}"):
run("./build-setup.sh --skip-validate")
with prefix('source sourceme-f1-manager.sh --skip-ssh-setup'):
# modify config_build.yaml
build_yaml = f"{manager_fsim_dir}/deploy/config_build.yaml"
build_yaml_lines = open(build_yaml).read().split("\n")
with open(build_yaml, "w") as byf:
for line in build_yaml_lines:
if "- firesim" in line:
# comment out AWS specific lines
byf.write("# " + line + '\n')
elif "- vitis_firesim" in line:
# remove comment on vitis line
byf.write(line.replace("# ", '') + '\n')
else:
byf.write(line + '\n')
# return a copy of config_build.yaml w/ hwdb entry uncommented + new build dir
def modify_config_build(hwdb_entry_to_gen: str) -> str:
build_yaml = f"{manager_fsim_dir}/deploy/config_build.yaml"
copy_build_yaml = f"{manager_fsim_dir}/deploy/config_build_{hwdb_entry_to_gen}.yaml"
build_yaml_lines = open(build_yaml).read().split("\n")
with open(copy_build_yaml, "w") as byf:
for line in build_yaml_lines:
if "- firesim" in line:
# comment out AWS specific lines
byf.write("# " + line + '\n')
elif f"- {hwdb_entry_to_gen}" in line:
# remove comment
byf.write(line.replace("# ", '') + '\n')
elif 'default_build_dir:' in line:
byf.write(line.replace('null', f"{manager_fsim_dir}/tmp_build_area") + '\n')
else:
byf.write(line + '\n')
return copy_build_yaml
run(f"Printing {build_yaml}...")
run(f"cat {build_yaml}")
def build_upload(build_yaml: str, hwdb_entry_name: str) -> str:
rc = 0
with settings(warn_only=True):
# pty=False needed to avoid issues with screen -ls stalling in fabric
rc = run("timeout 10h firesim buildbitstream --forceterminate", pty=False).return_code
print(f"Printing {build_yaml}...")
run(f"cat {build_yaml}")
rc = 0
with settings(warn_only=True):
# pty=False needed to avoid issues with screen -ls stalling in fabric
build_result = run(f"timeout 10h firesim buildbitstream -b {build_yaml} --forceterminate", pty=False)
rc = build_result.return_code
if rc != 0:
log_lines = 200
print(f"Buildbitstream failed. Printing {log_lines} of last log file:")
run(f"""LAST_LOG=$(ls | tail -n1) && if [ -f "$LAST_LOG" ]; then tail -n{log_lines} $LAST_LOG; fi""")
sys.exit(rc)
if rc != 0:
print("Buildbitstream failed")
sys.exit(rc)
else:
hwdb_entry_dir = f"{manager_fsim_dir}/deploy/built-hwdb-entries"
built_hwdb_entries = [x for x in os.listdir(hwdb_entry_dir) if os.path.isfile(os.path.join(hwdb_entry_dir, x))]
hwdb_entry = f"{hwdb_entry_dir}/{hwdb_entry_name}"
hwdb_to_link = {}
for hwdb in built_hwdb_entries:
with open(f"{hwdb_entry_dir}/{hwdb}") as hwdbef:
lines = hwdbef.readlines()
for line in lines:
if "xclbin:" in line:
file_path = Path(line.strip().split(' ')[1]) # 2nd element
file_name = f"{hwdb}_{ci_env['GITHUB_SHA'][0:6]}.xclbin"
if not upload_file(file_path, ci_env['AWS_BUCKET_NAME'], file_name):
print(f"Unable to upload the xclbin for {hwdb}")
else:
link = f"https://{ci_env['AWS_BUCKET_NAME']}.s3.{ci_env['AWS_DEFAULT_REGION']}.amazonaws.com/{file_name}"
print(f"Uploaded xclbin for {hwdb} to {link}")
hwdb_to_link[hwdb] = f"https://{ci_env['AWS_BUCKET_NAME']}.s3.{ci_env['AWS_DEFAULT_REGION']}.amazonaws.com/{file_name}"
print(f"Printing {hwdb_entry}...")
run(f"cat {hwdb_entry}")
# parse the output yamls, replace the sample hwdb's xclbin line only
sample_hwdb_filename = f"{manager_fsim_dir}/deploy/sample-backup-configs/sample_config_hwdb.yaml"
for hwdb in built_hwdb_entries:
sample_hwdb_lines = open(sample_hwdb_filename).read().split('\n')
with open(hwdb_entry, 'r') as hwdbef:
lines = hwdbef.readlines()
for line in lines:
if "xclbin:" in line:
file_path = Path(line.strip().split(' ')[1].replace('file://', '')) # 2nd element (i.e. the path) (no URI)
file_name = f"vitis/{hwdb_entry_name}.xclbin"
sha = upload_binary_file(file_path, file_name)
link = f"{URL_PREFIX}/{sha}/{file_name}"
print(f"Uploaded xclbin for {hwdb_entry_name} to {link}")
return link
with open(sample_hwdb_filename, "w") as sample_hwdb_file:
match_xclbin = False
for line in sample_hwdb_lines:
if hwdb in line.strip().split(' ')[0].replace(':', ''):
# hwdb entry matches key name
match_xclbin = True
sample_hwdb_file.write(line + '\n')
elif match_xclbin == True and ("xclbin:" in line.strip().split(' ')[0]):
# only replace this xclbin
match_xclbin = False
sys.exit(":ERROR: Something went wrong. Should have uploaded by now and returned a link.")
new_xclbin_line = f" xclbin: {hwdb_to_link[hwdb]}"
print(f"Replacing {line.strip()} with {new_xclbin_line}")
relative_hwdb_path = "deploy/sample-backup-configs/sample_config_hwdb.yaml"
sample_hwdb_filename = f"{manager_fsim_dir}/{relative_hwdb_path}"
# print out the xclbin line
sample_hwdb_file.write(new_xclbin_line + '\n')
else:
# if no match print other lines
sample_hwdb_file.write(line + '\n')
def replace_in_hwdb(hwdb_entry_name: str, link: str) -> None:
# replace the sample hwdb's xclbin line only
sample_hwdb_lines = open(sample_hwdb_filename).read().split('\n')
if match_xclbin == True:
sys.exit("::ERROR:: Unable to find matching xclbin key for HWDB entry")
with open(sample_hwdb_filename, "w") as sample_hwdb_file:
match_xclbin = False
for line in sample_hwdb_lines:
if hwdb_entry_name in line.strip().split(' ')[0].replace(':', ''):
# hwdb entry matches key name
match_xclbin = True
sample_hwdb_file.write(line + '\n')
elif match_xclbin == True and ("xclbin:" in line.strip().split(' ')[0]):
# only replace this xclbin
match_xclbin = False
print(f"Printing {sample_hwdb_filename}...")
run(f"cat {sample_hwdb_filename}")
new_xclbin_line = f" xclbin: {link}"
print(f"Replacing {line.strip()} with {new_xclbin_line}")
# print out the xclbin line
sample_hwdb_file.write(new_xclbin_line + '\n')
else:
# if no match print other lines
sample_hwdb_file.write(line + '\n')
if match_xclbin == True:
sys.exit(f"::ERROR:: Unable to replace URL for {hwdb_entry_name} in {sample_hwdb_filename}")
# roughly takes ~4h to generate
hwdb_entry_name = "vitis_firesim_rocket_singlecore_no_nic"
copy_build_yaml = modify_config_build(hwdb_entry_name)
replace_in_hwdb(hwdb_entry_name, build_upload(copy_build_yaml, hwdb_entry_name))
# roughly takes ~4h to generate
hwdb_entry_name = "vitis_firesim_gemmini_rocket_singlecore_no_nic"
copy_build_yaml = modify_config_build(hwdb_entry_name)
replace_in_hwdb(hwdb_entry_name, build_upload(copy_build_yaml, hwdb_entry_name))
print(f"Printing {sample_hwdb_filename}...")
run(f"cat {sample_hwdb_filename}")
# copy back to workspace area so you can PR it
run(f"cp -f {sample_hwdb_filename} {ci_env['GITHUB_WORKSPACE']}/{relative_hwdb_path}")
if __name__ == "__main__":
execute(run_xclbin_buildbitstream, hosts=["localhost"])

4
.github/scripts/workflow-monitor.py vendored
View File

@ -60,9 +60,9 @@ def main(platform: Platform, issue_id: int):
print(f"Workflow {ci_env['GITHUB_RUN_ID']} status: {state_status} {state_concl}")
# check that select instances are terminated on time
# check that select instances are terminated on time (45m for run farm insts, 12h for build farm insts)
platform_lib.check_and_terminate_run_farm_instances(45, ci_env['GITHUB_RUN_ID'], issue_id)
platform_lib.check_and_terminate_build_farm_instances(12, ci_env['GITHUB_RUN_ID'], issue_id)
platform_lib.check_and_terminate_build_farm_instances(12*60, ci_env['GITHUB_RUN_ID'], issue_id)
if state_status in ['completed']:
if state_concl in TERMINATE_STATES:

227
.github/workflows/firesim-run-tests.yml vendored
View File

@ -13,7 +13,6 @@ env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
AWS_BUCKET_NAME: firesim-ci-vitis-xclbins
AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
@ -30,6 +29,8 @@ env:
LC_ALL: "en_US.UTF-8"
CI_LABEL_DEBUG: ${{ contains(github.event.pull_request.labels.*.name, 'ci:debug') }}
CI_LABEL_PERSIST: ${{ contains(github.event.pull_request.labels.*.name, 'ci:persist-prior-workflows') }}
REMOTE_WORK_DIR: /scratch/buildbot/fs-shared/fs-${{ github.sha }}
TERM: xterm-256-color
jobs:
cancel-prior-workflows:
@ -139,7 +140,7 @@ jobs:
if: ${{ (env.CI_LABEL_DEBUG != 'true') }}
uses: ./.github/actions/setup-workflow-monitor
with:
max-runtime-hours: 4
max-runtime-hours: 8
- name: Initial Scala compilation
uses: ./.github/actions/initial-scala-compile
- name: Catch potentially orphaned manager
@ -156,8 +157,6 @@ jobs:
name: build-default-workloads
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Build default workloads (FireMarshal and paper workloads)
@ -167,8 +166,6 @@ jobs:
name: run-manager-pytests
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run pytests
@ -178,8 +175,6 @@ jobs:
name: run-manager-python-typecheck
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run manager Python type checker
@ -189,8 +184,6 @@ jobs:
name: run-ci-python-typecheck
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run CI Python type checker
@ -200,8 +193,6 @@ jobs:
name: run-scalafmt-check
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run Scalafmt on FireSim Scala main sources
@ -211,8 +202,6 @@ jobs:
name: run-check-docs-generated-components
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Check docs components that require manual re-generation (e.g. config_runtime.yaml example, help output)
@ -222,8 +211,6 @@ jobs:
name: build-f1-driver
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Runs compilation of F1 driver for the make-default tuple
@ -233,8 +220,6 @@ jobs:
name: run-targetutils-scala-tests
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run targetutils scala tests
@ -244,8 +229,6 @@ jobs:
name: run-midas-scala-tests
needs: [setup-manager]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run targetutils scala tests
@ -255,8 +238,6 @@ jobs:
name: run-test-groupA
needs: [build-f1-driver]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run CIGroupA Scala tests
@ -268,8 +249,6 @@ jobs:
name: run-test-groupB
needs: [run-test-groupA]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run CIGroupB Scala tests
@ -281,8 +260,6 @@ jobs:
name: run-test-firesim-lib
needs: [run-test-groupB]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run firesim-lib Scala tests
@ -296,8 +273,6 @@ jobs:
name: run-chipyard-tests
needs: [run-test-firesim-lib]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run other (CITests) Scala tests
@ -313,8 +288,6 @@ jobs:
# this behind the scalatest train. Remove once we're off SBT.
needs: [run-chipyard-tests]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run parallel Verilator metasimulation tests (deploy on localhost and on AWS instances)
@ -327,8 +300,6 @@ jobs:
# this behind the scalatest train. Remove once we're off SBT.
needs: [build-default-workloads, run-chipyard-tests, run-parallel-verilator-metasims]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run linux-poweroff test w/ AWS EC2 run farm
@ -339,104 +310,29 @@ jobs:
name: run-basic-linux-poweroff-externally-provisioned
needs: [build-default-workloads]
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Run linux-poweroff test w/ externally provisioned (AWS EC2) run farm
run: .github/scripts/run-linux-poweroff-externally-provisioned.py
run-parallel-vcs-metasims-and-vitis-driver:
name: run-parallel-vcs-metasims-and-vitis-driver
runs-on: local-fpga
env:
TERM: xterm-256-color
steps:
# This forces a fresh clone of the repo during the `checkout` step
# to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
- name: Delete old checkout
run: |
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
- uses: actions/checkout@v3
- name: Run parallel VCS metasims
run: .github/scripts/run-parallel-vcs-metasims.py
- name: Build Vitis driver
run: .github/scripts/build-vitis-driver.py
run-basic-linux-poweroff-vitis:
if: contains(github.event.pull_request.labels.*.name, 'ci:fpga-deploy')
name: run-basic-linux-poweroff-vitis
runs-on: local-fpga
env:
TERM: xterm-256-color
steps:
# This forces a fresh clone of the repo during the `checkout` step
# to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
- name: Delete old checkout
run: |
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
- uses: actions/checkout@v3
- name: Run simple linux poweroff test w/ vitis
run: .github/scripts/run-linux-poweroff-vitis.py
run-agfi-buildbitstream:
if: contains(github.event.pull_request.labels.*.name, 'ci:buildbitstream-deploy')
name: run-agfi-buildbitstream
needs: [build-f1-driver] # delay until known working scala compile
runs-on: aws-${{ github.run_id }}
env:
TERM: xterm-256-color
timeout-minutes: 600 # 10h
steps:
- uses: actions/checkout@v3
- name: Run buildbitstream command and update sample AGFIs
run: .github/scripts/run-agfi-buildbitstream.py
- uses: peter-evans/create-pull-request@v4
- uses: peter-evans/create-pull-request@v5
with:
# must align with `manager_fsim_dir/MANAGER_FIRESIM_LOCATION`
cwd: '/home/centos/firesim'
add-paths: |
'deploy/sample-backup-configs/sample_config_hwdb.yaml'
commit-message: "Update AGFI(s)"
base: ${{ github.head_ref }}
add-paths: "deploy/sample-backup-configs/sample_config_hwdb.yaml"
commit-message: "Update AGFI(s) [ci skip]"
body: "Update AGFI(s) for PR #${{ github.event.pull_request.number }}"
branch-suffix: short-commit-hash
title: "Update AGFI(s) for ${{ env.GITHUB_REF_NAME }}"
# TODO: Re-enable and just post bitstreams into a GH repo
#run-xclbin-buildbitstream:
# if: contains(github.event.pull_request.labels.*.name, 'ci:buildbitstream-deploy')
# name: run-xclbin-buildbitstream
# runs-on: local-fpga
# env:
# TERM: xterm-256-color
# steps:
# # This forces a fresh clone of the repo during the `checkout` step
# # to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
# - name: Delete old checkout
# run: |
# rm -rf ${{ github.workspace }}/* || true
# rm -rf ${{ github.workspace }}/.* || true
# - uses: actions/checkout@v3
# - uses: ./.github/actions/repo-setup-aws
# - name: Run buildbitstream command and update sample xclbins
# run: .github/scripts/run-xclbin-buildbitstream.py
# - uses: peter-evans/create-pull-request@v4
# with:
# add-paths: |
# 'deploy/sample-backup-configs/sample_config_hwdb.yaml'
# commit-message: "Update xclbin(s)"
# branch-suffix: short-commit-hash
# title: "Update xclbin(s) for ${{ env.GITHUB_REF_NAME }}"
run-vitis-check-docs-generated-components:
name: run-vitis-check-docs-generated-components
runs-on: local-fpga
env:
TERM: xterm-256-color
steps:
- uses: actions/checkout@v3
- name: Check Vitis docs components that require manual re-generation (e.g. config_runtime.yaml example)
run: ./.github/scripts/check-vitis-docs-generated-components.py
title: "Update AGFI(s) for PR #${{ github.event.pull_request.number }} (`${{ github.head_ref }}`)"
documentation-check:
name: documentation-check
@ -509,3 +405,106 @@ jobs:
export FIRESIM_ENV_SOURCED=1
export FIRESIM_STANDALONE=1
make -C sim clang-tidy
### Local FPGAs CI ###
setup-local-fpga-repo:
name: setup-local-fpga-repo
runs-on: local-fpga
steps:
# This forces a fresh clone of the repo during the `checkout` step
# to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
- name: Delete old checkout
run: |
ls -alh .
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
ls -alh .
- uses: actions/checkout@v3
- name: Setup repo copy
run: |
mkdir -p $(dirname ${{ env.REMOTE_WORK_DIR }})
git clone ${{ github.workspace }} ${{ env.REMOTE_WORK_DIR }}
- name: Setup repo
run: |
cd ${{ env.REMOTE_WORK_DIR }}
./build-setup.sh --skip-validate
source sourceme-f1-manager.sh --skip-ssh-setup
firesim managerinit --platform vitis
cd sw/firesim-software && ./init-submodules.sh
run-vitis-check-docs-generated-components:
name: run-vitis-check-docs-generated-components
needs: [setup-local-fpga-repo]
runs-on: local-fpga
steps:
- uses: actions/checkout@v3
- name: Check Vitis docs components that require manual re-generation (e.g. config_runtime.yaml example)
run: ./.github/scripts/check-vitis-docs-generated-components.py
run-parallel-vcs-metasims-and-vitis-driver:
name: run-parallel-vcs-metasims-and-vitis-driver
needs: [setup-local-fpga-repo]
runs-on: local-fpga
steps:
# This forces a fresh clone of the repo during the `checkout` step
# to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
- name: Delete old checkout
run: |
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
- uses: actions/checkout@v3
- name: Run parallel VCS metasims
run: .github/scripts/run-parallel-vcs-metasims.py
- name: Build Vitis driver
run: .github/scripts/build-vitis-driver.py
run-basic-linux-poweroff-vitis:
if: contains(github.event.pull_request.labels.*.name, 'ci:fpga-deploy')
needs: [setup-local-fpga-repo]
name: run-basic-linux-poweroff-vitis
runs-on: local-fpga
steps:
# This forces a fresh clone of the repo during the `checkout` step
# to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
- name: Delete old checkout
run: |
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
- uses: actions/checkout@v3
- name: Run simple linux poweroff test w/ vitis
run: .github/scripts/run-linux-poweroff-vitis.py
run-xclbin-buildbitstream:
if: contains(github.event.pull_request.labels.*.name, 'ci:buildbitstream-deploy')
needs: [setup-local-fpga-repo]
name: run-xclbin-buildbitstream
runs-on: local-fpga
timeout-minutes: 600
steps:
# This forces a fresh clone of the repo during the `checkout` step
# to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
- name: Delete old checkout
run: |
rm -rf ${{ github.workspace }}/* || true
rm -rf ${{ github.workspace }}/.* || true
- uses: actions/checkout@v3
- name: Run buildbitstream command and update sample xclbins
run: .github/scripts/run-xclbin-buildbitstream.py
- uses: peter-evans/create-pull-request@v5
with:
base: ${{ github.head_ref }}
add-paths: "deploy/sample-backup-configs/sample_config_hwdb.yaml"
commit-message: "Update xclbin(s) [ci skip]"
body: "Update xclbin(s) for PR #${{ github.event.pull_request.number }}"
branch-suffix: short-commit-hash
title: "Update xclbin(s) for PR #${{ github.event.pull_request.number }} (`${{ github.head_ref }}`)"
cleanup-local-fpga-repo:
name: cleanup-local-fpga-repo
needs: [run-basic-linux-poweroff-vitis, run-vitis-check-docs-generated-components, run-basic-linux-poweroff-vitis, run-xclbin-buildbitstream]
runs-on: local-fpga
if: ${{ always() }}
steps:
- name: Delete repo copy
run: rm -rf ${{ env.REMOTE_WORK_DIR }}

2
conda-reqs/ci-shared.yaml
View File

@ -34,6 +34,8 @@ dependencies:
- fsspec
- pygithub
- pip
- pip:
- fab-classic>=1.19.2

1229
conda-reqs/conda-reqs.conda-lock.yml generated
View File

File diff suppressed because it is too large Load Diff

3
deploy/sample-backup-configs/sample_config_build_recipes.yaml
View File

@ -235,7 +235,7 @@ vitis_firesim_rocket_singlecore_no_nic:
PLATFORM_CONFIG: BaseVitisConfig
deploy_quintuplet: null
platform_config_args:
fpga_frequency: 140
fpga_frequency: 30 # previously reached 140. using 30 for build speed
build_strategy: TIMING
post_build_hook: null
metasim_customruntimeconfig: null
@ -331,4 +331,3 @@ xilinx_vcu118_firesim_rocket_singlecore_4GB_no_nic:
post_build_hook: null
metasim_customruntimeconfig: null
bit_builder_recipe: bit-builder-recipes/xilinx_vcu118.yaml

6
deploy/sample-backup-configs/sample_config_hwdb.yaml
View File

@ -56,11 +56,11 @@ firesim_gemmini_rocket_singlecore_no_nic:
deploy_quintuplet_override: null
custom_runtime_config: null
vitis_firesim_rocket_singlecore_no_nic:
xclbin: https://firesim-ci-vitis-xclbins.s3.us-west-2.amazonaws.com/vitis_firesim_rocket_singlecore_no_nic_c12936.xclbin
xclbin: https://raw.githubusercontent.com/firesim/firesim-public-bitstreams/995b8ecbbc527f1be8bc17c55acbeadf45f917c9/vitis/vitis_firesim_rocket_singlecore_no_nic.xclbin
deploy_quintuplet_override: vitis-firesim-FireSim-FireSimRocketMMIOOnlyConfig-BaseVitisConfig
custom_runtime_config: null
vitis_firesim_gemmini_rocket_singlecore_no_nic:
xclbin: https://firesim-ci-vitis-xclbins.s3.us-west-2.amazonaws.com/vitis_firesim_gemmini_rocket_singlecore_no_nic_1ea5c4.xclbin
xclbin: https://raw.githubusercontent.com/firesim/firesim-public-bitstreams/183bfa31d95ac97495547497c5d13124208b68fd/vitis/vitis_firesim_gemmini_rocket_singlecore_no_nic.xclbin
deploy_quintuplet_override: vitis-firesim-FireSim-FireSimLeanGemminiRocketMMIOOnlyConfig-BaseVitisConfig
custom_runtime_config: null
# DOCREF START: Xilinx Alveo HWDB Entries
@ -73,3 +73,5 @@ alveo_u280_firesim_rocket_singlecore_no_nic:
deploy_quintuplet_override: null
custom_runtime_config: null
# DOCREF END: Xilinx Alveo HWDB Entries