firesim/.github/workflows/firesim-run-tests.yml

name: firesim-ci-process

on:
  # run ci when pring to main (note: ci runs on the merge commit of the pr!)
  pull_request:
    branches:
      - main
      - stable

defaults:
  run:
    shell: bash -leo pipefail {0}

env:
  PERSONAL_ACCESS_TOKEN: ${{ secrets.BARTENDER_PERSONAL_ACCESS_TOKEN }}
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID_08012023 }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY_08012023 }}
  AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION_08012023 }}
  FIRESIM_PEM: ${{ secrets.FIRESIM_PEM_08012023 }}
  FIRESIM_PEM_PUBLIC: ${{ secrets.FIRESIM_PEM_PUBLIC }}
  AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
  AZURE_CLIENT_SECRET: ${{ secrets.AZURE_CLIENT_SECRET }}
  AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
  AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
  AZURE_DEFAULT_REGION: ${{ secrets.AZURE_DEFAULT_REGION }}
  AZURE_RESOURCE_GROUP: ${{ secrets.AZURE_RESOURCE_GROUP }}
  AZURE_CI_SUBNET_ID : ${{ secrets.AZURE_CI_SUBNET_ID }}
  AZURE_CI_NSG_ID : ${{ secrets.AZURE_CI_NSG_ID }}
  MANAGER_FIRESIM_LOCATION: "~/firesim"
  LANG: "en_US.UTF-8" # required by SBT when it sees boost directories
  LANGUAGE: "en_US:en"
  LC_ALL: "en_US.UTF-8"
  CI_LABEL_DEBUG: ${{ contains(github.event.pull_request.labels.*.name, 'ci:debug') }}
  CI_LABEL_PERSIST: ${{ contains(github.event.pull_request.labels.*.name, 'ci:persist-prior-workflows') }}
  REMOTE_WORK_DIR: /scratch/buildbot/fs-shared/fs-${{ github.sha }}
  TERM: xterm-256-color

jobs:
  cancel-prior-workflows:
    name: cancel-prior-workflows
    runs-on: ubuntu-20.04
    steps:
      - name: Cancel previous workflow runs
        if: ${{ (env.CI_LABEL_PERSIST != 'true') }}
        uses: styfle/cancel-workflow-action@0.11.0
        with:
          access_token: ${{ github.token }}

  # Set up a set of boolean conditions to control which branches of the CI
  # workflow will execute This is based off the conditional job execution
  # example here: https://github.com/dorny/paths-filter#examples
  change-filters:
    name: filter-jobs-on-changes
    runs-on: ubuntu-20.04
    # Queried by downstream jobs to determine if they should run.
    outputs:
      needs-manager: ${{ steps.filter.outputs.all_count != steps.filter.outputs.skip-manager_count }}
      both-conda-reqs-lock-modified: ${{ ((steps.filter.outputs.conda-reqs == 'false') && (steps.filter.outputs.conda-lock == 'false')) || ((steps.filter.outputs.conda-reqs == 'true') && (steps.filter.outputs.conda-lock == 'true')) }}
    steps:
      - uses: actions/checkout@v3
      - uses: dorny/paths-filter@v2
        id: filter
        with:
          filters: |
            all:
              - '**'

            # If any of the files changed match, do a doc build
            docs: &docs-filter
              - 'docs/**'
              - '.readthedocs.yml'

            # If all files match to this filter, skip the main ci pipeline
            skip-manager:
              - *docs-filter
              - '**/*.md'
              - '**/.gitignore'
              - '.github/ISSUE_TEMPLATE/**'

            # If conda requirements was modified ensure the lock file is re-generated
            conda-reqs:
                - 'conda-reqs/firesim.yaml'
                - 'conda-reqs/ci-shared.yaml'

            # If conda requirements was modified ensure the lock file is re-generated
            conda-lock:
                - 'conda-reqs/conda-reqs.conda-lock.yml'            

  # Note: This doesn't check if the lock file is synced/faithful to the requirements file.
  # This just ensures that both were modified in the same PR (ideally the lock file was regenerated
  # from the requirements file). This job only runs when that condition is not met and
  # so always fails.
  check-conda-lock-modified:
    name: check-conda-lock-modified
    needs: change-filters
    if: needs.change-filters.outputs.both-conda-reqs-lock-modified == 'false'
    runs-on: ubuntu-20.04
    steps:
      - name: Check conda lock file was regenerated with conda requirements file
        run: |
          echo "ERROR: Either the conda-reqs/{firesim,ci-shared}.yaml or conda-reqs/conda-reqs.conda-lock.yml was not updated properly. See the developer docs for more information"
          false          

  setup-self-hosted-manager:
    name: setup-self-hosted-manager
    needs: change-filters
    if: needs.change-filters.outputs.needs-manager == 'true'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - uses: conda-incubator/setup-miniconda@v2
        with:
          environment-file: conda-reqs/ci-shared.yaml
          miniforge-version: latest
      - uses: ./.github/actions/repo-setup-aws
      - name: Launch AWS instance used for the FireSim manager (instance info found here)
        run: ./.github/scripts/launch-manager-instance.py
      - name: Setup N Github Actions Runners on AWS instance
        run: ./.github/scripts/setup-manager-self-hosted.py aws
      - name: Catch potentially orphaned manager
        if: ${{ (env.CI_LABEL_DEBUG != 'true') && (failure() || cancelled()) }}
        uses: ./.github/actions/change-workflow-instance-states
        with:
          new-state: terminate
          github-token: ${{ env.PERSONAL_ACCESS_TOKEN }}

  setup-manager:
    name: setup-manager
    needs: [setup-self-hosted-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Install FireSim pem
        run: ./.github/scripts/install-firesim-pem.py
      - name: Setup FireSim repo (.pem, build-setup.sh, AWS credentials, submodules) and CI daemons
        run: ./.github/scripts/initialize-repo.py
      - name: Run AWS configure
        run: ./.github/scripts/run-aws-configure.py
      - name: Setup Workflow monitor
        if: ${{ (env.CI_LABEL_DEBUG != 'true') }}
        uses: ./.github/actions/setup-workflow-monitor
        with:
          max-runtime-hours: 8
      - name: Initial Scala compilation
        uses: ./.github/actions/initial-scala-compile
      - name: Catch potentially orphaned manager
        if: ${{ (env.CI_LABEL_DEBUG != 'true') && (failure() || cancelled()) }}
        uses: ./.github/actions/change-workflow-instance-states
        with:
          new-state: terminate
          github-token: ${{ env.PERSONAL_ACCESS_TOKEN }}

  build-default-workloads:
    # Conditionally build rootfs images only if deploying to FPGA to save CI resources
    # https://stackoverflow.com/questions/62325286/run-github-actions-when-pull-requests-have-a-specific-label/62331521#comment122159108_62331521
    if: contains(github.event.pull_request.labels.*.name, 'ci:fpga-deploy')
    name: build-default-workloads
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Build default workloads (FireMarshal and paper workloads)
        run: .github/scripts/build-default-workloads.py

  run-manager-pytests:
    name: run-manager-pytests
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run pytests
        run: .github/scripts/run-manager-pytests.py

  run-manager-python-typecheck:
    name: run-manager-python-typecheck
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run manager Python type checker
        run: .github/scripts/run-manager-python-typechecking.py

  run-ci-python-typecheck:
      name: run-ci-python-typecheck
      needs: [setup-manager]
      runs-on: aws-${{ github.run_id }}
      steps:
      - uses: actions/checkout@v3
      - name: Run CI Python type checker
        run: .github/scripts/run-ci-python-typechecking.py

  run-scalafmt-check:
    name: run-scalafmt-check
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run Scalafmt on FireSim Scala main sources
        run: .github/scripts/run-scala-lint-check.py

  run-check-docs-generated-components:
    name: run-check-docs-generated-components
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Check docs components that require manual re-generation (e.g. config_runtime.yaml example, help output)
        run: ./.github/scripts/check-docs-generated-components.py

  build-f1-driver:
    name: build-f1-driver
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Runs compilation of F1 driver for the make-default tuple
        run: ./.github/scripts/build-f1-driver.py

  run-targetutils-scala-tests:
    name: run-targetutils-scala-tests
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run targetutils scala tests
        run: ./.github/scripts/run-sbt-command.py midasexamples "'project targetutils; test' "

  run-midas-scala-tests:
    name: run-midas-scala-tests
    needs: [setup-manager]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run targetutils scala tests
        run: ./.github/scripts/run-sbt-command.py midasexamples "'project midas; test' "

  run-test-groupA:
    name: run-test-groupA
    needs: [build-f1-driver]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run CIGroupA Scala tests
        uses: ./.github/actions/run-scala-test
        with:
          test-name: "CIGroupA"

  run-test-groupB:
    name: run-test-groupB
    needs: [run-test-groupA]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run CIGroupB Scala tests
        uses: ./.github/actions/run-scala-test
        with:
          test-name: "CIGroupB"

  run-test-firesim-lib:
    name: run-test-firesim-lib
    needs: [run-test-groupB]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run firesim-lib Scala tests
        uses: ./.github/actions/run-scala-test
        with:
          target-project: "bridges"
          test-package: "firesim.bridges"
          test-name: "BridgeTests"

  run-chipyard-tests:
    name: run-chipyard-tests
    needs: [run-test-firesim-lib]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run other (CITests) Scala tests
        uses: ./.github/actions/run-scala-test
        with:
          target-project: "firesim"
          test-package: "firesim.firesim"
          test-name: "CITests"

  run-parallel-verilator-metasims:
    name: run-parallel-verilator-metasims
    # Building the driver can cause concurrency issues with SBT, so serialize
    # this behind the scalatest train. Remove once we're off SBT.
    needs: [run-chipyard-tests]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run parallel Verilator metasimulation tests (deploy on localhost and on AWS instances)
        run: .github/scripts/run-parallel-verilator-metasims.py

  run-basic-linux-poweroff:
    if: contains(github.event.pull_request.labels.*.name, 'ci:fpga-deploy')
    name: run-basic-linux-poweroff
    # Building the driver can cause concurrency issues with SBT, so serialize
    # this behind the scalatest train. Remove once we're off SBT.
    needs: [build-default-workloads, run-chipyard-tests, run-parallel-verilator-metasims]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run linux-poweroff test w/ AWS EC2 run farm
        run: .github/scripts/run-linux-poweroff.py

  run-basic-linux-poweroff-externally-provisioned:
    if: contains(github.event.pull_request.labels.*.name, 'ci:fpga-deploy')
    name: run-basic-linux-poweroff-externally-provisioned
    needs: [build-default-workloads]
    runs-on: aws-${{ github.run_id }}
    steps:
      - uses: actions/checkout@v3
      - name: Run linux-poweroff test w/ externally provisioned (AWS EC2) run farm
        run: .github/scripts/run-linux-poweroff-externally-provisioned.py

  run-agfi-buildbitstream:
    if: contains(github.event.pull_request.labels.*.name, 'ci:agfi-buildbitstream-deploy')
    name: run-agfi-buildbitstream
    needs: [build-f1-driver] # delay until known working scala compile
    runs-on: aws-${{ github.run_id }}
    timeout-minutes: 600 # 10h
    steps:
      - uses: actions/checkout@v3
      - name: Run buildbitstream command and update sample AGFIs
        run: .github/scripts/run-agfi-buildbitstream.py
      - uses: peter-evans/create-pull-request@v5
        with:
          base: ${{ github.head_ref }}
          add-paths: "deploy/sample-backup-configs/sample_config_hwdb.yaml"
          commit-message: "Update AGFI(s) [ci skip]"
          body: "Update AGFI(s) for PR #${{ github.event.pull_request.number }}"
          branch-suffix: random
          title: "Update AGFI(s) for PR #${{ github.event.pull_request.number }} (`${{ github.head_ref }}`)"

  documentation-check:
    name: documentation-check
    needs: change-filters
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v3
      - uses: conda-incubator/setup-miniconda@v2
        with:
          environment-file: conda-reqs/docs.yaml
          miniforge-version: latest
      - name: Check that documentation builds with selective warnings/errors
        run: |
          make -C docs html
          ! grep -v "ERROR: Undefined substitution referenced" warnings.txt          
      - name: Show error log and dump objects.inv from sphinx if failed
        if: ${{ failure() }}
        run: |
          python3 -m sphinx.ext.intersphinx docs/_build/html/objects.inv
          cat /tmp/sphinx-err*.log          
  cpp-lint:
    name: cpp-lint
    runs-on: ubuntu-20.04
    steps:
      # Clone the repository (shallow to save time).
      - name: Checkout sources
        uses: actions/checkout@v3
        with:
          fetch-depth: 2
          submodules: false

      # Checkout chipyard submodules.
      - name: Checkout chipyard
        run: |
          git submodule update --init target-design/chipyard
          cd target-design/chipyard
          git submodule update --init generators/testchipip          
      # Install clang linters
      - name: Install Clang linters
        run: |
          sudo apt update
          sudo apt install clang-format clang-tidy-12 parallel -y          
      # Run 'clang-format', comparing against the base commit hash.
      # If anything got reformatted, fail and output a patch.
      - name: Run clang-format
        run: |
          git fetch --recurse-submodules=no origin ${{ github.base_ref }}
          DIFF_COMMIT=$(git rev-parse origin/${{ github.base_ref }})
          git clang-format $DIFF_COMMIT
          git diff --ignore-submodules > clang-format.patch
          if [ -s clang-format.patch ]; then
            echo "error: clang-format had to fix the following files:"
            git diff --ignore-submodules --name-only
            echo "----- 8< ---- PATCH ----- 8< ----"
            cat clang-format.patch
            echo "----- 8< ---- PATCH ----- 8< ----"
            git checkout .
            exit 1
          fi          
      # Run clang-tidy on the entire codebase. Error will be logged.
      - name: Run clang-tidy
        run: |
          export FIRESIM_ENV_SOURCED=1
          export FIRESIM_STANDALONE=1
          make -C sim clang-tidy          

  ### Local FPGAs CI ###

  setup-local-fpga-repo:
    name: setup-local-fpga-repo
    runs-on: local-fpga
    steps:
      # This forces a fresh clone of the repo during the `checkout` step
      # to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
      - name: Delete old checkout
        run: |
            ls -alh .
            rm -rf ${{ github.workspace }}/* || true
            rm -rf ${{ github.workspace }}/.* || true
            ls -alh .            
      - uses: actions/checkout@v3
      - name: Setup repo copy
        run: |
          mkdir -p $(dirname ${{ env.REMOTE_WORK_DIR }})
          git clone ${{ github.workspace }} ${{ env.REMOTE_WORK_DIR }}          
      - name: Setup repo
        run: |
          cd ${{ env.REMOTE_WORK_DIR }}
          ./build-setup.sh --skip-validate
          source sourceme-manager.sh --skip-ssh-setup
          firesim managerinit --platform vitis
          cd sw/firesim-software && ./init-submodules.sh          

  run-vitis-check-docs-generated-components:
    name: run-vitis-check-docs-generated-components
    needs: [setup-local-fpga-repo]
    runs-on: local-fpga
    steps:
      - uses: actions/checkout@v3
      - name: Check Vitis docs components that require manual re-generation (e.g. config_runtime.yaml example)
        run: ./.github/scripts/check-vitis-docs-generated-components.py

  run-parallel-vcs-metasims-and-vitis-driver:
    name: run-parallel-vcs-metasims-and-vitis-driver
    needs: [setup-local-fpga-repo]
    runs-on: local-fpga
    steps:
      # This forces a fresh clone of the repo during the `checkout` step
      # to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
      - name: Delete old checkout
        run: |
          rm -rf ${{ github.workspace }}/* || true
          rm -rf ${{ github.workspace }}/.* || true          
      - uses: actions/checkout@v3
      - name: Run parallel VCS metasims
        run: .github/scripts/run-parallel-vcs-metasims.py
      - name: Build Vitis driver
        run: .github/scripts/build-vitis-driver.py

  run-basic-linux-poweroff-vitis:
    if: contains(github.event.pull_request.labels.*.name, 'ci:fpga-deploy')
    needs: [setup-local-fpga-repo]
    name: run-basic-linux-poweroff-vitis
    runs-on: local-fpga
    steps:
      # This forces a fresh clone of the repo during the `checkout` step
      # to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
      - name: Delete old checkout
        run: |
          rm -rf ${{ github.workspace }}/* || true
          rm -rf ${{ github.workspace }}/.* || true          
      - uses: actions/checkout@v3
      - name: Run simple linux poweroff test w/ vitis
        run: .github/scripts/run-linux-poweroff-vitis.py

  run-local-fpga-buildbitstream:
    if: contains(github.event.pull_request.labels.*.name, 'ci:local-fpga-buildbitstream-deploy')
    needs: [setup-local-fpga-repo]
    name: run-local-fpga-buildbitstream
    runs-on: local-fpga
    timeout-minutes: 1200
    steps:
      # This forces a fresh clone of the repo during the `checkout` step
      # to resolve stale submodule URLs. See https://github.com/ucb-bar/chipyard/pull/1156.
      - name: Delete old checkout
        run: |
          rm -rf ${{ github.workspace }}/* || true
          rm -rf ${{ github.workspace }}/.* || true          
      - uses: actions/checkout@v3
      - name: Run buildbitstream command and update sample local bitstreams
        run: .github/scripts/run-local-buildbitstreams.py
      - uses: peter-evans/create-pull-request@v5
        with:
          base: ${{ github.head_ref }}
          add-paths: "deploy/sample-backup-configs/sample_config_hwdb.yaml"
          commit-message: "Update local bitstream(s) [ci skip]"
          body: "Update local bitstream(s) for PR #${{ github.event.pull_request.number }}"
          branch-suffix: random
          title: "Update local bitstream(s) for PR #${{ github.event.pull_request.number }} (`${{ github.head_ref }}`)"

  cleanup-local-fpga-repo:
    name: cleanup-local-fpga-repo
    needs: [run-parallel-vcs-metasims-and-vitis-driver, run-basic-linux-poweroff-vitis, run-vitis-check-docs-generated-components, run-local-fpga-buildbitstream]
    # uses a separate runner to cleanup (irrespective, of other jobs cancelled, running, etc)
    runs-on: local-fpga-cleanup
    if: ${{ always() }}
    steps:
      - name: Delete repo copy
        run: rm -rf ${{ env.REMOTE_WORK_DIR }}