[CI] Split elaboration into seperate step

.circleci/config.yml

@@ -6,7 +6,7 @@ executors:
       - image: firesim/firesim-ci:v1.0
         user: "centos"
     environment:
-        JVM_MEMORY: 1800M # Default JVM maximum heap limit
+        JVM_MEMORY: 3500M # Default JVM maximum heap limit
         LANG: en_US.UTF-8 # required by sbt when it sees boost directories
 
 commands:
@@ -75,16 +75,30 @@ commands:
       test-package:
         type: string
         default: "firesim.midasexamples"
+      separate-elaboration:
+        description: Runs separate elaboration step to avoid multiple invocations of SBT.
+        type: boolean
+        default: true
       timeout:
         type: string
         default: "120m"
     steps:
-        - run:
-            command: |
-                source env.sh
-                make -C sim TARGET_PROJECT=midasexamples sbt \
-                    SBT_COMMAND="project << parameters.project >>; testOnly << parameters.test-package >>.<< parameters.test-name >>"
-            no_output_timeout: << parameters.timeout >>
+      - when:
+          condition: <<parameters.separate-elaboration>>
+          steps:
+          - run:
+              command: |
+                  source env.sh
+                  make -C sim TARGET_PROJECT=midasexamples sbt \
+                      SBT_COMMAND="project << parameters.project >>; Test / runMain firesim.EmitCIElaborationScript elaborate.sh << parameters.test-package >>.<< parameters.test-name >>"
+                  bash -x sim/elaborate.sh || true
+              no_output_timeout: << parameters.timeout >>
+      - run:
+          command: |
+              source env.sh
+              make -C sim TARGET_PROJECT=midasexamples sbt \
+                  SBT_COMMAND="project << parameters.project >>; testOnly << parameters.test-package >>.<< parameters.test-name >> -- -Dci-skip-elaboration=true"
+          no_output_timeout: << parameters.timeout >>
 
   repo-setup:
     description: "Runs all baseline setup tasks up to scala compilation."
@@ -116,6 +130,7 @@ jobs:
            project: "midas"
            test-package: "*"
            test-name: "*"
+           separate-elaboration: false
 
   publish-scala-doc:
     executor: main-env

sim/firesim-lib/src/test/scala/EmitCIElaborationScript.scala

@@ -0,0 +1,30 @@
+//See LICENSE for license details.
+package firesim
+
+import java.io.File
+import java.io.FileWriter
+
+/**
+  * Instantiates a TestSuite and pulls out all of the required make calls to
+  * elaborate all tests.
+  *
+  * 1st argument: name of the output script
+  * 2nd argument: fully qualified suite name
+  */
+object EmitCIElaborationScript extends App {
+  override def main(args: Array[String]): Unit = {
+    assert(args.size == 2)
+    val Array(scriptName, className) = args
+    val inst = Class.forName(className).getConstructors().head.newInstance()
+
+    def recurse(suite: Any): Seq[String] = suite match {
+      case t: TestSuiteCommon => Seq(t.makeCommand(t.elaborateMakeTarget:_*).mkString(" "))
+      case t: org.scalatest.Suites => t.nestedSuites.flatMap(recurse)
+    }
+
+    val makeCommands = recurse(inst)
+    val writer = new FileWriter(new File(scriptName))
+    makeCommands.foreach { l =>  writer.write(l + "\n") }
+    writer.close
+  }
+}

sim/firesim-lib/src/test/scala/TestSuiteCommon.scala

@@ -4,6 +4,9 @@ package firesim
 import java.io.File
 import scala.sys.process.{stringSeqToProcess, ProcessLogger}
 
+/**
+  * NB: not thread-safe
+  */
 abstract class TestSuiteCommon extends org.scalatest.flatspec.AnyFlatSpec {
 
   def targetTuple: String
@@ -24,6 +27,21 @@ abstract class TestSuiteCommon extends org.scalatest.flatspec.AnyFlatSpec {
     }
   }
 
+  var ciSkipElaboration: Boolean = false
+  var transitiveFailure: Boolean = false
+
+	override def withFixture(test: NoArgTest) = {
+		// Perform setup
+    ciSkipElaboration = test.configMap.getOptional[String]("ci-skip-elaboration")
+      .map { _.toBoolean }
+      .getOrElse(false)
+    if (transitiveFailure) {
+      org.scalatest.Canceled("Due to prior failure")
+    } else {
+      super.withFixture(test)
+    }
+	}
+
   // These mirror those in the make files; invocation of the MIDAS compiler
   // is the one stage of the tests we don't invoke the Makefile for
   lazy val genDir  = new File(firesimDir, s"generated-src/${platformName}/${targetTuple}")
@@ -31,23 +49,53 @@ abstract class TestSuiteCommon extends org.scalatest.flatspec.AnyFlatSpec {
 
   implicit def toStr(f: File): String = f.toString replace (File.separator, "/")
 
+  // Defines a make target that will build all prerequistes for downstream
+  // tests that require a Scala invocation.
+  def elaborateMakeTarget: Seq[String] = Seq("compile")
+
+  def makeCommand(makeArgs: String*): Seq[String] = {
+    Seq("make", "-C", s"$firesimDir") ++ makeArgs.toSeq ++ commonMakeArgs ++ platformMakeArgs
+  }
+
   // Runs make passing default args to specify the right target design, project and platform
   def make(makeArgs: String*): Int = {
-    val cmd = Seq("make", "-C", s"$firesimDir") ++ makeArgs.toSeq ++ commonMakeArgs ++ platformMakeArgs
+    val cmd = makeCommand(makeArgs:_*)
     println("Running: %s".format(cmd mkString " "))
     cmd.!
   }
 
+  // As above, but if the RC is non-zero, cancels all downstream tests.  This
+  // is used to prevent re-invoking make on a target with a dependency on the
+  // result of this recipe. Which would lead to a second failure.
+  def makeCriticalDependency(makeArgs: String*): Int = {
+    val returnCode = make(makeArgs:_*)
+    transitiveFailure = returnCode != 0
+    returnCode
+  }
+
   def clean() { make("clean") }
   def mkdirs() { genDir.mkdirs; outDir.mkdirs }
 
   def isCmdAvailable(cmd: String) =
     Seq("which", cmd) ! ProcessLogger(_ => {}) == 0
 
+  // Running all scala-invocations required to take the design to verilog.
+  // Generally elaboration + GG compilation.
+  def elaborateAndCompile(behaviorDescription: String = "elaborate and compile through GG sucessfully") {
+    it should behaviorDescription in {
+      // Under CI, if make failed during elaboration we catch it here without
+      // attempting to rebuild
+      val target = (if (ciSkipElaboration) Seq("-q") else Seq()) ++ elaborateMakeTarget
+      assert(makeCriticalDependency(target:_*) == 0)
+    }
+  }
+
   // Compiles a MIDAS-level RTL simulator of the target
   def compileMlSimulator(b: String, debug: Boolean = false) {
     if (isCmdAvailable(b)) {
-      assert(make(s"$b%s".format(if (debug) "-debug" else "")) == 0)
+      it should s"compile sucessfully to ${b}" + { if (debug) " with waves enabled" else "" } in {
+        assert(makeCriticalDependency(s"$b%s".format(if (debug) "-debug" else "")) == 0)
+      }
     }
   }
 }
@@ -60,12 +108,14 @@ abstract class MidasUnitTestSuite(unitTestConfig: String, shouldFail: Boolean =
                            s"GENERATED_DIR=${genDir}",
                            s"OUTPUT_DIR=${outDir}")
 
+  // Use the default recipe which also will compile verilator since there's no
+  // separate target for just elaboration
+  override def elaborateMakeTarget = Seq("compile-midas-unittests")
   override lazy val genDir  = new File(s"generated-src/unittests/${targetTuple}")
   override lazy val outDir = new File(s"output/unittests/${targetTuple}")
 
   def runUnitTestSuite(backend: String, debug: Boolean = false) {
-    behavior of s"MIDAS unittest: ${unitTestConfig} running on ${backend}"
-    val testSpecString = if (shouldFail) "fail" else "pass"
+    val testSpecString = if (shouldFail) "fail" else "pass" + s" when running under ${backend}"
 
     if (isCmdAvailable(backend)) {
       lazy val result = make("run-midas-unittests%s".format(if (debug) "-debug" else ""),
@@ -78,8 +128,9 @@ abstract class MidasUnitTestSuite(unitTestConfig: String, shouldFail: Boolean =
     }
   }
 
-  clean
   mkdirs
+  behavior of s"MIDAS unittest: ${unitTestConfig}"
+  elaborateAndCompile("elaborate sucessfully")
   runUnitTestSuite("verilator")
 }
 

sim/src/test/scala/fasedtests/FASEDTestSuite.scala

@@ -46,7 +46,6 @@ abstract class FASEDTest(
     //runTest("vcs", true)
   }
 
-  clean
   behavior of s"FASED Instance configured with ${platformConfigs} driven by target: ${topModuleClass}"
   runTest("verilator", false)
 }

sim/src/test/scala/midasexamples/TutorialSuite.scala

@@ -38,9 +38,8 @@ abstract class TutorialSuite(
 
 
   def runTest(b: String, debug: Boolean = false) {
-    behavior of s"$targetName in $b"
     compileMlSimulator(b, debug)
-    val testEnv = "MIDAS-level simulation" + { if (debug) " with waves enabled" else "" }
+    val testEnv = s"${b} MIDAS-level simulation" + { if (debug) " with waves enabled" else "" }
     if (isCmdAvailable(b)) {
       it should s"pass in ${testEnv}" in {
         assert(run(b, debug, args = simulationArgs) == 0)
@@ -87,7 +86,9 @@ abstract class TutorialSuite(
     }
   }
 
-  clean
+  mkdirs()
+  behavior of s"$targetName"
+  elaborateAndCompile()
   runTest(backendSimulator)
 }