Merge pull request #889 from firesim/fased-reset-intialized-config

Bake-in FASED default runtime configuration into hardware

sim/firesim-lib/src/test/scala/TestSuiteCommon.scala

@@ -2,10 +2,21 @@
 package firesim
 
 import java.io.File
+import scala.io.Source
 import scala.sys.process.{stringSeqToProcess, ProcessLogger}
 
 /**
-  * NB: not thread-safe
+  * An base class for implementing FireSim integration tests that call out to the Make 
+  * buildsystem. These tests typically have three steps whose results are tracked by scalatest:
+  * 1) Elaborate the target and compile it through golden gate (ElaborateAndCompile)
+  * 2) Compile a metasimulator for the generated RTL (compileMlSimulator)
+  * 3) Run the metasimulator. Running a metasimualtion is somewaht
+  *    target-specific and is handled differently by different subclasses.
+  *
+  * Some tests inspect the simulation outputs, or run other simulators. See the
+  * [[TutorialSuite]] for examples of that.
+  *
+  * NB: Not thread-safe.
   */
 abstract class TestSuiteCommon extends org.scalatest.flatspec.AnyFlatSpec {
 
@@ -98,9 +109,48 @@ abstract class TestSuiteCommon extends org.scalatest.flatspec.AnyFlatSpec {
       }
     }
   }
+
+  /**
+    * Extracts all lines in a file that begin with a specific prefix, removing
+    * extra whitespace between the prefix and the remainder of the line
+    *
+    * @param filename Input file
+    * @param prefix The per-line prefix to filter with
+    * @param linesToDrop Some number of matched lines to be removed
+    * @param headerLines An initial number of lines to drop before filtering.
+    *        Assertions, Printf output have a single line header.
+    *        MLsim stdout has some unused output, so set this to 1 by default
+    *
+    */
+  def extractLines(filename: File, prefix: String, linesToDrop: Int = 0, headerLines: Int = 1): Seq[String] = {
+    val lines = Source.fromFile(filename).getLines.toList.drop(headerLines)
+    lines.filter(_.startsWith(prefix))
+         .dropRight(linesToDrop)
+         .map(_.stripPrefix(prefix).replaceAll(" +", " "))
+  }
+
+
+  /**
+    * Diffs two sets of lines. Wrap calls to this function in a scalatest
+    * behavior spec. @param aName and @param bName can be used to provide more
+    * insightful assertion messages in scalatest reporting.
+    */
+  def diffLines(
+      aLines: Seq[String],
+      bLines: Seq[String],
+      aName: String = "Actual output",
+      bName: String = "Expected output"): Unit = {
+    assert(aLines.size == bLines.size && aLines.nonEmpty,
+      s"\n${aName} length (${aLines.size}) and ${bName} length (${bLines.size}) differ.")
+    for ((a, b) <- bLines.zip(aLines)) {
+      assert(a == b)
+    }
+  }
 }
 
-// HACK: Hijacks TestSuiteCommon to run the MIDAS unit tests
+/**
+  * Hijacks TestSuiteCommon (mostly for make related features) to run the synthesizable unit tests.
+  */
 abstract class MidasUnitTestSuite(unitTestConfig: String, shouldFail: Boolean = false) extends TestSuiteCommon {
   val targetTuple = unitTestConfig
   // GENERATED_DIR & OUTPUT_DIR are only used to properly invoke `make clean`

sim/midas/src/main/cc/bridges/fased_memory_timing_model.cc

@@ -62,8 +62,17 @@ FASEDMemoryTimingModel::FASEDMemoryTimingModel(
       size_t delimit_idx = sub_arg.find("=");
       size_t suffix_idx = sub_arg.find(suffix);
       std::string key = sub_arg.substr(0, suffix_idx).c_str();
-      int value = std::stoi(sub_arg.substr(delimit_idx+1).c_str());
-      model_configuration[key] = value;
+
+      // This is the only nullary plusarg supported by fased
+      // All other plusargs are key-value pairs that will be written to the bridge module
+      if (key == std::string("useHardwareDefaultRuntimeSettings")) {
+        require_all_runtime_settings = false;
+      } else if (suffix_idx == std::string::npos) {
+        throw std::runtime_error("[FASED] unknown nullary plusarg: " + key);
+      } else {
+        int value = std::stoi(sub_arg.substr(delimit_idx+1).c_str());
+        model_configuration[key] = value;
+      }
     }
   }
 
@@ -125,11 +134,16 @@ void FASEDMemoryTimingModel::init() {
       }
 
       if (!exclude) {
-        char buf[100];
-        sprintf(buf, "No value provided for configuration register: %s", pair.first.c_str());
-        throw std::runtime_error(buf);
+        if (require_all_runtime_settings) {
+          char buf[100];
+          sprintf(buf, "[FASED] No value provided for configuration register: %s", pair.first.c_str());
+          throw std::runtime_error(buf);
+        } else {
+          auto init_val = read(pair.first);
+          fprintf(stderr, "[FASED] Using hardware default of %u for configuration register %s\n", init_val, pair.first.c_str());
+        }
       } else {
-        fprintf(stderr, "Ignoring writeable register: %s\n", pair.first.c_str());
+        fprintf(stderr, "[FASED] Ignoring writeable register: %s\n", pair.first.c_str());
       }
     }
   }

sim/midas/src/main/cc/bridges/fased_memory_timing_model.h

@@ -122,6 +122,13 @@ private:
 
   bool has_latency_histograms() { return histograms.size() > 0; };
   size_t mem_size;
+  // By default, FASED requires that plus args for all timing model parameters
+  // are passed in to prevent accidental misconfigurations (ex. when
+  // DRAM timing parameters are passed to an LBP). When this is set, using the plus arg
+  // +mm_useHardwareDefaultRuntimeSettings_<idx>,
+  // the driver will instead use the hardware reset values (which map to the values emitted in the
+  // runtime.conf) and print those values to the log instead.
+  bool require_all_runtime_settings = true;
 };
 
 #endif // __FASED_MEMORY_TIMING_MODEL_H

sim/midas/src/main/scala/midas/models/dram/LastLevelCache.scala

@@ -172,12 +172,12 @@ class LLCModel(cfg: BaseConfig)(implicit p: Parameters) extends NastiModule()(p)
   val mshr_available = mshrs.exists({m: MSHR => m.available() })
   val mshr_next_idx = mshrs.indexWhere({ m: MSHR => m.available() })
 
-  // TODO: Put this on a switch
   val mshrs_allocated = mshrs.count({m: MSHR => m.valid})
-  assert((mshrs_allocated < io.settings.activeMSHRs) || !mshr_available,
+  assert((mshrs_allocated < RegNext(io.settings.activeMSHRs)) || !mshr_available,
     "Too many runtime MSHRs exposed given runtime programmable limit")
-  assert((mshrs_allocated === io.settings.activeMSHRs) || mshr_available,
-    "Too few runtime MSHRs exposed given runtime programmable limit")
+
+  assert((mshrs_allocated === RegNext(io.settings.activeMSHRs)) || mshr_available,
+    "Too few runtime MSHRs exposed given runtime programmable limit.")
 
   val s2_ar_mem = Module(new Queue(new NastiReadAddressChannel, 2))
   val s2_aw_mem = Module(new Queue(new NastiWriteAddressChannel, 2))

sim/midas/src/main/scala/midas/models/dram/LatencyBandwidthPipe.scala

@@ -67,6 +67,7 @@ class LatencyPipe(cfg: LatencyPipeConfig)(implicit p: Parameters) extends SplitT
   wResp.bits := writePipe.io.deq.bits.xaction
   writePipe.io.deq.ready := wResp.ready && writeDone
 
+  assert(writePipe.io.enq.ready || !newWReq, "LBP write latency pipe would overflow.")
 
   // ***** Read Latency Pipe *****
   val readPipe = Module(new Queue(new ReadPipeEntry, cfg.maxReads, flow = true))
@@ -79,5 +80,7 @@ class LatencyPipe(cfg: LatencyPipeConfig)(implicit p: Parameters) extends SplitT
   rResp.valid := readPipe.io.deq.valid && readDone
   rResp.bits := readPipe.io.deq.bits.xaction
   readPipe.io.deq.ready := rResp.ready && readDone
+
+  assert(readPipe.io.enq.ready || !nastiReq.ar.fire, "LBP read latency pipe would overflow.")
 }
 

sim/midas/src/main/scala/midas/models/dram/TimingModel.scala

@@ -241,4 +241,6 @@ abstract class SplitTransactionModel(cfg: BaseConfig)(implicit p: Parameters)
   awQueue.io.enq.bits := nastiReq.aw.bits
   awQueue.io.enq.valid := nastiReq.aw.fire()
   awQueue.io.deq.ready := newWReq
+  assert(awQueue.io.enq.ready || !nastiReq.aw.fire,
+    "AW queue in SplitTransaction timing model would overflow.")
 }

sim/midas/src/main/scala/midas/widgets/Widget.scala

@@ -80,22 +80,35 @@ abstract class WidgetImp(wrapper: Widget) extends LazyModuleImp(wrapper) {
   //   For inputs, generates a registers and binds that to the map
   //   For outputs, direct binds the wire to the map
   def attachIO(io: Record, prefix: String = ""): Unit = {
-    def innerAttachIO(node: Data, name: String): Unit = node match {
+
+    /**
+      * For FASED memory timing models, initalize programmable registers to defaults if provided.
+      * See [[midas.models.HasProgrammableRegisters]] for more detail.
+      */
+    def getInitValue(field: Bits, parent: Data): Option[UInt] = parent match {
+      case p: midas.models.HasProgrammableRegisters if p.regMap.isDefinedAt(field) =>
+        Some(p.regMap(field).default.U)
+      case _ => None
+    }
+
+    def innerAttachIO(node: Data, parent: Data, name: String): Unit = node match {
       case (b: Bits) => (DataMirror.directionOf(b): @unchecked) match {
         case ActualDirection.Output => attach(b, s"${name}", ReadOnly)
-        case ActualDirection.Input => genWOReg(b, name)
+        case ActualDirection.Input =>
+          genAndAttachReg(b, name, getInitValue(b, parent))
       }
       case (v: Vec[_]) => {
-        (v.zipWithIndex).foreach({ case (elm, idx) => innerAttachIO(elm, s"${name}_$idx")})
+        (v.zipWithIndex).foreach({ case (elm, idx) => innerAttachIO(elm, node, s"${name}_$idx")})
       }
       case (r: Record) => {
-        r.elements.foreach({ case (subName, elm) => innerAttachIO(elm, s"${name}_${subName}")})
+        r.elements.foreach({ case (subName, elm) => innerAttachIO(elm, node, s"${name}_${subName}")})
       }
       case _ => new RuntimeException("Cannot bind to this sort of node...")
     }
-    io.elements.foreach({ case (name, elm) => innerAttachIO(elm, s"${prefix}${name}")})
+    io.elements.foreach({ case (name, elm) => innerAttachIO(elm, io, s"${prefix}${name}")})
   }
 
+
   def attachDecoupledSink(channel: DecoupledIO[UInt], name: String): Int = {
     crRegistry.allocate(DecoupledSinkEntry(channel, name))
   }

sim/src/main/makefrag/fasedtests/Makefrag

@@ -82,17 +82,24 @@ vcs = $(GENERATED_DIR)/$(DESIGN)
 vcs_debug = $(GENERATED_DIR)/$(DESIGN)-debug
 xsim = $(GENERATED_DIR)/$(DESIGN)-$(PLATFORM)
 
+logfile = $(if $(LOGFILE),$(abspath $(LOGFILE)),$(OUTPUT_DIR)/$1.out)
+waveform = $(if $(WAVEFORM),$(abspath $(WAVEFORM)),$(OUTPUT_DIR)/$1.$2)
+
 run-verilator: $(verilator)
-	cd $(<D) && ./$(<F) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) 2> err
+	mkdir -p $(OUTPUT_DIR)
+	cd $(<D) && ./$(<F) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) 2> $(call logfile,verilator)
 
 run-verilator-debug: $(verilator_debug)
-	cd $(<D) && ./$(<F) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) 2> err
+	mkdir -p $(OUTPUT_DIR)
+	cd $(<D) && ./$(<F) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +waveform=$(call waveform,verilator,vcd) 2> $(call logfile,verilator)
 
 run-vcs: $(vcs)
-	cd $(<D) && ./$(<F) $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) 2> err
+	mkdir -p $(OUTPUT_DIR)
+	cd $(<D) && ./$(<F) $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) 2> $(call logfile,vcs)
 
 run-vcs-debug: $(vcs_debug)
-	cd $(<D) && ./$(<F) $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +waveform=waves.vpd 2> err
+	mkdir -p $(OUTPUT_DIR)
+	cd $(<D) && ./$(<F) $(vcs_args) $(COMMON_SIM_ARGS) $(MIDAS_LEVEL_SIM_ARGS) $(EXTRA_SIM_ARGS) +waveform=$(call waveform,vcs,vpd) 2> $(call logfile,vcs)
 
 .PHONY: run-xsim
 run-xsim: $(xsim)

sim/src/main/scala/fasedtests/Config.scala

@@ -54,6 +54,7 @@ class WithNTransactions(num: Int) extends Config((site, here, up) => {
   case NumTransactions => num
 })
 
+class NT10e3 extends WithNTransactions(1000)
 class NT10e5 extends WithNTransactions(100000)
 class NT10e6 extends WithNTransactions(1000000)
 class NT10e7 extends WithNTransactions(10000000)

sim/src/test/scala/fasedtests/FASEDTestSuite.scala

@@ -14,11 +14,42 @@ import freechips.rocketchip.system.DefaultTestSuites._
 
 import firesim.configs.LlcKey
 
+/** Different runtime-configuration modes extend this trait. See below.  */
+sealed trait RuntimeConfig {
+  def behaviorString: String
+}
+
+/** Use the .conf generated by GG by default. */
+case object DefaultRuntimeConfig extends RuntimeConfig {
+  val behaviorString = "with default runtime conf"
+}
+
+/** Provide no conf file. PlusArgs must be called out in the test class. */
+case object EmptyRuntimeConfig extends RuntimeConfig {
+  val behaviorString = "with no base runtime conf"
+}
+
+/** Specific an alternate path to a conf file. */
+case class  CustomRuntimeConfig(pathRelativeToSim: String) extends RuntimeConfig {
+  val behaviorString = s"with runtime conf ${pathRelativeToSim}"
+}
+
+/**
+  * A specialization of TestSuiteCommon for FASED-specific testing. Mostly
+  * handles differences in the makefrag vs midasexamples..
+  *
+  * @param topModuleClass DESIGN: target top-level module class
+  * @param targetConfigs  TARGET_CONFIG: config string to parameterize the target
+  * @param platformConfigs PLATFORM_CONFIG: config string to configure GG
+  * @param baseRuntimeConfig Default runtime conf handling for runtest
+  * @param additionalPlusArgs Non-standard plusargs to add to runTest invocations by default
+  */
 abstract class FASEDTest(
     topModuleClass: String,
     targetConfigs: String,
     platformConfigs: String,
-    N: Int = 8
+    baseRuntimeConfig: RuntimeConfig = DefaultRuntimeConfig,
+    additionalPlusArgs: Seq[String] = Seq(),
   ) extends firesim.TestSuiteCommon {
   import scala.concurrent.duration._
   import ExecutionContext.Implicits.global
@@ -33,24 +64,66 @@ abstract class FASEDTest(
     make((s"run-${backend}%s".format(if (debug) "-debug" else "") +: args):_*)
   }
 
-  def runTest(backend: String, debug: Boolean, args: Seq[String] = Nil, name: String = "pass") = {
-    compileMlSimulator(backend, debug)
+  def runTest(
+      backend: String,
+      debug: Boolean = false,
+      logFile: Option[File] = None,
+      baseRuntimeConfig: RuntimeConfig = baseRuntimeConfig,
+      additionalPlusArgs: Seq[String] = additionalPlusArgs,
+      additionalMakeArgs: Seq[String] = Seq(),
+      behaviorSpec: Option[String] = None) = {
+    val runtimeConfArg: Option[String] = baseRuntimeConfig match {
+      case DefaultRuntimeConfig => None
+      case EmptyRuntimeConfig => Some(s"SIM_RUNTIME_CONF=")
+      case CustomRuntimeConfig(path) => Some(s"SIM_RUNTIME_CONF=$path")
+    }
+    val plusArgs = Seq(s"""EXTRA_SIM_ARGS=${additionalPlusArgs.mkString(" ")}""")
+    val logArg = logFile.map { logName => s"LOGFILE=${logName}" }
+
+    val makeArgs =
+      runtimeConfArg ++:
+      plusArgs ++:
+      logArg ++:
+      additionalMakeArgs
+
+
     if (isCmdAvailable(backend)) {
-      it should s"run on $backend" in {
-        assert(invokeMlSimulator(backend, debug, args) == 0)
+      it should behaviorSpec.getOrElse(s"run on $backend") in {
+        assert(invokeMlSimulator(backend, debug, makeArgs) == 0)
       }
     }
   }
-  def runTests() {
-    runTest("verilator", false)
-    //runTest("vcs", true)
-  }
 
-  behavior of s"FASED Instance configured with ${platformConfigs} driven by target: ${topModuleClass}"
+  compileMlSimulator("verilator", false)
   runTest("verilator", false)
 }
 
 class AXI4FuzzerLBPTest extends FASEDTest("AXI4Fuzzer", "DefaultConfig", "DefaultF1Config")
+
+// Sanity checks that target output is the same when using the default runtime
+// configuration and the hardwired values.
+class CheckHardwiredValuesTest extends FASEDTest("AXI4Fuzzer", "NT10e3_AddrBits16_DefaultConfig", "DefaultF1Config") {
+  val logA = new File(s"$outDir/using-runtime-conf.out")
+  runTest(
+    "verilator",
+    logFile = Some(logA),
+    behaviorSpec = Some("run using a runtime.conf"))
+
+  val logB = new File(s"$outDir/using-hardwired-settings.out")
+  runTest(
+    "verilator",
+    logFile = Some(logB),
+    baseRuntimeConfig = EmptyRuntimeConfig,
+    additionalPlusArgs = Seq("+mm_useHardwareDefaultRuntimeSettings_0"),
+    behaviorSpec = Some("run using initialization values"))
+
+  "Initialization values for configuration registers" should "produce the same target behavior as using the default runtime.conf" in {
+    val aLines = extractLines(logA, "AXI4FuzzMaster_0", headerLines = 0)
+    val bLines = extractLines(logB, "AXI4FuzzMaster_0", headerLines = 0)
+    diffLines(aLines, bLines, logA.getName, logB.getName)
+  }
+}
+
 class AXI4FuzzerMultiChannelTest extends FASEDTest("AXI4Fuzzer", "FuzzMask3FFF_QuadFuzzer_QuadChannel_DefaultConfig", "DefaultF1Config")
 class AXI4FuzzerFCFSTest extends FASEDTest("AXI4Fuzzer", "FCFSConfig", "DefaultF1Config")
 class AXI4FuzzerFRFCFSTest extends FASEDTest("AXI4Fuzzer", "FRFCFSConfig", "DefaultF1Config")

sim/src/test/scala/midasexamples/TutorialSuite.scala

@@ -63,33 +63,6 @@ abstract class TutorialSuite(
     }
   }
 
-  /**
-    * Extracts all lines in a file that begin with a specific prefix, removing
-    * extra whitespace between the prefix and the remainder of the line
-    *
-    * @param filename Input file
-    * @param prefix The per-line prefix to filter with
-    * @param linesToDrop Some number of matched lines to be removed
-    * @param headerLines An initial number of lines to drop before filtering.
-    *        Assertions, Printf output have a single line header.
-    *        MLsim stdout has some unused output, so set this to 1 by default
-    *
-    */
-  def extractLines(filename: File, prefix: String, linesToDrop: Int = 0, headerLines: Int = 1): Seq[String] = {
-    val lines = Source.fromFile(filename).getLines.toList.drop(headerLines)
-    lines.filter(_.startsWith(prefix))
-         .dropRight(linesToDrop)
-         .map(_.stripPrefix(prefix).replaceAll(" +", " "))
-  }
-
-  def diffLines(expectedLines: Seq[String], actualLines: Seq[String]): Unit = {
-    assert(actualLines.size == expectedLines.size && actualLines.nonEmpty,
-      s"\nActual output had length ${actualLines.size}. Expected ${expectedLines.size}")
-    for ((vPrint, sPrint) <- expectedLines.zip(actualLines)) {
-      assert(sPrint == vPrint)
-    }
-  }
-
   // Checks that a bridge generated log in ${genDir}/${synthLog} matches output
   // generated directly by the RTL simulator (usually with printfs)
   def diffSynthesizedLog(synthLog: String,