[ESI][Runtime] Create a 'trace' accelerator backend (#6396)

Trace recording and playback could be very useful. Could also be used for design introspection and runtime testing (without cosim). This is the start of that system.

include/circt/Dialect/ESI/ESIPasses.td

@@ -40,8 +40,6 @@ def ESIBuildManifest : Pass<"esi-build-manifest", "mlir::ModuleOp"> {
   let constructor = "circt::esi::createESIBuildManifestPass()";
   let dependentDialects = ["circt::hw::HWDialect", "circt::sv::SVDialect"];
   let options = [
-    Option<"toFile", "to-file", "std::string",
-           "", "Write the manifest JSON directly to this file">,
     Option<"top", "top", "std::string",
            "", "Root module of the instance hierarchy">
   ];

integration_test/Dialect/ESI/runtime/loopback.mlir

@@ -3,6 +3,7 @@
 // RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest=top=top --esi-clean-metadata > %t4.mlir
 // RUN: circt-opt %t4.mlir --lower-esi-to-physical --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim --lower-seq-to-sv --export-split-verilog -o %t3.mlir
 // RUN: cd ..
+// RUN: %python %s.py trace w:%t6/esi_system_manifest.json
 // RUN: esi-cosim-runner.py --exec %s.py %t6/*.sv
 
 !sendI8 = !esi.bundle<[!esi.channel<i8> to "send"]>

lib/Dialect/ESI/Passes/ESIBuildManifest.cpp

@@ -88,13 +88,14 @@ void ESIBuildManifestPass::runOnOperation() {
   // JSONify the manifest.
   std::string jsonManifest = json();
 
-  // Append a verbatim with the manifest to the end of the module.
-  OpBuilder b = OpBuilder::atBlockEnd(&mod->getRegion(0).getBlocks().front());
-  auto verbatim = b.create<sv::VerbatimOp>(b.getUnknownLoc(),
-                                           StringAttr::get(ctxt, jsonManifest));
-  auto outputFileAttr =
-      hw::OutputFileAttr::getFromFilename(ctxt, "esi_system_manifest.json");
-  verbatim->setAttr("output_file", outputFileAttr);
+  std::error_code ec;
+  llvm::raw_fd_ostream os("esi_system_manifest.json", ec);
+  if (ec) {
+    mod->emitError() << "Failed to open file for writing: " << ec.message();
+    signalPassFailure();
+  } else {
+    os << jsonManifest << "\n";
+  }
 
   // If zlib is available, compress the manifest and append it to the module.
   SmallVector<uint8_t, 10 * 1024> compressedManifest;
@@ -104,19 +105,19 @@ void ESIBuildManifestPass::runOnOperation() {
         ArrayRef((uint8_t *)jsonManifest.data(), jsonManifest.length()),
         compressedManifest, llvm::compression::zlib::BestSizeCompression);
 
-    // Append a verbatim with the compressed manifest to the end of the module.
-    auto compressedVerbatim = b.create<sv::VerbatimOp>(
-        b.getUnknownLoc(),
-        StringAttr::get(ctxt, StringRef((char *)compressedManifest.data(),
-                                        compressedManifest.size())));
-    auto compressedOutputFileAttr = hw::OutputFileAttr::getFromFilename(
-        ctxt, "esi_system_manifest.json.zlib");
-    compressedVerbatim->setAttr("output_file", compressedOutputFileAttr);
+    llvm::raw_fd_ostream bos("esi_system_manifest.json.zlib", ec);
+    if (ec) {
+      mod->emitError() << "Failed to open compressed file for writing: "
+                       << ec.message();
+      signalPassFailure();
+    } else {
+      bos.write((char *)compressedManifest.data(), compressedManifest.size());
+    }
 
-    b.setInsertionPoint(symCache.getDefinition(appidRoot.getTopModuleRefAttr())
-                            ->getRegion(0)
-                            .front()
-                            .getTerminator());
+    OpBuilder b(symCache.getDefinition(appidRoot.getTopModuleRefAttr())
+                    ->getRegion(0)
+                    .front()
+                    .getTerminator());
     b.create<CompressedManifestOp>(
         b.getUnknownLoc(),
         BlobAttr::get(ctxt, ArrayRef<char>(reinterpret_cast<char *>(
@@ -126,30 +127,6 @@ void ESIBuildManifestPass::runOnOperation() {
     mod->emitError() << "zlib not available but required for manifest support";
     signalPassFailure();
   }
-
-  // If directed, write the manifest to a file. Mostly for debugging.
-  if (!toFile.empty()) {
-    std::error_code ec;
-    llvm::raw_fd_ostream os(toFile, ec);
-    if (ec) {
-      mod->emitError() << "Failed to open file for writing: " << ec.message();
-      signalPassFailure();
-    } else {
-      os << jsonManifest << "\n";
-    }
-
-    // If the compressed manifest is available, output it also.
-    if (!compressedManifest.empty()) {
-      llvm::raw_fd_ostream bos(toFile + ".zlib", ec);
-      if (ec) {
-        mod->emitError() << "Failed to open compressed file for writing: "
-                         << ec.message();
-        signalPassFailure();
-      } else {
-        bos.write((char *)compressedManifest.data(), compressedManifest.size());
-      }
-    }
-  }
 }
 
 void ESIBuildManifestPass::emitNode(llvm::json::OStream &j,

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -46,6 +46,8 @@ set(ESIRuntimeSources
   cpp/lib/Accelerator.cpp
   cpp/lib/Manifest.cpp
   cpp/lib/StdServices.cpp
+
+  cpp/lib/backends/Trace.cpp
 )
 set(ESIRuntimeLinkLibraries
   ZLIB::ZLIB

lib/Dialect/ESI/runtime/cpp/include/esi/backends/Trace.h

@@ -0,0 +1,72 @@
+//===- Trace.h - ESI trace backend ------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a specialization of the ESI C++ API (backend) for trace-based
+// Accelerator interactions. This means that it will have the capability to read
+// trace files recorded from interactions with an actual connection. It also has
+// a mode wherein it will write to a file (for sends) and produce random data
+// (for receives). Both modes are intended for debugging without a simulation.
+//
+// DO NOT EDIT!
+// This file is distributed as part of an ESI package. The source for this file
+// should always be modified within CIRCT (lib/dialect/ESI/runtime/cpp/).
+//
+//===----------------------------------------------------------------------===//
+
+// NOLINTNEXTLINE(llvm-header-guard)
+#ifndef ESI_BACKENDS_COSIM_H
+#define ESI_BACKENDS_COSIM_H
+
+#include "esi/Accelerator.h"
+
+#include <filesystem>
+#include <memory>
+
+namespace esi {
+namespace backends {
+namespace trace {
+
+/// Connect to an ESI simulation.
+class TraceAccelerator : public esi::Accelerator {
+public:
+  enum Mode {
+    // Write data sent to the accelerator to the trace file. Produce random
+    // garbage data for reads from the accelerator.
+    Write,
+
+    // Sent data to the accelerator is compared against the trace file's record.
+    // Data read from the accelerator is read from the trace file.
+    // TODO: Full trace mode not yet supported.
+    // Read
+  };
+
+  /// Create a trace-based accelerator backend.
+  /// \param mode The mode of operation. See Mode.
+  /// \param manifestJson The path to the manifest JSON file.
+  /// \param traceFile The path to the trace file. For 'Write' mode, this file
+  ///   is opened for writing. For 'Read' mode, this file is opened for reading.
+  TraceAccelerator(Mode mode, std::filesystem::path manifestJson,
+                   std::filesystem::path traceFile);
+
+  /// Parse the connection string and instantiate the accelerator. Format is:
+  /// "<mode>:<manifest path>[:<traceFile>]".
+  static std::unique_ptr<Accelerator> connect(std::string connectionString);
+
+protected:
+  virtual Service *createService(Service::Type service) override;
+
+private:
+  struct Impl;
+  std::unique_ptr<Impl> impl;
+};
+
+} // namespace trace
+} // namespace backends
+} // namespace esi
+
+#endif // ESI_BACKENDS_COSIM_H

lib/Dialect/ESI/runtime/cpp/lib/backends/Trace.cpp

@@ -0,0 +1,123 @@
+//===- Trace.cpp - Implementation of trace backend -----------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// DO NOT EDIT!
+// This file is distributed as part of an ESI package. The source for this file
+// should always be modified within CIRCT (lib/dialect/ESI/runtime/cpp/lib/).
+//
+//===----------------------------------------------------------------------===//
+
+#include "esi/backends/Trace.h"
+#include "esi/StdServices.h"
+
+#include <fstream>
+#include <iostream>
+#include <regex>
+
+using namespace std;
+
+using namespace esi;
+using namespace esi::services;
+using namespace esi::backends::trace;
+
+// We only support v1.
+constexpr uint32_t ESIVersion = 1;
+
+unique_ptr<Accelerator> TraceAccelerator::connect(string connectionString) {
+  string modeStr;
+  string manifestPath;
+  string traceFile = "trace.json";
+
+  // Parse the connection string.
+  // <mode>:<manifest path>[:<traceFile>]
+  regex connPattern("(\\w):([^:]+)(:(\\w+))?");
+  smatch match;
+  if (regex_search(connectionString, match, connPattern)) {
+    modeStr = match[1];
+    manifestPath = match[2];
+    if (match[3].matched)
+      traceFile = match[3];
+  } else {
+    throw runtime_error("connection string must be of the form "
+                        "'<mode>:<manifest path>[:<traceFile>]'");
+  }
+
+  // Parse the mode.
+  Mode mode;
+  if (modeStr == "w")
+    mode = Write;
+  else
+    throw runtime_error("unknown mode '" + modeStr + "'");
+
+  return std::make_unique<TraceAccelerator>(
+      mode, filesystem::path(manifestPath), filesystem::path(traceFile));
+}
+
+namespace {
+class TraceSysInfo : public SysInfo {
+public:
+  TraceSysInfo(std::filesystem::path manifestJson)
+      : manifestJson(manifestJson) {}
+
+  uint32_t esiVersion() const override { return ESIVersion; }
+
+  std::string jsonManifest() const override {
+    // Read in the whole json file and return it.
+    ifstream manifest(manifestJson);
+    if (!manifest.is_open())
+      throw runtime_error("failed to open manifest file '" +
+                          manifestJson.string() + "'");
+    stringstream buffer;
+    buffer << manifest.rdbuf();
+    manifest.close();
+    return buffer.str();
+  }
+
+  std::vector<uint8_t> compressedManifest() const override {
+    throw runtime_error("compressed manifest not supported by trace backend");
+  }
+
+private:
+  std::filesystem::path manifestJson;
+};
+} // namespace
+
+struct esi::backends::trace::TraceAccelerator::Impl {
+  Impl(Mode mode, std::filesystem::path manifestJson,
+       std::filesystem::path traceFile)
+      : mode(mode), manifestJson(manifestJson), traceFile(traceFile) {
+    if (!filesystem::exists(manifestJson))
+      throw runtime_error("manifest file '" + manifestJson.string() +
+                          "' does not exist");
+  }
+
+  Service *createService(Service::Type svcType);
+
+private:
+  Mode mode;
+  std::filesystem::path manifestJson;
+  std::filesystem::path traceFile;
+};
+
+Service *TraceAccelerator::Impl::createService(Service::Type svcType) {
+  if (svcType == typeid(SysInfo))
+    return new TraceSysInfo(manifestJson);
+  return nullptr;
+}
+
+TraceAccelerator::TraceAccelerator(Mode mode,
+                                   std::filesystem::path manifestJson,
+                                   std::filesystem::path traceFile) {
+  impl = std::make_unique<Impl>(mode, manifestJson, traceFile);
+}
+
+Service *TraceAccelerator::createService(Service::Type svcType) {
+  return impl->createService(svcType);
+}
+
+REGISTER_ACCELERATOR("trace", TraceAccelerator);

test/Dialect/ESI/manifest.mlir

@@ -1,7 +1,7 @@
 // REQUIRES: zlib
-// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest="top=top to-file=%t1.json" > %t1.mlir 
+// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest="top=top" > %t1.mlir 
 // RUN: circt-opt %t1.mlir | FileCheck --check-prefix=HIER %s
-// RUN: FileCheck --input-file=%t1.json %s
+// RUN: FileCheck --input-file=esi_system_manifest.json %s
 // RUN: circt-opt %t1.mlir --esi-clean-metadata --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim | FileCheck --check-prefix=HW %s
 
 hw.type_scope @__hw_typedecls {