[ESI] Basic Python runtime for manifests (#6371)

Infrastructure for Python runtime. Loads and parses the manifest.

integration_test/CMakeLists.txt

@@ -32,8 +32,8 @@ get_target_property(ESI_COLLATERAL_PATH esi-collateral BINARY_DIR)
 llvm_canonicalize_cmake_booleans(ESI_RUNTIME)
 if (ESI_RUNTIME)
   list(APPEND CIRCT_INTEGRATION_TEST_DEPENDS
-    esiaccel
     ESIRuntime
+    ESIPythonRuntime
   )
 endif()
 

integration_test/Dialect/ESI/runtime/basic_mmio.sv.py

@@ -1,10 +1,10 @@
-import esiaccel
+import esi
 import os
 import sys
 
 conn = f"{sys.argv[1]}:{sys.argv[2]}"
 
-acc = esiaccel.Accelerator.connect("cosim", conn)
+acc = esi.Accelerator("cosim", conn)
 mmio = acc.get_service_mmio()
 
 r = mmio.read(40)
@@ -26,7 +26,3 @@ try:
   assert False, "above should have thrown exception"
 except Exception:
   print("caught expected exception")
-
-# Crashes with "magic num not found", which is expected since this isn't
-# supported yet.
-# acc.sysinfo().esi_version()

integration_test/Dialect/ESI/runtime/loopback.mlir

@@ -0,0 +1,29 @@
+// REQUIRES: esi-cosim
+// RUN: rm -rf %t6 && mkdir %t6 && cd %t6
+// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest=top=top --esi-clean-metadata > %t4.mlir
+// RUN: circt-opt %t4.mlir --lower-esi-to-physical --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim --lower-seq-to-sv --export-split-verilog -o %t3.mlir
+// RUN: cd ..
+// RUN: esi-cosim-runner.py --exec %s.py %t6/*.sv
+
+!sendI8 = !esi.bundle<[!esi.channel<i8> to "send"]>
+!recvI8 = !esi.bundle<[!esi.channel<i8> to "recv"]>
+
+esi.service.decl @HostComms {
+  esi.service.to_server @Send : !sendI8
+  esi.service.to_client @Recv : !recvI8
+}
+
+hw.module @Loopback (in %clk: !seq.clock) {
+  %dataInBundle = esi.service.req.to_client <@HostComms::@Recv> (#esi.appid<"loopback_tohw">) {esi.appid=#esi.appid<"loopback_tohw">} : !recvI8
+  %dataOut = esi.bundle.unpack from %dataInBundle : !recvI8
+  %dataOutBundle = esi.bundle.pack %dataOut : !sendI8
+  esi.service.req.to_server %dataOutBundle -> <@HostComms::@Send> (#esi.appid<"loopback_fromhw">) : !sendI8
+}
+
+esi.manifest.sym @Loopback name "LoopbackIP" version "v0.0" summary "IP which simply echos bytes" {foo=1}
+
+hw.module @top(in %clk: !seq.clock, in %rst: i1) {
+  esi.service.instance #esi.appid<"cosim"> svc @HostComms impl as "cosim" (%clk, %rst) : (!seq.clock, i1) -> ()
+  hw.instance "m1" @Loopback (clk: %clk: !seq.clock) -> () {esi.appid=#esi.appid<"loopback_inst"[0]>}
+  hw.instance "m2" @Loopback (clk: %clk: !seq.clock) -> () {esi.appid=#esi.appid<"loopback_inst"[1]>}
+}

integration_test/Dialect/ESI/runtime/loopback.mlir.py

@@ -0,0 +1,10 @@
+import esi
+import os
+import sys
+
+conn = f"{sys.argv[1]}:{sys.argv[2]}"
+
+acc = esi.Accelerator("cosim", conn)
+
+assert acc.sysinfo().esi_version() == 1
+assert acc.manifest._manifest['api_version'] == 1

integration_test/lit.cfg.py

@@ -67,11 +67,6 @@ config.test_exec_root = os.path.join(config.circt_obj_root, 'integration_test')
 
 # Tweak the PATH to include the tools dir.
 llvm_config.with_environment('PATH', config.llvm_tools_dir, append_path=True)
-# Substitute '%l' with the path to the build lib dir.
-
-# Tweak the PYTHONPATH to include the lib dir. Some pybind11 modules there.
-llvm_config.with_environment('PYTHONPATH', [config.llvm_lib_dir],
-                             append_path=True)
 
 # Tweak the PYTHONPATH to include the binary dir.
 if config.bindings_python_enabled:
@@ -178,6 +173,10 @@ if ieee_sims and ieee_sims[-1][1] == config.iverilog_path:
 if config.esi_runtime == "1":
   config.available_features.add('esi-runtime')
 
+  llvm_config.with_environment('PYTHONPATH',
+                               [f"{config.esi_runtime_path}/python/"],
+                               append_path=True)
+
   # Enable ESI cosim tests if they have been built.
   if config.esi_cosim_path != "":
     config.available_features.add('esi-cosim')

integration_test/lit.site.cfg.py.in

@@ -50,6 +50,7 @@ config.clang_tidy_path = "@CLANG_TIDY_PATH@"
 config.have_systemc = "@HAVE_SYSTEMC@"
 config.esi_capnp = "@ESI_CAPNP@"
 config.esi_runtime = "@ESI_RUNTIME@"
+config.esi_runtime_path = "@ESIRuntimePath@"
 config.esi_collateral_path = "@ESI_COLLATERAL_PATH@"
 config.bindings_python_enabled = @CIRCT_BINDINGS_PYTHON_ENABLED@
 config.bindings_tcl_enabled = @CIRCT_BINDINGS_TCL_ENABLED@

lib/Dialect/ESI/CMakeLists.txt

@@ -80,7 +80,7 @@ set(ESI_RUNTIME_SRCS
   runtime/cpp/lib/StdServices.cpp
   runtime/cpp/lib/backends/Cosim.cpp
   runtime/cpp/tools/esiquery.cpp
-  runtime/python/esi/esiaccel.cpp
+  runtime/python/esi/esiCppAccel.cpp
   runtime/cosim/CMakeLists.txt
   runtime/cosim/Cosim_DpiPkg.sv
   runtime/cosim/Cosim_Endpoint.sv

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -29,13 +29,21 @@ cmake_minimum_required(VERSION 3.20)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED YES)
 
+find_package(ZLIB REQUIRED)
+
 include_directories(cpp/include)
 
 set(ESIRuntimeSources
   cpp/lib/Accelerator.cpp
   cpp/lib/StdServices.cpp
 )
-set(ESIRuntimeLinkLibraries)
+set(ESIRuntimeLinkLibraries
+  ZLIB::ZLIB
+)
+set(ESIPythonRuntimeSources
+  python/esi/__init__.py
+  python/esi/accelerator.py
+)
 
 # If Cap'nProto hasn't been explicitly disabled, find it.
 option(CAPNP_DISABLE "Disable Cap'nProto (needed for cosimulation).")
@@ -74,6 +82,7 @@ if(CapnProto_FOUND)
   )
 endif()
 
+
 # The core API. For now, compile the backends into it directly.
 # TODO: make this a plugin architecture.
 add_library(ESIRuntime SHARED
@@ -87,6 +96,10 @@ add_executable(esiquery
 )
 target_link_libraries(esiquery PRIVATE ESIRuntime)
 
+# Global variable for the path to the ESI runtime for use by tests.
+set(ESIRuntimePath "${CMAKE_CURRENT_BINARY_DIR}"
+  CACHE INTERNAL "Path to ESI runtime" FORCE)
+
 # Pybind11 is used to wrap the ESIRuntime APIs.
 find_package(Python COMPONENTS Interpreter Development)
 if(Python_FOUND)
@@ -113,7 +126,35 @@ if(Python_FOUND)
   if (NOT pybind11_FOUND)
     message (STATUS "Could not find pybind11. Disabling Python API.")
   else()
-    pybind11_add_module(esiaccel python/esi/esiaccel.cpp)
-    target_link_libraries(esiaccel PRIVATE ESIRuntime)
+    # Compile Pybind11 module and copy to the correct python directory.
+    pybind11_add_module(esiCppAccel python/esi/esiCppAccel.cpp)
+    target_link_libraries(esiCppAccel PRIVATE ESIRuntime)
+    add_custom_command(
+        TARGET esiCppAccel
+        POST_BUILD
+        COMMAND ${CMAKE_COMMAND} -E copy
+            $<TARGET_FILE:esiCppAccel>
+            "${CMAKE_CURRENT_BINARY_DIR}/python/esi/$<TARGET_FILE_NAME:esiCppAccel>"
+    )
+
+    # Copy each of the Python sources to the build dir.
+    foreach(pysrc ${ESIPythonRuntimeSources})
+      add_custom_command(
+          OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${pysrc}
+          COMMAND ${CMAKE_COMMAND} -E copy
+              ${CMAKE_CURRENT_SOURCE_DIR}/${pysrc}
+              ${CMAKE_CURRENT_BINARY_DIR}/${pysrc}
+          DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${pysrc}
+      )
+    endforeach()
+
+    # Custom target for the Python runtime just aggregates the python sources
+    # and Pybind11 module.
+    add_custom_target(ESIPythonRuntime
+      DEPENDS
+        ${ESIPythonRuntimeSources}
+        esiCppAccel
+    )
+
   endif()
 endif()

lib/Dialect/ESI/runtime/cosim/CosimDpi.capnp

@@ -16,7 +16,7 @@
 @0x9fd65fec6e2d2779;
 
 # The primary interface exposed by an ESI cosim simulation.
-interface CosimDpiServer @0x85e029b5352bcdb5 {
+interface CosimDpiServer @0xe3d7f70c7065c46a {
   # List all the registered endpoints.
   list @0 () -> (ifaces :List(EsiDpiInterfaceDesc));
   # Open one of them. Specify both the send and recv data types if want type
@@ -24,7 +24,7 @@ interface CosimDpiServer @0x85e029b5352bcdb5 {
   open @1 [S, T] (iface :EsiDpiInterfaceDesc) -> (iface :EsiDpiEndpoint);
 
   # Get the zlib-compressed JSON system manifest.
-  getCompressedManifest @2 () -> (compressedManifest :Data);
+  getCompressedManifest @2 () -> (version :Int32, compressedManifest :Data);
 
   # Create a low level interface into the simulation.
   openLowLevel @3 () -> (lowLevel :EsiLowLevel);

lib/Dialect/ESI/runtime/cosim/Cosim_DpiPkg.sv

@@ -78,6 +78,7 @@ import "DPI-C" sv2cCosimserverEpTryGet =
 
 import "DPI-C" sv2cCosimserverSetManifest =
   function void cosim_set_manifest(
+    input int unsigned esi_version,
     input byte unsigned compressed_manifest[]
   );
 

lib/Dialect/ESI/runtime/cosim/Cosim_Manifest.sv

@@ -15,12 +15,13 @@ import Cosim_DpiPkg::*;
 
 module Cosim_Manifest
 #(
-  parameter int COMPRESSED_MANIFEST_SIZE = 0
+  parameter int COMPRESSED_MANIFEST_SIZE = 0,
+  parameter int unsigned ESI_VERSION = 1
 )(
   input byte unsigned compressed_manifest[COMPRESSED_MANIFEST_SIZE]
 );
 
   always@(compressed_manifest)
-    cosim_set_manifest(compressed_manifest);
+    cosim_set_manifest(ESI_VERSION, compressed_manifest);
 
 endmodule

lib/Dialect/ESI/runtime/cosim/cosim_dpi_server/DpiEntryPoints.cpp

@@ -28,7 +28,6 @@ using namespace esi::cosim;
 /// If non-null, log to this file. Protected by 'serverMutex`.
 static FILE *logFile;
 static RpcServer *server = nullptr;
-static std::vector<uint8_t> manifest;
 static std::mutex serverMutex;
 
 // ---- Helper functions ----
@@ -261,8 +260,10 @@ DPI int sv2cCosimserverInit() {
 // ---- Manifest DPI entry points ----
 
 DPI void
-sv2cCosimserverSetManifest(const svOpenArrayHandle compressedManifest) {
-  sv2cCosimserverInit();
+sv2cCosimserverSetManifest(unsigned int esiVersion,
+                           const svOpenArrayHandle compressedManifest) {
+  if (server == nullptr)
+    sv2cCosimserverInit();
 
   if (validateSvOpenArray(compressedManifest, sizeof(int8_t)) != 0) {
     printf("ERROR: DPI-func=%s line=%d event=invalid-sv-array\n", __func__,
@@ -276,7 +277,9 @@ sv2cCosimserverSetManifest(const svOpenArrayHandle compressedManifest) {
   for (int i = 0; i < size; ++i) {
     blob[i] = *(char *)svGetArrElemPtr1(compressedManifest, i);
   }
-  server->setManifest(blob);
+  printf("[cosim] Setting manifest (esiVersion=%d, size=%d)\n", esiVersion,
+         size);
+  server->setManifest(esiVersion, blob);
 }
 
 // ---- Low-level cosim DPI entry points ----

lib/Dialect/ESI/runtime/cosim/cosim_dpi_server/Server.cpp

@@ -87,10 +87,12 @@ class CosimServer final : public CosimDpiServer::Server {
   /// The registry of endpoints. The RpcServer class owns this.
   EndpointRegistry ®
   LowLevel &lowLevelBridge;
+  const unsigned int &esiVersion;
   const std::vector<uint8_t> &compressedManifest;
 
 public:
   CosimServer(EndpointRegistry &reg, LowLevel &lowLevelBridge,
+              const unsigned int &esiVersion,
               const std::vector<uint8_t> &compressedManifest);
 
   /// List all the registered interfaces.
@@ -194,9 +196,12 @@ kj::Promise<void> LowLevelServer::writeMMIO(WriteMMIOContext context) {
 /// ----- CosimServer definitions.
 
 CosimServer::CosimServer(EndpointRegistry &reg, LowLevel &lowLevelBridge,
+                         const unsigned int &esiVersion,
                          const std::vector<uint8_t> &compressedManifest)
-    : reg(reg), lowLevelBridge(lowLevelBridge),
-      compressedManifest(compressedManifest) {}
+    : reg(reg), lowLevelBridge(lowLevelBridge), esiVersion(esiVersion),
+      compressedManifest(compressedManifest) {
+  printf("version: %d\n", esiVersion);
+}
 
 kj::Promise<void> CosimServer::list(ListContext context) {
   auto ifaces = context.getResults().initIfaces((unsigned int)reg.size());
@@ -224,6 +229,7 @@ kj::Promise<void> CosimServer::open(OpenContext ctxt) {
 
 kj::Promise<void>
 CosimServer::getCompressedManifest(GetCompressedManifestContext ctxt) {
+  ctxt.getResults().setVersion(esiVersion);
   ctxt.getResults().setCompressedManifest(
       Data::Reader(compressedManifest.data(), compressedManifest.size()));
   return kj::READY_NOW;
@@ -250,9 +256,10 @@ static void writePort(uint16_t port) {
 }
 
 void RpcServer::mainLoop(uint16_t port) {
-  capnp::EzRpcServer rpcServer(
-      kj::heap<CosimServer>(endpoints, lowLevelBridge, compressedManifest),
-      /* bindAddress */ "*", port);
+  capnp::EzRpcServer rpcServer(kj::heap<CosimServer>(endpoints, lowLevelBridge,
+                                                     esiVersion,
+                                                     compressedManifest),
+                               /* bindAddress */ "*", port);
   auto &waitScope = rpcServer.getWaitScope();
   // If port is 0, ExRpcSever selects one and we have to wait to get the port.
   if (port == 0) {

lib/Dialect/ESI/runtime/cosim/include/cosim/Server.h

@@ -39,7 +39,9 @@ public:
   void run(uint16_t port);
   void stop();
 
-  void setManifest(const std::vector<uint8_t> &manifest) {
+  void setManifest(unsigned int esiVersion,
+                   const std::vector<uint8_t> &manifest) {
+    this->esiVersion = esiVersion;
     compressedManifest = manifest;
   }
 
@@ -53,6 +55,7 @@ private:
   volatile bool stopSig;
   std::mutex m;
 
+  unsigned int esiVersion;
   std::vector<uint8_t> compressedManifest;
 };
 

lib/Dialect/ESI/runtime/cosim/include/cosim/dpi.h

@@ -47,7 +47,8 @@ DPI int sv2cCosimserverInit();
 DPI void sv2cCosimserverFinish();
 
 /// Set the system zlib-compressed manifest.
-DPI void sv2cCosimserverSetManifest(const svOpenArrayHandle compressedManifest);
+DPI void sv2cCosimserverSetManifest(unsigned int esiVersion,
+                                    const svOpenArrayHandle compressedManifest);
 
 /// Register an MMIO module. Just checks that there is only one instantiated.
 DPI int sv2cCosimserverMMIORegister();

lib/Dialect/ESI/runtime/cpp/include/esi/StdServices.h

@@ -36,7 +36,10 @@ public:
   virtual uint32_t esiVersion() const = 0;
 
   /// Return the JSON-formatted system manifest.
-  virtual std::string rawJsonManifest() const = 0;
+  virtual std::string jsonManifest() const;
+
+  /// Return the zlib compressed JSON system manifest.
+  virtual std::vector<uint8_t> compressedManifest() const = 0;
 };
 
 class MMIO : public Service {
@@ -54,8 +57,8 @@ public:
   /// Get the ESI version number to check version compatibility.
   uint32_t esiVersion() const override;
 
-  /// Return the JSON-formatted system manifest.
-  std::string rawJsonManifest() const override;
+  /// Return the zlib compressed JSON system manifest.
+  virtual std::vector<uint8_t> compressedManifest() const override;
 
 private:
   const MMIO *mmio;

lib/Dialect/ESI/runtime/cpp/lib/StdServices.cpp

@@ -15,11 +15,29 @@
 
 #include "esi/StdServices.h"
 
+#include "zlib.h"
+
+#include <cassert>
 #include <stdexcept>
 
 using namespace esi;
 using namespace esi::services;
 
+// Allocate 10MB for the uncompressed manifest. This should be plenty.
+constexpr uint32_t MAX_MANIFEST_SIZE = 10 << 20;
+/// Get the compressed manifest, uncompress, and return it.
+std::string SysInfo::jsonManifest() const {
+  std::vector<uint8_t> compressed = compressedManifest();
+  std::vector<Bytef> dst(MAX_MANIFEST_SIZE);
+  uLongf dstSize = MAX_MANIFEST_SIZE;
+  int rc =
+      uncompress(dst.data(), &dstSize, compressed.data(), compressed.size());
+  if (rc != Z_OK)
+    throw std::runtime_error("zlib uncompress failed with rc=" +
+                             std::to_string(rc));
+  return std::string(reinterpret_cast<char *>(dst.data()), dstSize);
+}
+
 MMIOSysInfo::MMIOSysInfo(const MMIO *mmio) : mmio(mmio) {}
 
 uint32_t MMIOSysInfo::esiVersion() const {
@@ -30,4 +48,6 @@ uint32_t MMIOSysInfo::esiVersion() const {
   return mmio->read(VersionNumberOffset);
 }
 
-std::string MMIOSysInfo::rawJsonManifest() const { return ""; }
+std::vector<uint8_t> MMIOSysInfo::compressedManifest() const {
+  assert(false && "Not implemented");
+}

lib/Dialect/ESI/runtime/cpp/lib/backends/Cosim.cpp

@@ -105,11 +105,37 @@ private:
 };
 } // namespace
 
+namespace {
+class CosimSysInfo : public SysInfo {
+public:
+  CosimSysInfo(CosimDpiServer::Client &client, kj::WaitScope &waitScope)
+      : client(client), waitScope(waitScope) {}
+
+  uint32_t esiVersion() const override {
+    auto maniResp =
+        client.getCompressedManifestRequest().send().wait(waitScope);
+    return maniResp.getVersion();
+  }
+
+  std::vector<uint8_t> compressedManifest() const override {
+    auto maniResp =
+        client.getCompressedManifestRequest().send().wait(waitScope);
+    capnp::Data::Reader data = maniResp.getCompressedManifest();
+    return std::vector<uint8_t>(data.begin(), data.end());
+  }
+
+private:
+  CosimDpiServer::Client &client;
+  kj::WaitScope &waitScope;
+};
+} // namespace
+
 Service *CosimAccelerator::createService(Service::Type svcType) {
   if (svcType == typeid(MMIO))
     return new CosimMMIO(impl->lowLevel, impl->waitScope);
   else if (svcType == typeid(SysInfo))
-    return new MMIOSysInfo(getService<MMIO>());
+    // return new MMIOSysInfo(getService<MMIO>());
+    return new CosimSysInfo(impl->cosim, impl->waitScope);
   return nullptr;
 }
 

lib/Dialect/ESI/runtime/python/esi/__init__.py

@@ -0,0 +1,5 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from .accelerator import Accelerator

lib/Dialect/ESI/runtime/python/esi/accelerator.py

@@ -0,0 +1,24 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+from .esiCppAccel import Accelerator as CppAccelerator
+
+import json
+
+
+class Accelerator(CppAccelerator):
+  """A connection to an ESI accelerator."""
+
+  @property
+  def manifest(self) -> "AcceleratorManifest":
+    """Get and parse the accelerator manifest."""
+    return AcceleratorManifest(self)
+
+
+class AcceleratorManifest:
+  """An accelerator manifest. Essential for interacting with an accelerator."""
+
+  def __init__(self, accel: Accelerator) -> None:
+    self.accel = accel
+    self._manifest = json.loads(accel.sysinfo().json_manifest())

lib/Dialect/ESI/runtime/python/esi/esiCppAccel.cpp

@@ -21,10 +21,9 @@ using namespace esi;
 using namespace esi::services;
 
 // NOLINTNEXTLINE(readability-identifier-naming)
-PYBIND11_MODULE(esiaccel, m) {
+PYBIND11_MODULE(esiCppAccel, m) {
   py::class_<Accelerator>(m, "Accelerator")
-      .def_static("connect", &registry::connect,
-                  py::return_value_policy::take_ownership)
+      .def(py::init(&registry::connect))
       .def("sysinfo", &Accelerator::getService<SysInfo>,
            py::return_value_policy::reference_internal)
       .def("get_service_mmio", &Accelerator::getService<services::MMIO>,
@@ -32,7 +31,7 @@ PYBIND11_MODULE(esiaccel, m) {
 
   py::class_<SysInfo>(m, "SysInfo")
       .def("esi_version", &SysInfo::esiVersion)
-      .def("raw_json_manifest", &SysInfo::rawJsonManifest);
+      .def("json_manifest", &SysInfo::jsonManifest);
 
   py::class_<services::MMIO>(m, "MMIO")
       .def("read", &services::MMIO::read)