[HandshakeToHW] Add `handshake.memory` lowering (#4175)

include/circt/Dialect/Handshake/HandshakeOps.h

@@ -35,14 +35,14 @@
 namespace circt {
 namespace handshake {
 
-struct ExtMemLoadInterface {
+struct MemLoadInterface {
   unsigned index;
   mlir::Value addressIn;
   mlir::Value dataOut;
   mlir::Value doneOut;
 };
 
-struct ExtMemStoreInterface {
+struct MemStoreInterface {
   unsigned index;
   mlir::Value addressIn;
   mlir::Value dataIn;

include/circt/Dialect/Handshake/HandshakeOps.td

@@ -679,6 +679,12 @@ def MemoryOp : Handshake_Op<"memory", [
       ins "ValueRange":$operands, "int":$outputs, "int":$control_outputs, "bool":$lsq,
       "int":$id, "Value":$memref)>
   ];
+
+  let extraClassDeclaration = [{
+    llvm::SmallVector<handshake::MemLoadInterface> getLoadPorts();
+    llvm::SmallVector<handshake::MemStoreInterface> getStorePorts();
+  }];
+
   let assemblyFormat = "`[` `ld` `=` $ldCount `,` `st` `=`  $stCount `]` `(` $inputs `)` attr-dict `:` $memRefType `,` functional-type($inputs, $outputs)";
   let hasVerifier = 1;
 }
@@ -726,8 +732,8 @@ def ExternalMemoryOp : Handshake_Op<"extmemory", [
       return getMemref().getType().cast<mlir::MemRefType>();
     }
 
-    llvm::SmallVector<handshake::ExtMemLoadInterface> getLoadPorts();
-    llvm::SmallVector<handshake::ExtMemStoreInterface> getStorePorts();
+    llvm::SmallVector<handshake::MemLoadInterface> getLoadPorts();
+    llvm::SmallVector<handshake::MemStoreInterface> getStorePorts();
   }];
 }
 

include/circt/Dialect/Seq/SeqOps.td

@@ -130,6 +130,11 @@ def HLMemOp : SeqOp<"hlmem", [
     HLMemType getMemType() { return getHandle().getType().cast<HLMemType>(); }
   }];
 
+  let builders = [
+    OpBuilder<(ins "Value":$clk, "Value":$rst, "StringRef":$symName,
+                   "llvm::ArrayRef<int64_t>":$shape, "Type":$elementType)>
+  ];
+
   let assemblyFormat = "$sym_name $clk `,` $rst attr-dict `:` type($handle)";
 }
 
@@ -156,6 +161,9 @@ def ReadPortOp : SeqOp<"read", [
   );
   let results = (outs AnyType:$readData);
   let hasCustomAssemblyFormat = 1;
+  let builders = [OpBuilder<(
+    ins "Value":$memory, "ValueRange":$addresses, "Value":$rdEn, "unsigned":$latency)>
+  ];
 }
 
 def WritePortOp : SeqOp<"write", [

integration_test/Dialect/Handshake/tp_memory/tp_memory.mlir

@@ -11,6 +11,9 @@
 // RUN: hlstool %s --dynamic-firrtl --buffering-strategy=cycles --verilog --lowering-options=disallowLocalVariables > %t.sv && \
 // RUN: %PYTHON% %S/../cocotb_driver.py --objdir=%T --topLevel=top --pythonModule=tp_memory --pythonFolder=%S %t.sv 2>&1 | FileCheck %s
 
+// RUN: hlstool %s --sv-trace-iverilog --dynamic-hw --buffering-strategy=cycles --verilog --lowering-options=disallowLocalVariables > %t.sv && \
+// RUN: %PYTHON% %S/../cocotb_driver.py --objdir=%T --topLevel=top --pythonModule=tp_memory --pythonFolder=%S %t.sv 2>&1 | FileCheck %s
+
 // Locking the circt should yield the same result
 // RUN: hlstool %s --dynamic-firrtl --buffering-strategy=all --dynamic-parallelism=locking --verilog --lowering-options=disallowLocalVariables > %t.sv && \
 // RUN: %PYTHON% %S/../cocotb_driver.py --objdir=%T --topLevel=top --pythonModule=tp_memory --pythonFolder=%S %t.sv 2>&1 | FileCheck %s

lib/Conversion/HandshakeToHW/HandshakeToHW.cpp

@@ -528,6 +528,16 @@ struct RTLBuilder {
   // Bitwise 'not'.
   Value bNot(Value value, std::optional<StringRef> name = {}) {
     auto allOnes = constant(value.getType().getIntOrFloatBitWidth(), -1);
+    std::string inferedName;
+    if (!name) {
+      // Try to create a name from the input value.
+      if (auto valueName =
+              value.getDefiningOp()->getAttrOfType<StringAttr>("sv.namehint")) {
+        inferedName = ("not_" + valueName.getValue()).str();
+        name = inferedName;
+      }
+    }
+
     return buildNamedOp(
         [&]() { return b.create<comb::XorOp>(loc, value, allOnes); }, name);
 
@@ -1452,6 +1462,111 @@ public:
   };
 };
 
+class MemoryConversionPattern
+    : public HandshakeConversionPattern<handshake::MemoryOp> {
+public:
+  using HandshakeConversionPattern<
+      handshake::MemoryOp>::HandshakeConversionPattern;
+  void buildModule(handshake::MemoryOp op, BackedgeBuilder &bb, RTLBuilder &s,
+                   hw::HWModulePortAccessor &ports) const override {
+    auto loc = op.getLoc();
+
+    // Gather up the load and store ports.
+    auto unwrappedIO = this->unwrapIO(s, bb, ports);
+    struct LoadPort {
+      InputHandshake &addr;
+      OutputHandshake &data;
+      OutputHandshake &done;
+    };
+    struct StorePort {
+      InputHandshake &addr;
+      InputHandshake &data;
+      OutputHandshake &done;
+    };
+    SmallVector<LoadPort, 4> loadPorts;
+    SmallVector<StorePort, 4> storePorts;
+
+    unsigned stCount = op.getStCount();
+    unsigned ldCount = op.getLdCount();
+    for (unsigned i = 0, e = ldCount; i != e; ++i) {
+      LoadPort port = {unwrappedIO.inputs[stCount * 2 + i],
+                       unwrappedIO.outputs[i],
+                       unwrappedIO.outputs[ldCount + stCount + i]};
+      loadPorts.push_back(port);
+    }
+
+    for (unsigned i = 0, e = stCount; i != e; ++i) {
+      StorePort port = {unwrappedIO.inputs[i * 2 + 1],
+                        unwrappedIO.inputs[i * 2],
+                        unwrappedIO.outputs[ldCount + i]};
+      storePorts.push_back(port);
+    }
+
+    // used to drive the data wire of the control-only channels.
+    auto c0I0 = s.constant(0, 0);
+
+    auto cl2dim = llvm::Log2_64_Ceil(op.getMemRefType().getShape()[0]);
+    auto hlmem = s.b.create<seq::HLMemOp>(
+        loc, s.clk, s.rst, "_handshake_memory_" + std::to_string(op.getId()),
+        op.getMemRefType().getShape(), op.getMemRefType().getElementType());
+
+    // Create load ports...
+    for (auto &ld : loadPorts) {
+      llvm::SmallVector<Value> addresses = {s.truncate(ld.addr.data, cl2dim)};
+      auto readData = s.b.create<seq::ReadPortOp>(loc, hlmem.getHandle(),
+                                                  addresses, ld.addr.valid,
+                                                  /*latency=*/0);
+      ld.data.data->setValue(readData);
+      ld.done.data->setValue(c0I0);
+      // Create control fork for the load address valid and ready signals.
+      buildForkLogic(s, bb, ld.addr, {ld.data, ld.done});
+    }
+
+    // Create store ports...
+    for (auto &st : storePorts) {
+      // Create a register to buffer the valid path by 1 cycle, to match the
+      // write latency of 1.
+      auto writeValidBufferMuxBE = bb.get(s.b.getI1Type());
+      auto writeValidBuffer =
+          s.reg("writeValidBuffer", writeValidBufferMuxBE, s.constant(1, 0));
+      st.done.valid->setValue(writeValidBuffer);
+      st.done.data->setValue(c0I0);
+
+      // Create the logic for when both the buffered write valid signal and the
+      // store complete ready signal are asserted.
+      auto storeCompleted =
+          s.bAnd({st.done.ready, writeValidBuffer}, "storeCompleted");
+
+      // Create a signal for when the write valid buffer is empty or the output
+      // is ready.
+      auto notWriteValidBuffer = s.bNot(writeValidBuffer);
+      auto emptyOrComplete =
+          s.bOr({notWriteValidBuffer, storeCompleted}, "emptyOrComplete");
+
+      // Connect the gate to both the store address ready and store data ready
+      st.addr.ready->setValue(emptyOrComplete);
+      st.data.ready->setValue(emptyOrComplete);
+
+      // Create a wire for when both the store address and data are valid.
+      auto writeValid = s.bAnd({st.addr.valid, st.data.valid}, "writeValid");
+
+      // Create a mux that drives the buffer input. If the emptyOrComplete
+      // signal is asserted, the mux selects the writeValid signal. Otherwise,
+      // it selects the buffer output, keeping the output registered until the
+      // emptyOrComplete signal is asserted.
+      writeValidBufferMuxBE.setValue(
+          s.mux(emptyOrComplete, {writeValidBuffer, writeValid}));
+
+      // Instantiate the write port operation - truncate address width to memory
+      // width.
+      llvm::SmallVector<Value> addresses = {s.truncate(st.addr.data, cl2dim)};
+      s.b.create<seq::WritePortOp>(loc, hlmem.getHandle(), addresses,
+                                   st.data.data, writeValid,
+                                   /*latency=*/1);
+    }
+  }
+}; // namespace
+
 class SinkConversionPattern : public HandshakeConversionPattern<SinkOp> {
 public:
   using HandshakeConversionPattern<SinkOp>::HandshakeConversionPattern;
@@ -1792,7 +1907,7 @@ static LogicalResult convertFuncOp(ESITypeConverter &typeConverter,
       ComparisonConversionPattern, BufferConversionPattern,
       SourceConversionPattern, SinkConversionPattern, ConstantConversionPattern,
       MergeConversionPattern, ControlMergeConversionPattern,
-      LoadConversionPattern, StoreConversionPattern,
+      LoadConversionPattern, StoreConversionPattern, MemoryConversionPattern,
       // Arith operations.
       ExtendConversionPattern<arith::ExtUIOp, /*signExtend=*/false>,
       ExtendConversionPattern<arith::ExtSIOp, /*signExtend=*/true>,

lib/Dialect/Handshake/HandshakeOps.cpp

@@ -105,6 +105,49 @@ static void printOp(OpAsmPrinter &p, Operation *op, bool explicitSize) {
 }
 } // namespace sost
 
+template <typename TMemOp>
+llvm::SmallVector<handshake::MemLoadInterface> getLoadPorts(TMemOp op) {
+  llvm::SmallVector<MemLoadInterface> ports;
+  // Memory interface refresher:
+  // Operands:
+  //   all stores (stdata1, staddr1, stdata2, staddr2, ...)
+  //   then all loads (ldaddr1, ldaddr2,...)
+  // Outputs: load addresses (lddata1, lddata2, ...), followed by all none
+  // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...)
+  unsigned stCount = op.getStCount();
+  unsigned ldCount = op.getLdCount();
+  for (unsigned i = 0, e = ldCount; i != e; ++i) {
+    MemLoadInterface ldif;
+    ldif.index = i;
+    ldif.addressIn = op.getInputs()[stCount * 2 + i];
+    ldif.dataOut = op.getResult(i);
+    ldif.doneOut = op.getResult(ldCount + stCount + i);
+    ports.push_back(ldif);
+  }
+  return ports;
+}
+
+template <typename TMemOp>
+llvm::SmallVector<handshake::MemStoreInterface> getStorePorts(TMemOp op) {
+  llvm::SmallVector<MemStoreInterface> ports;
+  // Memory interface refresher:
+  // Operands:
+  //   all stores (stdata1, staddr1, stdata2, staddr2, ...)
+  //   then all loads (ldaddr1, ldaddr2,...)
+  // Outputs: load data (lddata1, lddata2, ...), followed by all none
+  // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...)
+  unsigned ldCount = op.getLdCount();
+  for (unsigned i = 0, e = op.getStCount(); i != e; ++i) {
+    MemStoreInterface stif;
+    stif.index = i;
+    stif.dataIn = op.getInputs()[i * 2];
+    stif.addressIn = op.getInputs()[i * 2 + 1];
+    stif.doneOut = op.getResult(ldCount + i);
+    ports.push_back(stif);
+  }
+  return ports;
+}
+
 void ForkOp::build(OpBuilder &builder, OperationState &result, Value operand,
                    int outputs) {
   auto type = operand.getType();
@@ -1230,6 +1273,16 @@ void ExternalMemoryOp::build(OpBuilder &builder, OperationState &result,
   result.addAttribute("stCount", builder.getIntegerAttr(i32Type, stCount));
 }
 
+llvm::SmallVector<handshake::MemLoadInterface>
+ExternalMemoryOp::getLoadPorts() {
+  return ::getLoadPorts(*this);
+}
+
+llvm::SmallVector<handshake::MemStoreInterface>
+ExternalMemoryOp::getStorePorts() {
+  return ::getStorePorts(*this);
+}
+
 void MemoryOp::build(OpBuilder &builder, OperationState &result,
                      ValueRange operands, int outputs, int controlOutputs,
                      bool lsq, int id, Value memref) {
@@ -1256,47 +1309,12 @@ void MemoryOp::build(OpBuilder &builder, OperationState &result,
   }
 }
 
-llvm::SmallVector<handshake::ExtMemLoadInterface>
-ExternalMemoryOp::getLoadPorts() {
-  llvm::SmallVector<ExtMemLoadInterface> ports;
-  // Extmem interface refresher:
-  // Operands:
-  //   all stores (stdata1, staddr1, stdata2, staddr2, ...)
-  //   then all loads (ldaddr1, ldaddr2,...)
-  // Outputs: load addresses (lddata1, lddata2, ...), followed by all none
-  // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...)
-  unsigned stCount = getStCount();
-  unsigned ldCount = getLdCount();
-  for (unsigned i = 0, e = ldCount; i != e; ++i) {
-    ExtMemLoadInterface ldif;
-    ldif.index = i;
-    ldif.addressIn = getInputs()[stCount * 2 + i];
-    ldif.dataOut = getResult(i);
-    ldif.doneOut = getResult(ldCount + stCount + i);
-    ports.push_back(ldif);
-  }
-  return ports;
+llvm::SmallVector<handshake::MemLoadInterface> MemoryOp::getLoadPorts() {
+  return ::getLoadPorts(*this);
 }
 
-llvm::SmallVector<handshake::ExtMemStoreInterface>
-ExternalMemoryOp::getStorePorts() {
-  llvm::SmallVector<ExtMemStoreInterface> ports;
-  // Extmem interface refresher:
-  // Operands:
-  //   all stores (stdata1, staddr1, stdata2, staddr2, ...)
-  //   then all loads (ldaddr1, ldaddr2,...)
-  // Outputs: load data (lddata1, lddata2, ...), followed by all none
-  // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...)
-  unsigned ldCount = getLdCount();
-  for (unsigned i = 0, e = getStCount(); i != e; ++i) {
-    ExtMemStoreInterface stif;
-    stif.index = i;
-    stif.dataIn = getInputs()[i * 2];
-    stif.addressIn = getInputs()[i * 2 + 1];
-    stif.doneOut = getResult(ldCount + i);
-    ports.push_back(stif);
-  }
-  return ports;
+llvm::SmallVector<handshake::MemStoreInterface> MemoryOp::getStorePorts() {
+  return ::getStorePorts(*this);
 }
 
 bool handshake::MemoryOp::allocateMemory(

lib/Dialect/Handshake/Transforms/LowerExtmemToHW.cpp

@@ -267,7 +267,7 @@ static Value truncateToMemoryWidth(Location loc, OpBuilder &b, Value v,
 }
 
 static Value plumbLoadPort(Location loc, OpBuilder &b,
-                           const handshake::ExtMemLoadInterface &ldif,
+                           const handshake::MemLoadInterface &ldif,
                            Value loadData, MemRefType memrefType) {
   // We need to feed both the load data and the load done outputs.
   // Fork the extracted load data into two, and 'join' the second one to
@@ -287,7 +287,7 @@ static Value plumbLoadPort(Location loc, OpBuilder &b,
 }
 
 static Value plumbStorePort(Location loc, OpBuilder &b,
-                            const handshake::ExtMemStoreInterface &stif,
+                            const handshake::MemStoreInterface &stif,
                             Value done, Type outType, MemRefType memrefType) {
   stif.doneOut.replaceAllUsesWith(done);
   // Return the store address and data to be fed to the top-level output.

lib/Dialect/Seq/SeqOps.cpp

@@ -130,6 +130,13 @@ void ReadPortOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
   setNameFn(getReadData(), (memName + "_rdata").str());
 }
 
+void ReadPortOp::build(OpBuilder &builder, OperationState &result, Value memory,
+                       ValueRange addresses, Value rdEn, unsigned latency) {
+  auto memType = memory.getType().cast<seq::HLMemType>();
+  ReadPortOp::build(builder, result, memType.getElementType(), memory,
+                    addresses, rdEn, latency);
+}
+
 //===----------------------------------------------------------------------===//
 // WritePortOp
 //===----------------------------------------------------------------------===//
@@ -181,6 +188,13 @@ void HLMemOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
   setNameFn(getHandle(), getName());
 }
 
+void HLMemOp::build(OpBuilder &builder, OperationState &result, Value clk,
+                    Value rst, StringRef symName, llvm::ArrayRef<int64_t> shape,
+                    Type elementType) {
+  HLMemType t = HLMemType::get(builder.getContext(), shape, elementType);
+  HLMemOp::build(builder, result, t, clk, rst, symName);
+}
+
 //===----------------------------------------------------------------------===//
 // CompRegOp
 

tools/hlstool/hlstool.cpp

@@ -255,6 +255,7 @@ static void loadFIRRTLLoweringPipeline(OpPassManager &pm) {
 
 static void loadHWLoweringPipeline(OpPassManager &pm) {
   pm.addPass(createSimpleCanonicalizerPass());
+  pm.nest<hw::HWModuleOp>().addPass(circt::seq::createLowerSeqHLMemPass());
   pm.nest<hw::HWModuleOp>().addPass(seq::createSeqFIRRTLLowerToSVPass());
   pm.addPass(sv::createHWMemSimImplPass(false, false));
   pm.addPass(seq::createSeqLowerToSVPass());