diff --git a/include/circt/Dialect/Handshake/HandshakeOps.h b/include/circt/Dialect/Handshake/HandshakeOps.h index dfb3130c75..388acd0884 100644 --- a/include/circt/Dialect/Handshake/HandshakeOps.h +++ b/include/circt/Dialect/Handshake/HandshakeOps.h @@ -35,14 +35,14 @@ namespace circt { namespace handshake { -struct ExtMemLoadInterface { +struct MemLoadInterface { unsigned index; mlir::Value addressIn; mlir::Value dataOut; mlir::Value doneOut; }; -struct ExtMemStoreInterface { +struct MemStoreInterface { unsigned index; mlir::Value addressIn; mlir::Value dataIn; diff --git a/include/circt/Dialect/Handshake/HandshakeOps.td b/include/circt/Dialect/Handshake/HandshakeOps.td index 1815e043ce..ddc76f4e0e 100644 --- a/include/circt/Dialect/Handshake/HandshakeOps.td +++ b/include/circt/Dialect/Handshake/HandshakeOps.td @@ -679,6 +679,12 @@ def MemoryOp : Handshake_Op<"memory", [ ins "ValueRange":$operands, "int":$outputs, "int":$control_outputs, "bool":$lsq, "int":$id, "Value":$memref)> ]; + + let extraClassDeclaration = [{ + llvm::SmallVector getLoadPorts(); + llvm::SmallVector getStorePorts(); + }]; + let assemblyFormat = "`[` `ld` `=` $ldCount `,` `st` `=` $stCount `]` `(` $inputs `)` attr-dict `:` $memRefType `,` functional-type($inputs, $outputs)"; let hasVerifier = 1; } @@ -726,8 +732,8 @@ def ExternalMemoryOp : Handshake_Op<"extmemory", [ return getMemref().getType().cast(); } - llvm::SmallVector getLoadPorts(); - llvm::SmallVector getStorePorts(); + llvm::SmallVector getLoadPorts(); + llvm::SmallVector getStorePorts(); }]; } diff --git a/include/circt/Dialect/Seq/SeqOps.td b/include/circt/Dialect/Seq/SeqOps.td index 7a5ab86b31..0623fd9c09 100644 --- a/include/circt/Dialect/Seq/SeqOps.td +++ b/include/circt/Dialect/Seq/SeqOps.td @@ -130,6 +130,11 @@ def HLMemOp : SeqOp<"hlmem", [ HLMemType getMemType() { return getHandle().getType().cast(); } }]; + let builders = [ + OpBuilder<(ins "Value":$clk, "Value":$rst, "StringRef":$symName, + "llvm::ArrayRef":$shape, "Type":$elementType)> + ]; + let assemblyFormat = "$sym_name $clk `,` $rst attr-dict `:` type($handle)"; } @@ -156,6 +161,9 @@ def ReadPortOp : SeqOp<"read", [ ); let results = (outs AnyType:$readData); let hasCustomAssemblyFormat = 1; + let builders = [OpBuilder<( + ins "Value":$memory, "ValueRange":$addresses, "Value":$rdEn, "unsigned":$latency)> + ]; } def WritePortOp : SeqOp<"write", [ diff --git a/integration_test/Dialect/Handshake/tp_memory/tp_memory.mlir b/integration_test/Dialect/Handshake/tp_memory/tp_memory.mlir index e256834944..c45dfdddc4 100644 --- a/integration_test/Dialect/Handshake/tp_memory/tp_memory.mlir +++ b/integration_test/Dialect/Handshake/tp_memory/tp_memory.mlir @@ -11,6 +11,9 @@ // RUN: hlstool %s --dynamic-firrtl --buffering-strategy=cycles --verilog --lowering-options=disallowLocalVariables > %t.sv && \ // RUN: %PYTHON% %S/../cocotb_driver.py --objdir=%T --topLevel=top --pythonModule=tp_memory --pythonFolder=%S %t.sv 2>&1 | FileCheck %s +// RUN: hlstool %s --sv-trace-iverilog --dynamic-hw --buffering-strategy=cycles --verilog --lowering-options=disallowLocalVariables > %t.sv && \ +// RUN: %PYTHON% %S/../cocotb_driver.py --objdir=%T --topLevel=top --pythonModule=tp_memory --pythonFolder=%S %t.sv 2>&1 | FileCheck %s + // Locking the circt should yield the same result // RUN: hlstool %s --dynamic-firrtl --buffering-strategy=all --dynamic-parallelism=locking --verilog --lowering-options=disallowLocalVariables > %t.sv && \ // RUN: %PYTHON% %S/../cocotb_driver.py --objdir=%T --topLevel=top --pythonModule=tp_memory --pythonFolder=%S %t.sv 2>&1 | FileCheck %s diff --git a/lib/Conversion/HandshakeToHW/HandshakeToHW.cpp b/lib/Conversion/HandshakeToHW/HandshakeToHW.cpp index 7de8887eb3..2dd615548f 100644 --- a/lib/Conversion/HandshakeToHW/HandshakeToHW.cpp +++ b/lib/Conversion/HandshakeToHW/HandshakeToHW.cpp @@ -528,6 +528,16 @@ struct RTLBuilder { // Bitwise 'not'. Value bNot(Value value, std::optional name = {}) { auto allOnes = constant(value.getType().getIntOrFloatBitWidth(), -1); + std::string inferedName; + if (!name) { + // Try to create a name from the input value. + if (auto valueName = + value.getDefiningOp()->getAttrOfType("sv.namehint")) { + inferedName = ("not_" + valueName.getValue()).str(); + name = inferedName; + } + } + return buildNamedOp( [&]() { return b.create(loc, value, allOnes); }, name); @@ -1452,6 +1462,111 @@ public: }; }; +class MemoryConversionPattern + : public HandshakeConversionPattern { +public: + using HandshakeConversionPattern< + handshake::MemoryOp>::HandshakeConversionPattern; + void buildModule(handshake::MemoryOp op, BackedgeBuilder &bb, RTLBuilder &s, + hw::HWModulePortAccessor &ports) const override { + auto loc = op.getLoc(); + + // Gather up the load and store ports. + auto unwrappedIO = this->unwrapIO(s, bb, ports); + struct LoadPort { + InputHandshake &addr; + OutputHandshake &data; + OutputHandshake &done; + }; + struct StorePort { + InputHandshake &addr; + InputHandshake &data; + OutputHandshake &done; + }; + SmallVector loadPorts; + SmallVector storePorts; + + unsigned stCount = op.getStCount(); + unsigned ldCount = op.getLdCount(); + for (unsigned i = 0, e = ldCount; i != e; ++i) { + LoadPort port = {unwrappedIO.inputs[stCount * 2 + i], + unwrappedIO.outputs[i], + unwrappedIO.outputs[ldCount + stCount + i]}; + loadPorts.push_back(port); + } + + for (unsigned i = 0, e = stCount; i != e; ++i) { + StorePort port = {unwrappedIO.inputs[i * 2 + 1], + unwrappedIO.inputs[i * 2], + unwrappedIO.outputs[ldCount + i]}; + storePorts.push_back(port); + } + + // used to drive the data wire of the control-only channels. + auto c0I0 = s.constant(0, 0); + + auto cl2dim = llvm::Log2_64_Ceil(op.getMemRefType().getShape()[0]); + auto hlmem = s.b.create( + loc, s.clk, s.rst, "_handshake_memory_" + std::to_string(op.getId()), + op.getMemRefType().getShape(), op.getMemRefType().getElementType()); + + // Create load ports... + for (auto &ld : loadPorts) { + llvm::SmallVector addresses = {s.truncate(ld.addr.data, cl2dim)}; + auto readData = s.b.create(loc, hlmem.getHandle(), + addresses, ld.addr.valid, + /*latency=*/0); + ld.data.data->setValue(readData); + ld.done.data->setValue(c0I0); + // Create control fork for the load address valid and ready signals. + buildForkLogic(s, bb, ld.addr, {ld.data, ld.done}); + } + + // Create store ports... + for (auto &st : storePorts) { + // Create a register to buffer the valid path by 1 cycle, to match the + // write latency of 1. + auto writeValidBufferMuxBE = bb.get(s.b.getI1Type()); + auto writeValidBuffer = + s.reg("writeValidBuffer", writeValidBufferMuxBE, s.constant(1, 0)); + st.done.valid->setValue(writeValidBuffer); + st.done.data->setValue(c0I0); + + // Create the logic for when both the buffered write valid signal and the + // store complete ready signal are asserted. + auto storeCompleted = + s.bAnd({st.done.ready, writeValidBuffer}, "storeCompleted"); + + // Create a signal for when the write valid buffer is empty or the output + // is ready. + auto notWriteValidBuffer = s.bNot(writeValidBuffer); + auto emptyOrComplete = + s.bOr({notWriteValidBuffer, storeCompleted}, "emptyOrComplete"); + + // Connect the gate to both the store address ready and store data ready + st.addr.ready->setValue(emptyOrComplete); + st.data.ready->setValue(emptyOrComplete); + + // Create a wire for when both the store address and data are valid. + auto writeValid = s.bAnd({st.addr.valid, st.data.valid}, "writeValid"); + + // Create a mux that drives the buffer input. If the emptyOrComplete + // signal is asserted, the mux selects the writeValid signal. Otherwise, + // it selects the buffer output, keeping the output registered until the + // emptyOrComplete signal is asserted. + writeValidBufferMuxBE.setValue( + s.mux(emptyOrComplete, {writeValidBuffer, writeValid})); + + // Instantiate the write port operation - truncate address width to memory + // width. + llvm::SmallVector addresses = {s.truncate(st.addr.data, cl2dim)}; + s.b.create(loc, hlmem.getHandle(), addresses, + st.data.data, writeValid, + /*latency=*/1); + } + } +}; // namespace + class SinkConversionPattern : public HandshakeConversionPattern { public: using HandshakeConversionPattern::HandshakeConversionPattern; @@ -1792,7 +1907,7 @@ static LogicalResult convertFuncOp(ESITypeConverter &typeConverter, ComparisonConversionPattern, BufferConversionPattern, SourceConversionPattern, SinkConversionPattern, ConstantConversionPattern, MergeConversionPattern, ControlMergeConversionPattern, - LoadConversionPattern, StoreConversionPattern, + LoadConversionPattern, StoreConversionPattern, MemoryConversionPattern, // Arith operations. ExtendConversionPattern, ExtendConversionPattern, diff --git a/lib/Dialect/Handshake/HandshakeOps.cpp b/lib/Dialect/Handshake/HandshakeOps.cpp index 842596ad50..a11e6fbda5 100644 --- a/lib/Dialect/Handshake/HandshakeOps.cpp +++ b/lib/Dialect/Handshake/HandshakeOps.cpp @@ -105,6 +105,49 @@ static void printOp(OpAsmPrinter &p, Operation *op, bool explicitSize) { } } // namespace sost +template +llvm::SmallVector getLoadPorts(TMemOp op) { + llvm::SmallVector ports; + // Memory interface refresher: + // Operands: + // all stores (stdata1, staddr1, stdata2, staddr2, ...) + // then all loads (ldaddr1, ldaddr2,...) + // Outputs: load addresses (lddata1, lddata2, ...), followed by all none + // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...) + unsigned stCount = op.getStCount(); + unsigned ldCount = op.getLdCount(); + for (unsigned i = 0, e = ldCount; i != e; ++i) { + MemLoadInterface ldif; + ldif.index = i; + ldif.addressIn = op.getInputs()[stCount * 2 + i]; + ldif.dataOut = op.getResult(i); + ldif.doneOut = op.getResult(ldCount + stCount + i); + ports.push_back(ldif); + } + return ports; +} + +template +llvm::SmallVector getStorePorts(TMemOp op) { + llvm::SmallVector ports; + // Memory interface refresher: + // Operands: + // all stores (stdata1, staddr1, stdata2, staddr2, ...) + // then all loads (ldaddr1, ldaddr2,...) + // Outputs: load data (lddata1, lddata2, ...), followed by all none + // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...) + unsigned ldCount = op.getLdCount(); + for (unsigned i = 0, e = op.getStCount(); i != e; ++i) { + MemStoreInterface stif; + stif.index = i; + stif.dataIn = op.getInputs()[i * 2]; + stif.addressIn = op.getInputs()[i * 2 + 1]; + stif.doneOut = op.getResult(ldCount + i); + ports.push_back(stif); + } + return ports; +} + void ForkOp::build(OpBuilder &builder, OperationState &result, Value operand, int outputs) { auto type = operand.getType(); @@ -1230,6 +1273,16 @@ void ExternalMemoryOp::build(OpBuilder &builder, OperationState &result, result.addAttribute("stCount", builder.getIntegerAttr(i32Type, stCount)); } +llvm::SmallVector +ExternalMemoryOp::getLoadPorts() { + return ::getLoadPorts(*this); +} + +llvm::SmallVector +ExternalMemoryOp::getStorePorts() { + return ::getStorePorts(*this); +} + void MemoryOp::build(OpBuilder &builder, OperationState &result, ValueRange operands, int outputs, int controlOutputs, bool lsq, int id, Value memref) { @@ -1256,47 +1309,12 @@ void MemoryOp::build(OpBuilder &builder, OperationState &result, } } -llvm::SmallVector -ExternalMemoryOp::getLoadPorts() { - llvm::SmallVector ports; - // Extmem interface refresher: - // Operands: - // all stores (stdata1, staddr1, stdata2, staddr2, ...) - // then all loads (ldaddr1, ldaddr2,...) - // Outputs: load addresses (lddata1, lddata2, ...), followed by all none - // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...) - unsigned stCount = getStCount(); - unsigned ldCount = getLdCount(); - for (unsigned i = 0, e = ldCount; i != e; ++i) { - ExtMemLoadInterface ldif; - ldif.index = i; - ldif.addressIn = getInputs()[stCount * 2 + i]; - ldif.dataOut = getResult(i); - ldif.doneOut = getResult(ldCount + stCount + i); - ports.push_back(ldif); - } - return ports; +llvm::SmallVector MemoryOp::getLoadPorts() { + return ::getLoadPorts(*this); } -llvm::SmallVector -ExternalMemoryOp::getStorePorts() { - llvm::SmallVector ports; - // Extmem interface refresher: - // Operands: - // all stores (stdata1, staddr1, stdata2, staddr2, ...) - // then all loads (ldaddr1, ldaddr2,...) - // Outputs: load data (lddata1, lddata2, ...), followed by all none - // outputs, ordered as operands(stnone1, stnone2, ... ldnone1, ldnone2, ...) - unsigned ldCount = getLdCount(); - for (unsigned i = 0, e = getStCount(); i != e; ++i) { - ExtMemStoreInterface stif; - stif.index = i; - stif.dataIn = getInputs()[i * 2]; - stif.addressIn = getInputs()[i * 2 + 1]; - stif.doneOut = getResult(ldCount + i); - ports.push_back(stif); - } - return ports; +llvm::SmallVector MemoryOp::getStorePorts() { + return ::getStorePorts(*this); } bool handshake::MemoryOp::allocateMemory( diff --git a/lib/Dialect/Handshake/Transforms/LowerExtmemToHW.cpp b/lib/Dialect/Handshake/Transforms/LowerExtmemToHW.cpp index cc8d2e4ab5..622bb1ae7e 100644 --- a/lib/Dialect/Handshake/Transforms/LowerExtmemToHW.cpp +++ b/lib/Dialect/Handshake/Transforms/LowerExtmemToHW.cpp @@ -267,7 +267,7 @@ static Value truncateToMemoryWidth(Location loc, OpBuilder &b, Value v, } static Value plumbLoadPort(Location loc, OpBuilder &b, - const handshake::ExtMemLoadInterface &ldif, + const handshake::MemLoadInterface &ldif, Value loadData, MemRefType memrefType) { // We need to feed both the load data and the load done outputs. // Fork the extracted load data into two, and 'join' the second one to @@ -287,7 +287,7 @@ static Value plumbLoadPort(Location loc, OpBuilder &b, } static Value plumbStorePort(Location loc, OpBuilder &b, - const handshake::ExtMemStoreInterface &stif, + const handshake::MemStoreInterface &stif, Value done, Type outType, MemRefType memrefType) { stif.doneOut.replaceAllUsesWith(done); // Return the store address and data to be fed to the top-level output. diff --git a/lib/Dialect/Seq/SeqOps.cpp b/lib/Dialect/Seq/SeqOps.cpp index 776d03265f..022307b10d 100644 --- a/lib/Dialect/Seq/SeqOps.cpp +++ b/lib/Dialect/Seq/SeqOps.cpp @@ -130,6 +130,13 @@ void ReadPortOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { setNameFn(getReadData(), (memName + "_rdata").str()); } +void ReadPortOp::build(OpBuilder &builder, OperationState &result, Value memory, + ValueRange addresses, Value rdEn, unsigned latency) { + auto memType = memory.getType().cast(); + ReadPortOp::build(builder, result, memType.getElementType(), memory, + addresses, rdEn, latency); +} + //===----------------------------------------------------------------------===// // WritePortOp //===----------------------------------------------------------------------===// @@ -181,6 +188,13 @@ void HLMemOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) { setNameFn(getHandle(), getName()); } +void HLMemOp::build(OpBuilder &builder, OperationState &result, Value clk, + Value rst, StringRef symName, llvm::ArrayRef shape, + Type elementType) { + HLMemType t = HLMemType::get(builder.getContext(), shape, elementType); + HLMemOp::build(builder, result, t, clk, rst, symName); +} + //===----------------------------------------------------------------------===// // CompRegOp diff --git a/tools/hlstool/hlstool.cpp b/tools/hlstool/hlstool.cpp index 3651842009..08f847b4dd 100644 --- a/tools/hlstool/hlstool.cpp +++ b/tools/hlstool/hlstool.cpp @@ -255,6 +255,7 @@ static void loadFIRRTLLoweringPipeline(OpPassManager &pm) { static void loadHWLoweringPipeline(OpPassManager &pm) { pm.addPass(createSimpleCanonicalizerPass()); + pm.nest().addPass(circt::seq::createLowerSeqHLMemPass()); pm.nest().addPass(seq::createSeqFIRRTLLowerToSVPass()); pm.addPass(sv::createHWMemSimImplPass(false, false)); pm.addPass(seq::createSeqLowerToSVPass());