From a480b0fbed915079de0e3053575b00c85f3eb038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Thu, 1 Oct 2026 14:21:57 +0200 Subject: [PATCH 1/4] asap7/l2_dir_hit: an L2 directory's hit, broadcast across the slice in one cycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A new test design with one concern: a register-to-register path that starts at the L2 directory's s3 registers beside its SRAM macros, decides hit and way, and in the same cycle steers the slice: MainPipe's meta write and through the single-ported directory RequestArb's ready to SinkC's 512-bit buffer read, the MSHR allocation, the L1 hint queue and the data array's request. It is one cycle of XiangShan's CoupledL2 (XSCache at 300515bc), written as readable, parameterised SystemVerilog read by slang; the sources cite the Scala lines each piece models. The SRAMs are behavioural modules in the firtool port convention, which AUTO_MEMORIES turns into macros at XiangShan's own shapes. constraint.sdc declares the data array's two-cycle contract, which the RTL states and asserts. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/designs/asap7/l2_dir_hit/README.md | 93 +++++ flow/designs/asap7/l2_dir_hit/config.mk | 33 ++ flow/designs/asap7/l2_dir_hit/constraint.sdc | 32 ++ .../src/l2_dir_hit/l2_custom_l1_hint.sv | 54 +++ flow/designs/src/l2_dir_hit/l2_data_sram.sv | 31 ++ flow/designs/src/l2_dir_hit/l2_dir_hit.sv | 382 ++++++++++++++++++ flow/designs/src/l2_dir_hit/l2_directory.sv | 135 +++++++ flow/designs/src/l2_dir_hit/l2_meta_sram.sv | 27 ++ flow/designs/src/l2_dir_hit/l2_mshr_ctl.sv | 40 ++ flow/designs/src/l2_dir_hit/l2_sink_c.sv | 93 +++++ flow/designs/src/l2_dir_hit/l2_tag_sram.sv | 31 ++ 11 files changed, 951 insertions(+) create mode 100644 flow/designs/asap7/l2_dir_hit/README.md create mode 100644 flow/designs/asap7/l2_dir_hit/config.mk create mode 100644 flow/designs/asap7/l2_dir_hit/constraint.sdc create mode 100644 flow/designs/src/l2_dir_hit/l2_custom_l1_hint.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_data_sram.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_dir_hit.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_directory.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_meta_sram.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_mshr_ctl.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_sink_c.sv create mode 100644 flow/designs/src/l2_dir_hit/l2_tag_sram.sv diff --git a/flow/designs/asap7/l2_dir_hit/README.md b/flow/designs/asap7/l2_dir_hit/README.md new file mode 100644 index 0000000000..08b61fc9fe --- /dev/null +++ b/flow/designs/asap7/l2_dir_hit/README.md @@ -0,0 +1,93 @@ +# l2_dir_hit + +One cycle of an L2 cache slice's main pipeline. The directory decides +**hit** and **way** in s3, from tag and meta registers that its SRAM +macros filled the cycle before, and in the same cycle that decision +steers the slice: + +- MainPipe writes the line's new meta into the directory, and the + replacer is written on every hit. The arrays are single ported, so + either write makes the directory unready, RequestArb withholds its ready + from SinkC, and SinkC's task does not fire: the enable of a 512-bit + register that reads SinkC's data buffer. +- A miss allocates one of 16 MSHRs, with the state the hit decided. +- A hit answered by MainPipe itself enqueues an L1 hint. +- The data array is read or written at the hit's way. + +The design exists to measure one thing: how much of a register-to-register +path that starts beside SRAM macros, decides in one place and is +broadcast across the slice in the same cycle goes to repeaters and wire +rather than logic. + +## The path + +```mermaid +flowchart LR + subgraph DIR[directory] + T[tag SRAMs, 2] --> TR[s3 tag registers] + M[meta SRAM] --> MR[s3 meta registers] + end + TR & MR --> H[8 tag compares, hit, way one-hot, Mux1H of the meta] + H --> D[MainPipe: need_mshr, meta write] + D --> W[directory busy] --> R[RequestArb ready] --> C[SinkC task fire: 512-bit read enable] + D --> MS[MSHR allocation, 16 MSHRs] + D --> Q[hint queue] + D -->|two cycles| DS[data SRAMs, 4] +``` + +XiangShan's designers name the path in `MainPipe.scala`: "The +combinational logic path of Directory metaAll -> Directory response -> +MainPipe judging whether to respond data is too long", and latch the +SinkB response to s4 for it. The paths that remain single cycle are the +ones above. + +## Where it comes from + +XiangShan's L2, CoupledL2 (XSCache at 300515bc), in the configuration of +a 512 KiB, 8-way slice with a 64-byte block: + +| piece | source | +|---|---| +| tag, meta arrays and the s3 hit, the single port | `coupledL2/Directory.scala:169-343, 390` | +| MainPipe's decision and its directory writes | `coupledL2/MainPipe.scala:244-262, 538-616` | +| the MSHR allocation | `coupledL2/MainPipe.scala:309, 1021-1059`, `coupledL2/MSHRCtl.scala:97-137` | +| RequestArb's ready, one task every other cycle | `coupledL2/RequestArb.scala:134-208` | +| SinkC's buffers and its read | `coupledL2/SinkC.scala:48-193` | +| the hint queue | `coupledL2/CustomL1Hint.scala:91-126` | +| the data array's request | `coupledL2/MainPipe.scala:484-519`, `coupledL2/DataStorage.scala:52-131` | + +The source files cite the lines each piece models, and say what is left +out because it is not on the path. + +## The two-cycle data array + +XiangShan builds the data array with `readMCP2 = true`: "read data is +set MultiCycle Path 2", and the request, way, set and write data "must +hold for 2 cycles". The RTL asserts both that they hold and that no +request follows a request, and RequestArb stalls a cycle after every task +to make it so. `constraint.sdc` declares that contract, into the array +and out of it; the clock gate's enable is a one-cycle pulse and keeps one +cycle. + +Without the declaration the data array's address and read data would be +among the worst paths of the design, and they would be failures the +designers never asked the flow to close. + +## Configuration + +| | | +|---|---| +| RTL | Readable, parameterised SystemVerilog, read by slang (`SYNTH_HDL_FRONTEND = slang`) | +| SRAMs | Behavioural modules in the firtool port convention (`RW0_*`). `AUTO_MEMORIES = 1` turns each into a generated macro at XiangShan's own shapes: two 1024x156 tag halves, a 1024x128 meta array and four 8192x137 data banks | +| Constraints | `constraint.sdc` sources `$PLATFORM_DIR/constraints.sdc`. It is a macro, so the boundaries are `set_max_delay` optimisation targets, with no `set_input_delay`/`set_output_delay`, and only reg2reg can fail. The 473 ps period is derived in the file | +| Flow | ORFS defaults throughout, all three asap7 Vt's, the full flow through final, no waivers | + +## Results + +The minimum clock period is the SDC period minus the worst slack of the +`reg2reg` group, at global route with the route's parasitics. + +| | minimum period at grt | worst path | logic / repeaters / wire | +|---|---|---|---| +| XiangShan, CoupledL2 in XSTile | 1,128 ps | directory `req_s3_tag` to SinkC's buffer read | 341 ps (24 cells) / 473 ps (35) / 247 ps | +| this design | _pending_ | _pending_ | _pending_ | diff --git a/flow/designs/asap7/l2_dir_hit/config.mk b/flow/designs/asap7/l2_dir_hit/config.mk new file mode 100644 index 0000000000..43d7c4c90a --- /dev/null +++ b/flow/designs/asap7/l2_dir_hit/config.mk @@ -0,0 +1,33 @@ +# l2_dir_hit: an L2 directory's hit, decided in one cycle from SRAM-fed +# registers, steering the cache's pipeline. README.md says what it is for +# and where it comes from. +export PLATFORM = asap7 +export DESIGN_NAME = l2_dir_hit +export DESIGN_NICKNAME = l2_dir_hit + +export VERILOG_FILES = $(sort $(wildcard $(DESIGN_HOME)/src/$(DESIGN_NICKNAME)/*.sv)) +export SDC_FILE = $(DESIGN_HOME)/$(PLATFORM)/$(DESIGN_NICKNAME)/constraint.sdc + +# Readable, parameterised SystemVerilog, read by slang. +export SYNTH_HDL_FRONTEND = slang + +# The tag, meta and data arrays are behavioural modules in the firtool +# port convention; AUTO_MEMORIES turns each into a generated macro at +# XiangShan's own shapes. +export AUTO_MEMORIES = 1 + +# All three threshold voltages, so the resizer can trade leakage for +# speed per path, as a 3 GHz-class core does. +export ASAP7_USE_VT = RVT LVT SLVT + +export CORE_UTILIZATION = 35 +export CORE_MARGIN = 2 +# Channels between macros: room for the power grid to reach every SRAM. +export RTLMP_MIN_CHANNEL_SIZE = 4 4 + +# The slice's units stay modules, as CoupledL2's own flow keeps them: the +# macro placer clusters by them. Flat, one data bank is ~96 % of the area +# of the cluster around it and the area-balanced bisection cannot split +# it (MPL-0045). +export SYNTH_HIERARCHICAL = 1 +export SYNTH_KEEP_MODULES = l2_directory l2_sink_c l2_mshr_ctl l2_custom_l1_hint diff --git a/flow/designs/asap7/l2_dir_hit/constraint.sdc b/flow/designs/asap7/l2_dir_hit/constraint.sdc new file mode 100644 index 0000000000..4831afe820 --- /dev/null +++ b/flow/designs/asap7/l2_dir_hit/constraint.sdc @@ -0,0 +1,32 @@ +# 473 ps is XiangShan's synthesis period on this library: its published +# goal is 3 GHz (333 ps) on 7 nm, 41.1 FO4 at ASAP7's published 8.1 ps, +# which is 591 ps at this flow's 14.37 ps FO4, and 0.8 of that. +set clk_name clk +set clk_port_name clock +set clk_period 473 + +# A macro: only reg2reg can fail closure. The boundaries are +# optimisation targets, set_max_delay in the platform's constraints.sdc, +# not set_input_delay/set_output_delay. +set in2reg_max [expr { $clk_period * 0.8 }] +set reg2out_max [expr { $clk_period * 0.8 }] +set in2out_max [expr { $clk_period * 0.6 }] + +source $::env(PLATFORM_DIR)/constraints.sdc + +# The data array is a two-cycle macro, a contract the RTL states: +# readMCP2 = true, "read data is set MultiCycle Path 2", and its request, +# way, set and write data "must hold for 2 cycles"; the RTL asserts both +# that they hold and that no request follows a request +# (coupledL2/DataStorage.scala:52-80, 119-131). Into the array and out of +# it, setup has two cycles and hold stays at the launching edge. The +# clock gate's enable is not part of the contract: it is a one-cycle +# pulse, a register in this design, and keeps one cycle. +set data_srams [get_cells -hierarchical -filter "ref_name == l2_data_sram"] +if { [llength $data_srams] == 0 } { + error "constraint.sdc: no l2_data_sram instance, the multicycle contract has nothing to hold" +} +set_multicycle_path -setup 2 -to $data_srams +set_multicycle_path -hold 1 -to $data_srams +set_multicycle_path -setup 2 -from $data_srams +set_multicycle_path -hold 1 -from $data_srams diff --git a/flow/designs/src/l2_dir_hit/l2_custom_l1_hint.sv b/flow/designs/src/l2_dir_hit/l2_custom_l1_hint.sv new file mode 100644 index 0000000000..c1028546db --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_custom_l1_hint.sv @@ -0,0 +1,54 @@ +// CustomL1Hint: a hit answered by MainPipe tells the L1 at s3, through an +// arbiter with the s1 hints and a 16-entry queue of flops +// (coupledL2/CustomL1Hint.scala:91-126, XSCache at 300515bc). The queue is +// chisel3.util.Queue as Chisel builds it: enqueue and dequeue pointers and +// maybe_full. +module l2_custom_l1_hint #( + parameter int SOURCEBITS = 7, + parameter int ENTRIES = 16 // mshrsAll +) ( + input logic clock, + input logic reset, + input logic enq_s3, + input logic enq_s3_has_data, + input logic [SOURCEBITS-1:0] enq_s3_source, + input logic s1_valid, + input logic [SOURCEBITS-1:0] s1_source, + input logic hint_ready, + output logic hint_valid, + output logic [SOURCEBITS-1:0] hint_source, + output logic hint_has_data +); + // the arbiter, s3 first (:124) + logic enq_valid, enq_data, enq_fire, deq_fire; + logic [SOURCEBITS-1:0] enq_source; + assign enq_valid = enq_s3 || s1_valid; + assign enq_source = enq_s3 ? enq_s3_source : s1_source; + assign enq_data = enq_s3 ? enq_s3_has_data : 1'b0; + + logic [SOURCEBITS:0] q[ENTRIES]; + logic [$clog2(ENTRIES)-1:0] enq_ptr, deq_ptr; + logic maybe_full, ptr_match, empty, full; + assign ptr_match = enq_ptr == deq_ptr; + assign empty = ptr_match && !maybe_full; + assign full = ptr_match && maybe_full; + assign enq_fire = enq_valid && !full; + assign deq_fire = !empty && hint_ready; + always_ff @(posedge clock) begin + if (enq_fire) q[enq_ptr] <= {enq_data, enq_source}; + if (reset) begin + enq_ptr <= '0; + deq_ptr <= '0; + maybe_full <= 1'b0; + end else begin + if (enq_fire) enq_ptr <= enq_ptr + 1'b1; + if (deq_fire) deq_ptr <= deq_ptr + 1'b1; + if (enq_fire != deq_fire) maybe_full <= enq_fire; + end + end + always_ff @(posedge clock) begin + hint_valid <= !reset && !empty; + hint_source <= q[deq_ptr][SOURCEBITS-1:0]; + hint_has_data <= q[deq_ptr][SOURCEBITS]; + end +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_data_sram.sv b/flow/designs/src/l2_dir_hit/l2_data_sram.sv new file mode 100644 index 0000000000..bec51a60f4 --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_data_sram.sv @@ -0,0 +1,31 @@ +// One quarter of the L2 data array: 8192 entries (1024 sets of eight +// ways) of a 128-bit word with a 9-bit SECDED code, single-ported. +// XiangShan's CoupledL2 builds four of exactly this macro +// (array_8192x137) for a 64-byte block: coupledL2/DataStorage.scala:68-80, +// a GatedSplittedSRAM with dataSplit = 4 and readMCP2 = true. +// +// The array is a two-cycle macro in XiangShan: its inputs hold for two +// cycles and its read data is captured two cycles after the read edge +// (DataStorage.scala:52-65, 119-131). constraint.sdc declares that +// contract; this model is the ordinary one-cycle form, which is all +// AUTO_MEMORIES needs to generate the macro. +// +// Port convention and fixed size as in l2_tag_sram.sv. +module l2_data_sram ( + input logic RW0_clk, + input logic RW0_en, + input logic RW0_wmode, // 1 write, 0 read + input logic [ 12:0] RW0_addr, // {way, set} + input logic [136:0] RW0_wdata, + output logic [136:0] RW0_rdata +); + logic [136:0] mem[8192]; + logic ren_q; + logic [ 12:0] raddr_q; + always_ff @(posedge RW0_clk) begin + ren_q <= RW0_en && !RW0_wmode; + raddr_q <= RW0_addr; + if (RW0_en && RW0_wmode) mem[RW0_addr] <= RW0_wdata; + end + assign RW0_rdata = ren_q ? mem[raddr_q] : 'x; +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv new file mode 100644 index 0000000000..91fa56b2d0 --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv @@ -0,0 +1,382 @@ +// One cycle of an L2 cache slice: the directory's hit, decided in s3 from +// tag and meta registers fed by SRAM macros, steers the slice in the same +// cycle: MainPipe's meta write into the directory, the directory's single +// port and through it RequestArb's ready to SinkC, which enables SinkC's +// data-buffer read; the MSHR allocation; the L1 hint queue; and the data +// array's request. XiangShan's CoupledL2 (XSCache at 300515bc): +// coupledL2/Directory.scala, MainPipe.scala, RequestArb.scala, SinkC.scala, +// MSHRCtl.scala, CustomL1Hint.scala and DataStorage.scala; each piece +// cites its lines. README.md says what the design is for. +// +// Every input is registered at its port and every output leaves a +// register, so the only paths that can fail are register to register +// (constraint.sdc). +module l2_dir_hit #( + parameter int SETS = 1024, + parameter int WAYS = 8, + parameter int TAGBITS = 32, + parameter int METABITS = 16, + parameter int SOURCEBITS = 7, + parameter int MSHRS = 16, // mshrsAll + parameter int BUFBLOCKS = 4, // SinkC's bufBlocks (CoupledL2.scala:75) + parameter int BEATBITS = 256, // beatBytes = 32 + parameter int BANKS = 4, // dataSplit (DataStorage.scala:68) + parameter int BANKBITS = 137 // SECDED(128) +) ( + input logic clock, + input logic reset, + // channel A: an acquire or a get from the L1 + input logic a_valid, + input logic [ 2:0] a_opcode, + input logic a_need_t, + input logic [ $clog2(SETS)-1:0] a_set, + input logic [ TAGBITS-1:0] a_tag, + input logic [ SOURCEBITS-1:0] a_source, + output logic a_ready, + // channel C: a release with data from the L1, two beats + input logic c_valid, + input logic c_last, + input logic [ $clog2(SETS)-1:0] c_set, + input logic [ TAGBITS-1:0] c_tag, + input logic [ BEATBITS-1:0] c_data, + output logic c_ready, + // an MSHR task: a refill, its way, its tag write + input logic m_valid, + input logic [ $clog2(SETS)-1:0] m_set, + input logic [ TAGBITS-1:0] m_tag, + input logic [ $clog2(WAYS)-1:0] m_way, + input logic [BANKS*BANKBITS-1:0] m_refill_data, + input logic [ WAYS-1:0] repl_way_oh, // the replacer's choice + // an MSHR finishing, and a status read + input logic [ MSHRS-1:0] mshr_free, + input logic [ $clog2(MSHRS)-1:0] mshr_query, + output logic [ TAGBITS-1:0] mshr_status_tag, + output logic [ 5:0] mshr_status_state, + // the hint queue's other source, at s1 (CustomL1Hint.scala:107-122) + input logic hint_s1_valid, + input logic [ SOURCEBITS-1:0] hint_s1_source, + input logic hint_ready, + output logic hint_valid, + output logic [ SOURCEBITS-1:0] hint_source, + output logic hint_has_data, + // the data array + output logic ds_clock_en, // the array's clock gate + output logic [BANKS*BANKBITS-1:0] ds_rdata +); + localparam int SETBITS = $clog2(SETS); + localparam int WAYBITS = $clog2(WAYS); + localparam int BUFBITS = $clog2(BUFBLOCKS); + // TileLink opcodes + localparam logic [2:0] GET = 4, ACQUIRE_BLOCK = 6, ACQUIRE_PERM = 7; + localparam logic [2:0] ACCESS_ACK_DATA = 1, GRANT = 4, GRANT_DATA = 5; + // coupledL2/MetaData.scala + localparam logic [1:0] BRANCH = 1, TRUNK = 2, TIP = 3; + + // SECDED over a 128-bit word, as DataStorage encodes a bank + // (DataStorage.scala:91-104): a Hamming code over positions 1..136 and + // an overall parity bit. On a two-cycle path (constraint.sdc). + function automatic logic [BANKBITS-1:0] secded128(input logic [127:0] d); + logic [135:0] h; + logic [ 7:0] c; + int k; + h = '0; + k = 0; + for (int p = 1; p <= 136; p++) + if ((p & (p - 1)) != 0) begin + h[p-1] = d[k]; + k++; + end + c = '0; + for (int p = 1; p <= 136; p++) for (int b = 0; b < 8; b++) if (p[b]) c[b] ^= h[p-1]; + return {^{c, d}, c, d}; + endfunction + + // ---- port registers ---- + logic in_a_valid, in_a_need_t, in_c_valid, in_c_last, in_m_valid; + logic [ 2:0] in_a_opcode; + logic [SETBITS-1:0] in_a_set, in_c_set, in_m_set; + logic [TAGBITS-1:0] in_a_tag, in_c_tag, in_m_tag; + logic [SOURCEBITS-1:0] in_a_source, in_hint_s1_source; + logic [ BEATBITS-1:0] in_c_data; + logic [ WAYBITS-1:0] in_m_way; + logic [BANKS*BANKBITS-1:0] in_m_refill_data; + logic [ WAYS-1:0] in_repl_way_oh; + logic [ MSHRS-1:0] in_mshr_free; + logic [ $clog2(MSHRS)-1:0] in_mshr_query; + logic in_hint_s1_valid, in_hint_ready; + always_ff @(posedge clock) begin + in_a_valid <= !reset && a_valid; + in_c_valid <= !reset && c_valid; + in_m_valid <= !reset && m_valid; + in_hint_s1_valid <= !reset && hint_s1_valid; + in_hint_ready <= !reset && hint_ready; + in_mshr_free <= reset ? '0 : mshr_free; + in_a_opcode <= a_opcode; + in_a_need_t <= a_need_t; + in_a_set <= a_set; + in_a_tag <= a_tag; + in_a_source <= a_source; + in_c_last <= c_last; + in_c_set <= c_set; + in_c_tag <= c_tag; + in_c_data <= c_data; + in_m_set <= m_set; + in_m_tag <= m_tag; + in_m_way <= m_way; + in_m_refill_data <= m_refill_data; + in_repl_way_oh <= repl_way_oh; + in_mshr_query <= mshr_query; + in_hint_s1_source <= hint_s1_source; + end + + // ---- SinkC (SinkC.scala) ---- + logic c_task_valid, sink_c_ready, c_task_fire; + logic [ SETBITS-1:0] c_task_set; + logic [ TAGBITS-1:0] c_task_tag; + logic [ BUFBITS-1:0] c_task_idx; + logic [2*BEATBITS-1:0] buf_resp_s3; + l2_sink_c #( + .SETBITS(SETBITS), + .TAGBITS(TAGBITS), + .BUFBLOCKS(BUFBLOCKS), + .BEATBITS(BEATBITS) + ) u_sink_c ( + .clock (clock), + .reset (reset), + .c_valid (in_c_valid), + .c_last (in_c_last), + .c_set (in_c_set), + .c_tag (in_c_tag), + .c_data (in_c_data), + .c_ready (c_ready), + .task_valid (c_task_valid), + .task_set (c_task_set), + .task_tag (c_task_tag), + .task_buf_idx(c_task_idx), + .task_ready (sink_c_ready), + .buf_resp (buf_resp_s3) + ); + assign c_task_fire = c_task_valid && sink_c_ready; + + // ---- RequestArb s1: an MSHR task first, then C, then A, into s2 when + // the directory can be read (RequestArb.scala:134-208) ---- + logic dir_read_ready, stall_q, s2_ready, sink_ready_basic; + assign s2_ready = !stall_q; // ds_mcp2_stall (:201-204) + assign sink_ready_basic = dir_read_ready && !in_m_valid && s2_ready; // :151 + assign sink_c_ready = sink_ready_basic; // :155 + assign a_ready = sink_ready_basic && !c_task_valid; + + logic fire_m, fire_c, fire_a, fire_s1; + assign fire_m = in_m_valid && s2_ready; + assign fire_c = c_task_fire; + assign fire_a = in_a_valid && a_ready; + assign fire_s1 = fire_m || fire_c || fire_a; + always_ff @(posedge clock) stall_q <= !reset && fire_s1; + + // the task, s1 to s3 + typedef struct packed { + logic mshr; + logic from_a; + logic from_c; + logic [2:0] opcode; + logic need_t; + logic [SETBITS-1:0] setidx; + logic [TAGBITS-1:0] tag; + logic [WAYBITS-1:0] way; + logic [SOURCEBITS-1:0] source; + logic [BUFBITS-1:0] buf_idx; + } task_t; + task_t task_s1, task_s2, task_s3; + always_comb begin + task_s1 = '0; + task_s1.mshr = fire_m; + task_s1.from_c = !fire_m && fire_c; + task_s1.from_a = !fire_m && !fire_c; + task_s1.opcode = in_a_opcode; + task_s1.need_t = in_a_need_t; + task_s1.setidx = fire_m ? in_m_set : fire_c ? c_task_set : in_a_set; + task_s1.tag = fire_m ? in_m_tag : fire_c ? c_task_tag : in_a_tag; + task_s1.way = in_m_way; + task_s1.source = in_a_source; + task_s1.buf_idx = c_task_idx; + end + + logic v_s2, v_s3, v_s3_hold2; + always_ff @(posedge clock) begin + v_s2 <= !reset && fire_s1; + v_s3 <= !reset && v_s2; + // task_s3_valid_hold2 (MainPipe.scala:500) + if (reset) v_s3_hold2 <= 1'b0; + else if (!v_s3) v_s3_hold2 <= v_s2; + if (fire_s1) task_s2 <= task_s1; + if (v_s2) task_s3 <= task_s2; + end + + // ---- the directory: read in s1 for a channel task, written by MainPipe + // at s3 ---- + logic hit, dir_valid; + logic [ WAYBITS-1:0] dir_way; + logic [METABITS-1:0] meta_on_hit; + logic meta_wen, tag_wen, replacer_wen; + logic [METABITS-1:0] meta_wmeta; + logic [WAYS-1:0] meta_wway_oh, tag_wway_oh; + l2_directory #( + .SETS(SETS), + .WAYS(WAYS), + .TAGBITS(TAGBITS), + .METABITS(METABITS) + ) u_dir ( + .clock (clock), + .reset (reset), + .read_fire (fire_s1 && !fire_m), + .read_set (task_s1.setidx), + .read_tag (task_s1.tag), + .tag_wen (tag_wen), + .tag_wset (task_s3.setidx), + .tag_wway_oh (tag_wway_oh), + .tag_wtag (task_s3.tag), + .meta_wen (meta_wen), + .meta_wset (task_s3.setidx), + .meta_wway_oh (meta_wway_oh), + .meta_wmeta (meta_wmeta), + .repl_way_oh_s3(in_repl_way_oh), + .resp_valid_s3 (dir_valid), + .hit_s3 (hit), + .way_s3 (dir_way), + .meta_s3 (), + .meta_on_hit_s3(meta_on_hit) + ); + // the arrays are single ported: a write this cycle is no read + // (Directory.scala:343); the replacer is written on every hit (:390) + assign replacer_wen = dir_valid && hit; + assign dir_read_ready = !meta_wen && !tag_wen && !replacer_wen; + + // ---- s3: MainPipe's decision (MainPipe.scala:244-262) ---- + logic chn_s3, a_s3, c_s3, m_s3; + logic req_get, req_acquire_block, req_acquire, has_clients; + assign chn_s3 = v_s3 && !task_s3.mshr; + assign a_s3 = chn_s3 && task_s3.from_a; + assign c_s3 = chn_s3 && task_s3.from_c; + assign m_s3 = v_s3 && task_s3.mshr; + assign req_get = task_s3.opcode == GET; + assign req_acquire_block = task_s3.opcode == ACQUIRE_BLOCK; + assign req_acquire = task_s3.opcode == ACQUIRE_BLOCK || task_s3.opcode == ACQUIRE_PERM; + assign has_clients = meta_on_hit[3]; + + logic acquire_on_hit, need_acquire, need_probe, need_mshr; + assign acquire_on_hit = meta_on_hit[1:0] == BRANCH && task_s3.need_t; + assign need_acquire = a_s3 && (hit ? acquire_on_hit : req_acquire || req_get); + assign need_probe = a_s3 && hit && has_clients && req_get && meta_on_hit[1:0] == TRUNK; + assign need_mshr = need_acquire || need_probe; + + // MainPipe's directory writes (MainPipe.scala:538-616): an A request + // answered here updates its line, a release that hits marks it dirty, a + // refill writes its tag and a fresh line + logic [METABITS-1:0] meta_w_a, meta_w_c, meta_w_m; + always_comb begin + meta_w_a = meta_on_hit; + meta_w_a[1:0] = task_s3.need_t ? TRUNK : meta_on_hit[1:0]; + meta_w_a[3] = 1'b1; // clients + meta_w_c = meta_on_hit; + meta_w_c[1:0] = TIP; + meta_w_c[2] = 1'b1; // dirty + meta_w_c[3] = 1'b0; + meta_w_m = '0; + meta_w_m[1:0] = TIP; + end + assign meta_wen = a_s3 && !need_mshr && !req_get || c_s3 && hit || m_s3; + assign meta_wmeta = a_s3 ? meta_w_a : c_s3 ? meta_w_c : meta_w_m; + assign meta_wway_oh = m_s3 ? WAYS'(1) << task_s3.way : WAYS'(1) << dir_way; + assign tag_wen = m_s3; + assign tag_wway_oh = WAYS'(1) << task_s3.way; + + // ---- s3: MSHR allocation (MainPipe.scala:309, 1021-1059; + // MSHRCtl.scala:97-137): the lowest idle MSHR takes the request and the + // state its hit decided ---- + logic [5:0] alloc_state; + assign alloc_state = { + !need_probe, // s_rprobe + !need_acquire, // s_acquire + hit, // w_replResp + !(need_acquire && req_acquire), // s_rcompack + !need_probe, // w_rprobeackfirst + !(hit && req_acquire_block) // s_refill + }; + l2_mshr_ctl #( + .MSHRS (MSHRS), + .TAGBITS(TAGBITS) + ) u_mshr_ctl ( + .clock (clock), + .reset (reset), + .alloc_valid (need_mshr), + .alloc_state (alloc_state), + .alloc_tag (task_s3.tag), + .free (in_mshr_free), + .query (in_mshr_query), + .status_tag (mshr_status_tag), + .status_state(mshr_status_state) + ); + + // ---- s3: the L1 hint, "chnTask Hit will fire@s3" (CustomL1Hint.scala:91-126) ---- + logic sink_resp, enq_s3, enq_has_data; + logic [2:0] d_opcode; + assign sink_resp = a_s3 && !need_mshr; + assign d_opcode = req_get ? ACCESS_ACK_DATA : req_acquire_block ? GRANT_DATA : GRANT; + assign enq_s3 = sink_resp && (d_opcode == GRANT || d_opcode == GRANT_DATA || + d_opcode == ACCESS_ACK_DATA); + assign enq_has_data = d_opcode != GRANT; + + l2_custom_l1_hint #( + .SOURCEBITS(SOURCEBITS), + .ENTRIES (MSHRS) + ) u_custom_l1_hint ( + .clock (clock), + .reset (reset), + .enq_s3 (enq_s3), + .enq_s3_has_data(enq_has_data), + .enq_s3_source (task_s3.source), + .s1_valid (in_hint_s1_valid), + .s1_source (in_hint_s1_source), + .hint_ready (in_hint_ready), + .hint_valid (hint_valid), + .hint_source (hint_source), + .hint_has_data (hint_has_data) + ); + + // ---- s3: the data array's request (MainPipe.scala:484-519) ---- + // en is the cycle's pulse that opens the array's clock gate; req holds + // for two cycles, and so do its way, set and write data + // (DataStorage.scala:52-65). The gate's enable is a register here: a + // clock gate's latch has the same setup check at the same edge. A + // release that hits writes its data, from SinkC's buffer; a refill + // writes the MSHR's. + logic ren, wen, ds_en, ds_req_valid, ds_cg_en; + logic [WAYBITS-1:0] ds_way; + assign ren = a_s3 && hit && sink_resp && (req_get || req_acquire_block); + assign wen = c_s3 && hit || m_s3; + assign ds_en = v_s3 && (ren || wen); + assign ds_req_valid = v_s3_hold2 && (ren || wen); + assign ds_way = m_s3 ? task_s3.way : dir_way; + always_ff @(posedge clock) ds_cg_en <= !reset && ds_en; + assign ds_clock_en = ds_cg_en; + + logic [BANKBITS-1:0] bank_rdata[BANKS]; + for (genvar b = 0; b < BANKS; b++) begin : g_bank + logic [BANKBITS-1:0] wdata; + assign wdata = m_s3 ? in_m_refill_data[b*BANKBITS+:BANKBITS] : + secded128(buf_resp_s3[b*128+:128]); + l2_data_sram u_sram ( + .RW0_clk (clock), + .RW0_en (ds_req_valid), + .RW0_wmode(wen), + .RW0_addr ({ds_way, task_s3.setidx}), + .RW0_wdata(wdata), + .RW0_rdata(bank_rdata[b]) + ); + end + + // s5: the read data, two cycles after the read edge ("s3 read, s4 pass + // and s5 to destination", DataStorage.scala:119-121) + always_ff @(posedge clock) + for (int b = 0; b < BANKS; b++) ds_rdata[b*BANKBITS+:BANKBITS] <= bank_rdata[b]; +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_directory.sv b/flow/designs/src/l2_dir_hit/l2_directory.sv new file mode 100644 index 0000000000..8076c6c34e --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_directory.sv @@ -0,0 +1,135 @@ +// The L2 directory of one slice: read the tag and meta arrays in s1, +// register what they return at s3, and in s3 decide hit, way and the +// chosen way's meta, combinationally, for MainPipe to use the same cycle. +// coupledL2/Directory.scala, of XSCache at 300515bc. +// +// What is left out, because it is not on the path: the replacer's own +// SRAM (its choice arrives as repl_way_oh_s3, as io.replResp does), the +// refill-retry free-way mask (:274-300), CMO by-way requests and the +// tag ECC check, which feeds only error_s3 (:259-268, :321), never the hit. +module l2_directory #( + parameter int SETS = 1024, + parameter int WAYS = 8, + parameter int TAGBITS = 32, + parameter int METABITS = 16 +) ( + input logic clock, + input logic reset, + // s1: a read, at most every other cycle (RequestArb's MCP2 stall) + input logic read_fire, + input logic [$clog2(SETS)-1:0] read_set, + input logic [TAGBITS-1:0] read_tag, + // tag and meta writes from MainPipe and the MSHRs, one way at a time + input logic tag_wen, + input logic [$clog2(SETS)-1:0] tag_wset, + input logic [WAYS-1:0] tag_wway_oh, + input logic [TAGBITS-1:0] tag_wtag, + input logic meta_wen, + input logic [$clog2(SETS)-1:0] meta_wset, + input logic [WAYS-1:0] meta_wway_oh, + input logic [METABITS-1:0] meta_wmeta, + // s3 + input logic [WAYS-1:0] repl_way_oh_s3, + output logic resp_valid_s3, + output logic hit_s3, + output logic [$clog2(WAYS)-1:0] way_s3, + output logic [METABITS-1:0] meta_s3, + output logic [METABITS-1:0] meta_on_hit_s3 +); + localparam int ENCTAG = TAGBITS + 7; // SECDED(32): 39 bits + localparam int HALF = WAYS / 2; // waySplit = 2 (:173) + localparam logic [1:0] INVALID = 0; // coupledL2/MetaData.scala + + // SECDED(39,32) as XiangShan's dataCode encodes a tag for writing + // (:221-226): a Hamming code over positions 1..38 and an overall + // parity bit. Not on the read path: the compare uses the tag bits + // as stored (:254-258, :270). + function automatic logic [ENCTAG-1:0] secded(input logic [TAGBITS-1:0] d); + logic [37:0] h; + logic [ 5:0] c; + int k; + h = '0; + k = 0; + for (int p = 1; p <= 38; p++) + if ((p & (p - 1)) != 0) begin + h[p-1] = d[k]; + k++; + end + c = '0; + for (int p = 1; p <= 38; p++) for (int b = 0; b < 6; b++) if (p[b]) c[b] ^= h[p-1]; + return {^{c, d}, c, d}; + endfunction + + // s1: one read of both arrays, writes when there is no read (single port) + logic [ENCTAG*HALF-1:0] tag_rdata[2]; + logic [ENCTAG*HALF-1:0] tag_wdata; + for (genvar w = 0; w < HALF; w++) begin : g_enc + assign tag_wdata[w*ENCTAG+:ENCTAG] = secded(tag_wtag); + end + for (genvar s = 0; s < 2; s++) begin : g_tag + l2_tag_sram u_sram ( + .RW0_clk (clock), + .RW0_en (read_fire || tag_wen), + .RW0_wmode(!read_fire), + .RW0_addr (read_fire ? read_set : tag_wset), + .RW0_wmask(tag_wway_oh[s*HALF+:HALF]), + .RW0_wdata(tag_wdata), + .RW0_rdata(tag_rdata[s]) + ); + end + + logic [METABITS*WAYS-1:0] meta_rdata; + l2_meta_sram u_meta ( + .RW0_clk (clock), + .RW0_en (read_fire || meta_wen), + .RW0_wmode(!read_fire), + .RW0_addr (read_fire ? read_set : meta_wset), + .RW0_wmask(meta_wway_oh), + .RW0_wdata({WAYS{meta_wmeta}}), + .RW0_rdata(meta_rdata) + ); + + // s2: the arrays answer; s3 latches them (:215-218, :252-253) + logic valid_s2; + logic [TAGBITS-1:0] tag_s2; + logic valid_s3; + logic [TAGBITS-1:0] tag_s3; + logic [ENCTAG*WAYS-1:0] tag_read_s3; + logic [METABITS*WAYS-1:0] meta_all_s3; + always_ff @(posedge clock) begin + valid_s2 <= !reset && read_fire; + valid_s3 <= !reset && valid_s2; + if (read_fire) tag_s2 <= read_tag; + if (valid_s2) begin + tag_s3 <= tag_s2; + tag_read_s3 <= {tag_rdata[1], tag_rdata[0]}; + meta_all_s3 <= meta_rdata; + end + end + + // s3: hit and way, "in stage 3, Cuz SRAM latency is high under high + // frequency" (:209-213, :270-272, :295-312) + logic [WAYS-1:0] hit_vec, inv_vec, inv_oh, chosen_oh, way_oh; + for (genvar w = 0; w < WAYS; w++) begin : g_way + logic [METABITS-1:0] m; + assign m = meta_all_s3[w*METABITS+:METABITS]; + assign inv_vec[w] = m[1:0] == INVALID; + assign hit_vec[w] = tag_read_s3[w*ENCTAG+:TAGBITS] == tag_s3 && !inv_vec[w]; + end + assign inv_oh = inv_vec & -inv_vec; // invalid_way_sel: the first invalid way + assign chosen_oh = |inv_vec ? inv_oh : repl_way_oh_s3; + assign hit_s3 = |hit_vec; + assign way_oh = hit_s3 ? hit_vec : chosen_oh; + + always_comb begin + way_s3 = '0; + meta_s3 = '0; + meta_on_hit_s3 = '0; + for (int w = 0; w < WAYS; w++) begin // OHToUInt and Mux1H + if (way_oh[w]) way_s3 |= w[$clog2(WAYS)-1:0]; + if (way_oh[w]) meta_s3 |= meta_all_s3[w*METABITS+:METABITS]; + if (hit_vec[w]) meta_on_hit_s3 |= meta_all_s3[w*METABITS+:METABITS]; + end + end + assign resp_valid_s3 = valid_s3; +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_meta_sram.sv b/flow/designs/src/l2_dir_hit/l2_meta_sram.sv new file mode 100644 index 0000000000..2390d2d25c --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_meta_sram.sv @@ -0,0 +1,27 @@ +// The L2 directory's meta array: 1024 sets of eight ways, each way a +// 16-bit coherence entry (state, dirty, clients, prefetch and error +// bits), single-ported, synchronous read. XiangShan's CoupledL2 builds +// one of exactly this macro (array_1024x128): coupledL2/Directory.scala:198, +// an SRAMTemplate of MetaEntry, read latency 1. +// +// Port convention and fixed size as in l2_tag_sram.sv. +module l2_meta_sram ( + input logic RW0_clk, + input logic RW0_en, + input logic RW0_wmode, // 1 write, 0 read + input logic [ 9:0] RW0_addr, + input logic [ 7:0] RW0_wmask, // one bit per way + input logic [127:0] RW0_wdata, + output logic [127:0] RW0_rdata +); + logic [127:0] mem[1024]; + logic ren_q; + logic [ 9:0] raddr_q; + always_ff @(posedge RW0_clk) begin + ren_q <= RW0_en && !RW0_wmode; + raddr_q <= RW0_addr; + for (int w = 0; w < 8; w++) + if (RW0_en && RW0_wmode && RW0_wmask[w]) mem[RW0_addr][w*16+:16] <= RW0_wdata[w*16+:16]; + end + assign RW0_rdata = ren_q ? mem[raddr_q] : 'x; +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_mshr_ctl.sv b/flow/designs/src/l2_dir_hit/l2_mshr_ctl.sv new file mode 100644 index 0000000000..1aae1f9942 --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_mshr_ctl.sv @@ -0,0 +1,40 @@ +// MSHRCtl's allocation: the lowest idle MSHR takes the request MainPipe +// decided needs one, with the state the hit decided +// (coupledL2/MSHRCtl.scala:97-137, MainPipe.scala:309, 1021-1059, XSCache +// at 300515bc). An MSHR's own state machine is left out: it frees when +// told, and its state is read back through one status port. +module l2_mshr_ctl #( + parameter int MSHRS = 16, // mshrsAll + parameter int TAGBITS = 32 +) ( + input logic clock, + input logic reset, + input logic alloc_valid, // mshr_alloc_s3.valid + input logic [ 5:0] alloc_state, + input logic [ TAGBITS-1:0] alloc_tag, + input logic [ MSHRS-1:0] free, + input logic [$clog2(MSHRS)-1:0] query, + output logic [ TAGBITS-1:0] status_tag, + output logic [ 5:0] status_state +); + logic [MSHRS-1:0] valid, idle, sel; + assign idle = ~valid; + assign sel = idle & -idle; // MSHRSelector: ParallelPriorityMux + + logic [TAGBITS-1:0] tags [MSHRS]; + logic [ 5:0] states[MSHRS]; + always_ff @(posedge clock) + for (int i = 0; i < MSHRS; i++) begin + if (reset) valid[i] <= 1'b0; + else if (sel[i] && alloc_valid) valid[i] <= 1'b1; + else if (free[i]) valid[i] <= 1'b0; + if (sel[i] && alloc_valid) begin // m.io.alloc.valid (:137) + states[i] <= alloc_state; + tags[i] <= alloc_tag; + end + end + always_ff @(posedge clock) begin + status_tag <= tags[query]; + status_state <= states[query]; + end +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_sink_c.sv b/flow/designs/src/l2_dir_hit/l2_sink_c.sv new file mode 100644 index 0000000000..1111520b70 --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_sink_c.sv @@ -0,0 +1,93 @@ +// SinkC: a release's two beats into a data buffer, its task into a task +// buffer that waits for RequestArb, and the task's data read the cycle it +// fires (coupledL2/SinkC.scala:48-193, XSCache at 300515bc). The read +// register's enable is task_fire, so RequestArb's ready reaches every bit +// of it. +module l2_sink_c #( + parameter int SETBITS = 10, + parameter int TAGBITS = 32, + parameter int BUFBLOCKS = 4, // bufBlocks (CoupledL2.scala:75) + parameter int BEATBITS = 256 // beatBytes = 32 +) ( + input logic clock, + input logic reset, + // channel C + input logic c_valid, + input logic c_last, + input logic [ SETBITS-1:0] c_set, + input logic [ TAGBITS-1:0] c_tag, + input logic [ BEATBITS-1:0] c_data, + output logic c_ready, + // the task to RequestArb + output logic task_valid, + output logic [ SETBITS-1:0] task_set, + output logic [ TAGBITS-1:0] task_tag, + output logic [$clog2(BUFBLOCKS)-1:0] task_buf_idx, + input logic task_ready, + // bufResp, the task's data two cycles after it fired (:190) + output logic [ 2*BEATBITS-1:0] buf_resp +); + localparam int BUFBITS = $clog2(BUFBLOCKS); + + logic [BEATBITS-1:0] data_buf [BUFBLOCKS][2]; + logic [ SETBITS-1:0] task_sets [BUFBLOCKS]; + logic [ TAGBITS-1:0] task_tags [BUFBLOCKS]; + logic [BUFBLOCKS-1:0] task_valids, data_valids; + logic first_beat, buf_full; + logic [BUFBITS-1:0] next_ptr, next_ptr_reg, c_ptr; + assign buf_full = &(task_valids | data_valids); + always_comb begin // PriorityEncoder(~bufValids) (:64) + next_ptr = '0; + for (int i = BUFBLOCKS - 1; i >= 0; i--) + if (!(task_valids[i] || data_valids[i])) next_ptr = i[BUFBITS-1:0]; + end + assign c_ptr = first_beat ? next_ptr : next_ptr_reg; + assign c_ready = !first_beat || !buf_full; // :188 + + // taskArb: the lowest valid entry goes to RequestArb (:134-147) + always_comb begin + task_valid = |task_valids; + task_buf_idx = '0; + for (int i = BUFBLOCKS - 1; i >= 0; i--) if (task_valids[i]) task_buf_idx = i[BUFBITS-1:0]; + end + assign task_set = task_sets[task_buf_idx]; + assign task_tag = task_tags[task_buf_idx]; + + logic task_fire; + assign task_fire = task_valid && task_ready; + + // bufResp := RegNext(RegEnable(dataBuf(bufIdx), io.task.fire)); the + // entry frees the cycle after (:190-193) + logic [2*BEATBITS-1:0] buf_resp_q; + logic fired_q; + logic [ BUFBITS-1:0] fired_idx_q; + always_ff @(posedge clock) begin + if (task_fire) buf_resp_q <= {data_buf[task_buf_idx][1], data_buf[task_buf_idx][0]}; + buf_resp <= buf_resp_q; + fired_q <= !reset && task_fire; + fired_idx_q <= task_buf_idx; + end + + always_ff @(posedge clock) begin + if (reset) begin + task_valids <= '0; + data_valids <= '0; + first_beat <= 1'b1; + next_ptr_reg <= '0; + end else begin + if (fired_q) data_valids[fired_idx_q] <= 1'b0; + if (task_fire) task_valids[task_buf_idx] <= 1'b0; + if (c_valid && c_ready) begin + data_buf[c_ptr][!first_beat] <= c_data; + data_valids[c_ptr] <= 1'b1; + if (first_beat) next_ptr_reg <= next_ptr; + first_beat <= c_last; + if (c_last) begin + task_valids[c_ptr] <= 1'b1; + task_sets[c_ptr] <= c_set; + task_tags[c_ptr] <= c_tag; + end + end + end + end +endmodule diff --git a/flow/designs/src/l2_dir_hit/l2_tag_sram.sv b/flow/designs/src/l2_dir_hit/l2_tag_sram.sv new file mode 100644 index 0000000000..900ea935da --- /dev/null +++ b/flow/designs/src/l2_dir_hit/l2_tag_sram.sv @@ -0,0 +1,31 @@ +// Half of the L2 directory's tag array: 1024 sets of four ways, each way +// a 32-bit tag with a 7-bit SECDED code, single-ported, synchronous read. +// XiangShan's CoupledL2 builds two of exactly this macro (array_1024x156) +// for a 512 KiB, 8-way slice: coupledL2/Directory.scala:169-184, a +// SplittedSRAM of 8 ways in two halves (waySplit = 2), read latency 1. +// +// Every port follows the firtool memory convention (RW0_), so +// ORFS's AUTO_MEMORIES detects this module and replaces it with a +// generated macro; the body below is its behavioural model. The size is +// fixed on purpose: the detector names the macro after the module, and a +// parameterised module would be renamed by the frontend. +module l2_tag_sram ( + input logic RW0_clk, + input logic RW0_en, + input logic RW0_wmode, // 1 write, 0 read + input logic [ 9:0] RW0_addr, + input logic [ 3:0] RW0_wmask, // one bit per way + input logic [155:0] RW0_wdata, + output logic [155:0] RW0_rdata +); + logic [155:0] mem[1024]; + logic ren_q; + logic [ 9:0] raddr_q; + always_ff @(posedge RW0_clk) begin + ren_q <= RW0_en && !RW0_wmode; + raddr_q <= RW0_addr; + for (int w = 0; w < 4; w++) + if (RW0_en && RW0_wmode && RW0_wmask[w]) mem[RW0_addr][w*39+:39] <= RW0_wdata[w*39+:39]; + end + assign RW0_rdata = ren_q ? mem[raddr_q] : 'x; +endmodule From 853589155f8cf114de34c8c8143684971fa89823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Thu, 1 Oct 2026 14:32:52 +0200 Subject: [PATCH 2/4] l2_dir_hit: hold the data array's request for both cycles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MainPipe decides on the task's bits, which hold for two cycles, and applies the valid where each consumer takes it (MainPipe.scala:309, :499-504, :605). The decision here was gated by the one-cycle s3 valid, so in the hold cycle the data array's enable, write mode, way and write data changed before the second edge. The refill data came straight from a port register; it is now read at s2 and registered for s3, as the refill buffer's response is. A simulation of DataStorage.scala:124-131's three asserts on the array's pins, 20,000 cycles of random traffic: 8,696 violations before, none after, over 5,966 requests and 4,869 writes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/designs/src/l2_dir_hit/l2_dir_hit.sv | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv index 91fa56b2d0..7da402fdd2 100644 --- a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv +++ b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv @@ -212,6 +212,11 @@ module l2_dir_hit #( if (v_s2) task_s3 <= task_s2; end + // the refill buffer, read at s2 and its response registered for s3, so + // it holds with the task (RequestArb.scala:235-238) + logic [BANKS*BANKBITS-1:0] refill_s3; + always_ff @(posedge clock) if (v_s2) refill_s3 <= in_m_refill_data; + // ---- the directory: read in s1 for a channel task, written by MainPipe // at s3 ---- logic hit, dir_valid; @@ -252,12 +257,15 @@ module l2_dir_hit #( assign dir_read_ready = !meta_wen && !tag_wen && !replacer_wen; // ---- s3: MainPipe's decision (MainPipe.scala:244-262) ---- + // As MainPipe does, the decision is made on the task's bits, which hold + // for two cycles, and the valid is applied where each consumer takes it + // (:309, :499-504, :605): the data array's request must hold for both. logic chn_s3, a_s3, c_s3, m_s3; logic req_get, req_acquire_block, req_acquire, has_clients; - assign chn_s3 = v_s3 && !task_s3.mshr; + assign chn_s3 = !task_s3.mshr; assign a_s3 = chn_s3 && task_s3.from_a; assign c_s3 = chn_s3 && task_s3.from_c; - assign m_s3 = v_s3 && task_s3.mshr; + assign m_s3 = task_s3.mshr; assign req_get = task_s3.opcode == GET; assign req_acquire_block = task_s3.opcode == ACQUIRE_BLOCK; assign req_acquire = task_s3.opcode == ACQUIRE_BLOCK || task_s3.opcode == ACQUIRE_PERM; @@ -284,10 +292,10 @@ module l2_dir_hit #( meta_w_m = '0; meta_w_m[1:0] = TIP; end - assign meta_wen = a_s3 && !need_mshr && !req_get || c_s3 && hit || m_s3; + assign meta_wen = v_s3 && (a_s3 && !need_mshr && !req_get || c_s3 && hit || m_s3); assign meta_wmeta = a_s3 ? meta_w_a : c_s3 ? meta_w_c : meta_w_m; assign meta_wway_oh = m_s3 ? WAYS'(1) << task_s3.way : WAYS'(1) << dir_way; - assign tag_wen = m_s3; + assign tag_wen = v_s3 && m_s3; assign tag_wway_oh = WAYS'(1) << task_s3.way; // ---- s3: MSHR allocation (MainPipe.scala:309, 1021-1059; @@ -308,7 +316,7 @@ module l2_dir_hit #( ) u_mshr_ctl ( .clock (clock), .reset (reset), - .alloc_valid (need_mshr), + .alloc_valid (v_s3 && need_mshr), .alloc_state (alloc_state), .alloc_tag (task_s3.tag), .free (in_mshr_free), @@ -322,7 +330,7 @@ module l2_dir_hit #( logic [2:0] d_opcode; assign sink_resp = a_s3 && !need_mshr; assign d_opcode = req_get ? ACCESS_ACK_DATA : req_acquire_block ? GRANT_DATA : GRANT; - assign enq_s3 = sink_resp && (d_opcode == GRANT || d_opcode == GRANT_DATA || + assign enq_s3 = v_s3 && sink_resp && (d_opcode == GRANT || d_opcode == GRANT_DATA || d_opcode == ACCESS_ACK_DATA); assign enq_has_data = d_opcode != GRANT; @@ -363,7 +371,7 @@ module l2_dir_hit #( logic [BANKBITS-1:0] bank_rdata[BANKS]; for (genvar b = 0; b < BANKS; b++) begin : g_bank logic [BANKBITS-1:0] wdata; - assign wdata = m_s3 ? in_m_refill_data[b*BANKBITS+:BANKBITS] : + assign wdata = m_s3 ? refill_s3[b*BANKBITS+:BANKBITS] : secded128(buf_resp_s3[b*128+:128]); l2_data_sram u_sram ( .RW0_clk (clock), From 75b4e6b0711a10931997ac6803ae4c710c02178a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Thu, 1 Oct 2026 16:27:18 +0200 Subject: [PATCH 3/4] l2_dir_hit: the data array's s5 read register inside the slice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read data's s5 destinations are inside the slice (DataStorage.scala: 119-121). As an output register, s5 was drawn to its pin, and the first global route's worst path was a bank's read data crossing the die to it on one net: the harness's boundary, not CoupledL2's path. The port now takes s5 a cycle later. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/designs/src/l2_dir_hit/l2_dir_hit.sv | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv index 7da402fdd2..1d07ea1038 100644 --- a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv +++ b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv @@ -384,7 +384,12 @@ module l2_dir_hit #( end // s5: the read data, two cycles after the read edge ("s3 read, s4 pass - // and s5 to destination", DataStorage.scala:119-121) - always_ff @(posedge clock) - for (int b = 0; b < BANKS; b++) ds_rdata[b*BANKBITS+:BANKBITS] <= bank_rdata[b]; + // and s5 to destination", DataStorage.scala:119-121). Its destinations + // are inside the slice, so s5 is too, and the port takes it a cycle + // later: an output register is drawn to its pin. + logic [BANKS*BANKBITS-1:0] rdata_s5; + always_ff @(posedge clock) begin + for (int b = 0; b < BANKS; b++) rdata_s5[b*BANKBITS+:BANKBITS] <= bank_rdata[b]; + ds_rdata <= rdata_s5; + end endmodule From 900174b1ee85ecbe66919575325e06b688c22ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98yvind=20Harboe?= Date: Thu, 1 Oct 2026 18:31:05 +0200 Subject: [PATCH 4/4] l2_dir_hit: MainPipe's whole s3 decision on the directory's hit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first measurement at global route put this design's worst path on XiangShan's own, directory s3 to SinkC's buffer read, at about half the period, with 5 logic cells in MainPipe's part where XiangShan has about 14. MainPipe now decides as XiangShan's does: channel A with prefetch, CMOs and the alias compare (MainPipe.scala:172-265), the snoop classes of channel B and forwarding (:281-304), the replacer's answer held for s3 (:237-239), the five meta-write terms (:538-616), and the data array's ren and wen terms (:484-495). The directory's tag ECC error on the hit way feeds the snoop decision (Directory.scala:315-319), and RequestArb takes B between C and A, behind MainPipe's set blocks (RequestArb.scala:134-160, MainPipe.scala:954-979). The simulation of DataStorage.scala:124-131's asserts still reports no violations, over 3,669 requests and 2,101 writes. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Øyvind Harboe --- flow/designs/src/l2_dir_hit/l2_dir_hit.sv | 256 +++++++++++++++----- flow/designs/src/l2_dir_hit/l2_directory.sv | 20 +- 2 files changed, 207 insertions(+), 69 deletions(-) diff --git a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv index 1d07ea1038..977d71e064 100644 --- a/flow/designs/src/l2_dir_hit/l2_dir_hit.sv +++ b/flow/designs/src/l2_dir_hit/l2_dir_hit.sv @@ -25,20 +25,28 @@ module l2_dir_hit #( ) ( input logic clock, input logic reset, - // channel A: an acquire or a get from the L1 + // channel A: an acquire, a get, a prefetch or a CMO from the L1 input logic a_valid, - input logic [ 2:0] a_opcode, + input logic [ 3:0] a_opcode, input logic a_need_t, + input logic [ 1:0] a_alias, input logic [ $clog2(SETS)-1:0] a_set, input logic [ TAGBITS-1:0] a_tag, input logic [ SOURCEBITS-1:0] a_source, output logic a_ready, + // channel B: a snoop from the interconnect, by class (MainPipe.scala:266-302) + input logic b_valid, + input logic [ $clog2(SETS)-1:0] b_set, + input logic [ TAGBITS-1:0] b_tag, + input logic [ 4:0] b_snoop, // {resp data, fwd, to N, to B, stable} + output logic b_ready, // channel C: a release with data from the L1, two beats input logic c_valid, input logic c_last, input logic [ $clog2(SETS)-1:0] c_set, input logic [ TAGBITS-1:0] c_tag, input logic [ BEATBITS-1:0] c_data, + input logic c_from_t, // isParamFromT(param) output logic c_ready, // an MSHR task: a refill, its way, its tag write input logic m_valid, @@ -46,7 +54,13 @@ module l2_dir_hit #( input logic [ TAGBITS-1:0] m_tag, input logic [ $clog2(WAYS)-1:0] m_way, input logic [BANKS*BANKBITS-1:0] m_refill_data, + input logic m_meta_wen, + input logic m_ds_wen, + input logic m_refill, // mshr_refill_s3 input logic [ WAYS-1:0] repl_way_oh, // the replacer's choice + input logic repl_valid_hold, // io.replResp, at s3 + input logic repl_retry, + input logic [ 1:0] repl_state, // an MSHR finishing, and a status read input logic [ MSHRS-1:0] mshr_free, input logic [ $clog2(MSHRS)-1:0] mshr_query, @@ -66,11 +80,16 @@ module l2_dir_hit #( localparam int SETBITS = $clog2(SETS); localparam int WAYBITS = $clog2(WAYS); localparam int BUFBITS = $clog2(BUFBLOCKS); - // TileLink opcodes - localparam logic [2:0] GET = 4, ACQUIRE_BLOCK = 6, ACQUIRE_PERM = 7; - localparam logic [2:0] ACCESS_ACK_DATA = 1, GRANT = 4, GRANT_DATA = 5; + // TileLink A opcodes, and XiangShan's CMO extension; the CMO encodings + // are this design's, XiangShan takes them from its rocket-chip fork + localparam logic [3:0] GET = 4, HINT = 5, ACQUIRE_BLOCK = 6, ACQUIRE_PERM = 7; + localparam logic [3:0] CBO_CLEAN = 8, CBO_FLUSH = 9, CBO_INVAL = 10; + localparam logic [2:0] ACCESS_ACK_DATA = 1, HINT_ACK = 2, GRANT = 4, GRANT_DATA = 5; // coupledL2/MetaData.scala - localparam logic [1:0] BRANCH = 1, TRUNK = 2, TIP = 3; + localparam logic [1:0] INVALID = 0, BRANCH = 1, TRUNK = 2, TIP = 3; + // the meta entry's fields, as this design packs MetaEntry + // (Directory.scala:31-48): state, dirty, clients, alias, tagErr + localparam int M_DIRTY = 2, M_CLIENTS = 3, M_ALIAS = 4, M_TAGERR = 8; // SECDED over a 128-bit word, as DataStorage encodes a bank // (DataStorage.scala:91-104): a Hamming code over positions 1..136 and @@ -92,10 +111,13 @@ module l2_dir_hit #( endfunction // ---- port registers ---- - logic in_a_valid, in_a_need_t, in_c_valid, in_c_last, in_m_valid; - logic [ 2:0] in_a_opcode; - logic [SETBITS-1:0] in_a_set, in_c_set, in_m_set; - logic [TAGBITS-1:0] in_a_tag, in_c_tag, in_m_tag; + logic in_a_valid, in_a_need_t, in_c_valid, in_c_last, in_m_valid, in_b_valid, in_c_from_t; + logic in_m_meta_wen, in_m_ds_wen, in_m_refill, in_repl_valid_hold, in_repl_retry; + logic [ 3:0] in_a_opcode; + logic [ 1:0] in_a_alias, in_repl_state; + logic [ 4:0] in_b_snoop; + logic [SETBITS-1:0] in_a_set, in_c_set, in_m_set, in_b_set; + logic [TAGBITS-1:0] in_a_tag, in_c_tag, in_m_tag, in_b_tag; logic [SOURCEBITS-1:0] in_a_source, in_hint_s1_source; logic [ BEATBITS-1:0] in_c_data; logic [ WAYBITS-1:0] in_m_way; @@ -106,6 +128,18 @@ module l2_dir_hit #( logic in_hint_s1_valid, in_hint_ready; always_ff @(posedge clock) begin in_a_valid <= !reset && a_valid; + in_b_valid <= !reset && b_valid; + in_a_alias <= a_alias; + in_b_set <= b_set; + in_b_tag <= b_tag; + in_b_snoop <= b_snoop; + in_c_from_t <= c_from_t; + in_m_meta_wen <= m_meta_wen; + in_m_ds_wen <= m_ds_wen; + in_m_refill <= m_refill; + in_repl_valid_hold <= repl_valid_hold; + in_repl_retry <= repl_retry; + in_repl_state <= repl_state; in_c_valid <= !reset && c_valid; in_m_valid <= !reset && m_valid; in_hint_s1_valid <= !reset && hint_s1_valid; @@ -158,19 +192,24 @@ module l2_dir_hit #( ); assign c_task_fire = c_task_valid && sink_c_ready; - // ---- RequestArb s1: an MSHR task first, then C, then A, into s2 when + // ---- RequestArb s1: an MSHR task first, then C, B and A, into s2 when // the directory can be read (RequestArb.scala:134-208) ---- - logic dir_read_ready, stall_q, s2_ready, sink_ready_basic; + logic dir_read_ready, stall_q, s2_ready, sink_ready_basic, block_b, block_c; + logic c_req, b_req; assign s2_ready = !stall_q; // ds_mcp2_stall (:201-204) assign sink_ready_basic = dir_read_ready && !in_m_valid && s2_ready; // :151 - assign sink_c_ready = sink_ready_basic; // :155 - assign a_ready = sink_ready_basic && !c_task_valid; + assign c_req = c_task_valid && !block_c; // sinkValids (:141-145) + assign b_req = in_b_valid && !block_b; + assign sink_c_ready = sink_ready_basic && !block_c; // :153-155 + assign b_ready = sink_ready_basic && !block_b && !c_req; + assign a_ready = sink_ready_basic && !b_req && !c_req; - logic fire_m, fire_c, fire_a, fire_s1; + logic fire_m, fire_c, fire_b, fire_a, fire_s1; assign fire_m = in_m_valid && s2_ready; assign fire_c = c_task_fire; + assign fire_b = in_b_valid && b_ready; assign fire_a = in_a_valid && a_ready; - assign fire_s1 = fire_m || fire_c || fire_a; + assign fire_s1 = fire_m || fire_c || fire_b || fire_a; always_ff @(posedge clock) stall_q <= !reset && fire_s1; // the task, s1 to s3 @@ -178,7 +217,14 @@ module l2_dir_hit #( logic mshr; logic from_a; logic from_c; - logic [2:0] opcode; + logic from_b; + logic [3:0] opcode; + logic [1:0] alias_bits; + logic [4:0] snoop; + logic c_from_t; + logic meta_wen; + logic ds_wen; + logic refill; logic need_t; logic [SETBITS-1:0] setidx; logic [TAGBITS-1:0] tag; @@ -191,11 +237,18 @@ module l2_dir_hit #( task_s1 = '0; task_s1.mshr = fire_m; task_s1.from_c = !fire_m && fire_c; - task_s1.from_a = !fire_m && !fire_c; + task_s1.from_b = !fire_m && !fire_c && fire_b; + task_s1.from_a = !fire_m && !fire_c && !fire_b; + task_s1.alias_bits = in_a_alias; + task_s1.snoop = in_b_snoop; + task_s1.c_from_t = in_c_from_t; + task_s1.meta_wen = in_m_meta_wen; + task_s1.ds_wen = in_m_ds_wen; + task_s1.refill = in_m_refill; task_s1.opcode = in_a_opcode; task_s1.need_t = in_a_need_t; - task_s1.setidx = fire_m ? in_m_set : fire_c ? c_task_set : in_a_set; - task_s1.tag = fire_m ? in_m_tag : fire_c ? c_task_tag : in_a_tag; + task_s1.setidx = fire_m ? in_m_set : fire_c ? c_task_set : fire_b ? in_b_set : in_a_set; + task_s1.tag = fire_m ? in_m_tag : fire_c ? c_task_tag : fire_b ? in_b_tag : in_a_tag; task_s1.way = in_m_way; task_s1.source = in_a_source; task_s1.buf_idx = c_task_idx; @@ -212,14 +265,33 @@ module l2_dir_hit #( if (v_s2) task_s3 <= task_s2; end + // MainPipe's set blocks (MainPipe.scala:954-979): a C task waits for an + // s2 task of its set that may write the directory, a snoop for one at s2 + // or s3 + assign block_c = v_s2 && task_s2.setidx == c_task_set && !(task_s2.mshr && !task_s2.meta_wen); + assign block_b = v_s2 && task_s2.setidx == in_b_set || v_s3 && task_s3.setidx == in_b_set; + // the refill buffer, read at s2 and its response registered for s3, so // it holds with the task (RequestArb.scala:235-238) logic [BANKS*BANKBITS-1:0] refill_s3; always_ff @(posedge clock) if (v_s2) refill_s3 <= in_m_refill_data; + // the replacer's answer for the task at s3, held with it (validHold, + // MainPipe.scala:237) + logic repl_valid_s3, repl_retry_s3; + logic [ 1:0] repl_state_s3; + logic [WAYS-1:0] repl_way_oh_s3; + always_ff @(posedge clock) + if (v_s2) begin + repl_way_oh_s3 <= in_repl_way_oh; + repl_valid_s3 <= in_repl_valid_hold; + repl_retry_s3 <= in_repl_retry; + repl_state_s3 <= in_repl_state; + end + // ---- the directory: read in s1 for a channel task, written by MainPipe // at s3 ---- - logic hit, dir_valid; + logic hit, dir_valid, err_on_snp; logic [ WAYBITS-1:0] dir_way; logic [METABITS-1:0] meta_on_hit; logic meta_wen, tag_wen, replacer_wen; @@ -244,58 +316,105 @@ module l2_dir_hit #( .meta_wset (task_s3.setidx), .meta_wway_oh (meta_wway_oh), .meta_wmeta (meta_wmeta), - .repl_way_oh_s3(in_repl_way_oh), + .repl_way_oh_s3(repl_way_oh_s3), .resp_valid_s3 (dir_valid), .hit_s3 (hit), .way_s3 (dir_way), .meta_s3 (), - .meta_on_hit_s3(meta_on_hit) + .meta_on_hit_s3(meta_on_hit), + .err_on_snp_s3 (err_on_snp) ); // the arrays are single ported: a write this cycle is no read // (Directory.scala:343); the replacer is written on every hit (:390) assign replacer_wen = dir_valid && hit; assign dir_read_ready = !meta_wen && !tag_wen && !replacer_wen; - // ---- s3: MainPipe's decision (MainPipe.scala:244-262) ---- + // ---- s3: MainPipe's decision (MainPipe.scala:160-304) ---- // As MainPipe does, the decision is made on the task's bits, which hold // for two cycles, and the valid is applied where each consumer takes it // (:309, :499-504, :605): the data array's request must hold for both. - logic chn_s3, a_s3, c_s3, m_s3; - logic req_get, req_acquire_block, req_acquire, has_clients; - assign chn_s3 = !task_s3.mshr; - assign a_s3 = chn_s3 && task_s3.from_a; - assign c_s3 = chn_s3 && task_s3.from_c; - assign m_s3 = task_s3.mshr; - assign req_get = task_s3.opcode == GET; - assign req_acquire_block = task_s3.opcode == ACQUIRE_BLOCK; - assign req_acquire = task_s3.opcode == ACQUIRE_BLOCK || task_s3.opcode == ACQUIRE_PERM; - assign has_clients = meta_on_hit[3]; + logic a_s3, b_s3, c_s3, m_s3; + assign a_s3 = !task_s3.mshr && task_s3.from_a; + assign b_s3 = !task_s3.mshr && task_s3.from_b; + assign c_s3 = !task_s3.mshr && task_s3.from_c; + assign m_s3 = task_s3.mshr; + + logic req_get, req_hint, req_acquire_block, req_acquire, cbo_clean, cbo_flush, cbo_inval, cmo; + assign req_get = a_s3 && task_s3.opcode == GET; // :172-179 + assign req_hint = a_s3 && task_s3.opcode == HINT; + assign req_acquire_block = a_s3 && task_s3.opcode == ACQUIRE_BLOCK; + assign req_acquire = a_s3 && (task_s3.opcode == ACQUIRE_BLOCK || task_s3.opcode == ACQUIRE_PERM); + assign cbo_clean = a_s3 && task_s3.opcode == CBO_CLEAN; + assign cbo_flush = a_s3 && task_s3.opcode == CBO_FLUSH; + assign cbo_inval = a_s3 && task_s3.opcode == CBO_INVAL; + assign cmo = cbo_clean || cbo_flush || cbo_inval; // :220 + + logic [1:0] state; + logic has_clients, dirty, tag_err; + assign state = meta_on_hit[1:0]; + assign has_clients = meta_on_hit[M_CLIENTS]; + assign dirty = meta_on_hit[M_DIRTY]; + assign tag_err = meta_on_hit[M_TAGERR]; + + // a hit whose client holds the line under another alias (:225-226) + logic cache_alias; + assign cache_alias = req_acquire && hit && has_clients && meta_on_hit[M_ALIAS+:2] != task_s3.alias_bits; + + // channel A (:244-265) + logic acquire_on_miss, acquire_on_hit, need_acquire_a, need_probe_a, need_mshr_a; + assign acquire_on_miss = req_acquire || req_hint || req_get; + assign acquire_on_hit = state == BRANCH && task_s3.need_t && !req_hint; + assign need_acquire_a = a_s3 && ((hit ? acquire_on_hit : acquire_on_miss) || cmo); + assign need_probe_a = hit && has_clients && + (req_get && state == TRUNK || cbo_clean && state == TRUNK || cbo_flush || cbo_inval); + assign need_mshr_a = need_acquire_a || need_probe_a || cache_alias; + + // channel B, a snoop (:281-302) + logic snp_stable, snp_to_b, snp_to_n, snp_fwd, snp_resp_data; + assign {snp_resp_data, snp_fwd, snp_to_n, snp_to_b, snp_stable} = task_s3.snoop; + logic do_fwd, need_pprobe_b, need_mshr_b; + assign do_fwd = snp_fwd && hit && !(tag_err || err_on_snp); + assign need_pprobe_b = b_s3 && hit && ( + snp_stable && state == TRUNK && has_clients || + snp_to_b && state == TRUNK && has_clients || + snp_to_n && has_clients || + err_on_snp || tag_err); + assign need_mshr_b = need_pprobe_b || b_s3 && do_fwd; + + logic need_mshr; + assign need_mshr = need_mshr_a || need_mshr_b; // :304 - logic acquire_on_hit, need_acquire, need_probe, need_mshr; - assign acquire_on_hit = meta_on_hit[1:0] == BRANCH && task_s3.need_t; - assign need_acquire = a_s3 && (hit ? acquire_on_hit : req_acquire || req_get); - assign need_probe = a_s3 && hit && has_clients && req_get && meta_on_hit[1:0] == TRUNK; - assign need_mshr = need_acquire || need_probe; + // the replacer's answer for a refill (:237-239) + logic retry, need_repl; + assign retry = repl_valid_s3 && repl_retry_s3; + assign need_repl = repl_valid_s3 && repl_state_s3 != INVALID; - // MainPipe's directory writes (MainPipe.scala:538-616): an A request - // answered here updates its line, a release that hits marks it dirty, a - // refill writes its tag and a fresh line - logic [METABITS-1:0] meta_w_a, meta_w_c, meta_w_m; + // MainPipe's directory writes (:538-616) + logic mw_a, mw_b, mw_c, mw_m, mw_cmo; + assign mw_a = a_s3 && !need_mshr_a && !req_get && !req_hint && !cmo; + assign mw_b = b_s3 && !need_mshr_b && hit && (state == TIP || state == BRANCH && snp_to_n); + assign mw_c = c_s3 && hit; + assign mw_m = m_s3 && task_s3.meta_wen && !(task_s3.refill && retry); + assign mw_cmo = cbo_inval && hit; + logic [METABITS-1:0] meta_w_a, meta_w_b, meta_w_c, meta_w_m; always_comb begin - meta_w_a = meta_on_hit; - meta_w_a[1:0] = task_s3.need_t ? TRUNK : meta_on_hit[1:0]; - meta_w_a[3] = 1'b1; // clients - meta_w_c = meta_on_hit; - meta_w_c[1:0] = TIP; - meta_w_c[2] = 1'b1; // dirty - meta_w_c[3] = 1'b0; - meta_w_m = '0; - meta_w_m[1:0] = TIP; + meta_w_a = meta_on_hit; + meta_w_a[1:0] = task_s3.need_t ? TRUNK : state; + meta_w_a[M_CLIENTS] = 1'b1; + meta_w_a[M_ALIAS+:2] = task_s3.alias_bits; + meta_w_b = meta_on_hit; + meta_w_b[1:0] = snp_to_n ? INVALID : BRANCH; + meta_w_c = meta_on_hit; + meta_w_c[1:0] = TIP; + meta_w_c[M_DIRTY] = dirty || task_s3.c_from_t; + meta_w_c[M_CLIENTS] = 1'b0; + meta_w_m = '0; + meta_w_m[1:0] = TIP; end - assign meta_wen = v_s3 && (a_s3 && !need_mshr && !req_get || c_s3 && hit || m_s3); - assign meta_wmeta = a_s3 ? meta_w_a : c_s3 ? meta_w_c : meta_w_m; + assign meta_wen = v_s3 && (mw_a || mw_b || mw_c || mw_m || mw_cmo); // :605-606 + assign meta_wmeta = mw_a ? meta_w_a : mw_b ? meta_w_b : mw_c ? meta_w_c : mw_m ? meta_w_m : '0; assign meta_wway_oh = m_s3 ? WAYS'(1) << task_s3.way : WAYS'(1) << dir_way; - assign tag_wen = v_s3 && m_s3; + assign tag_wen = v_s3 && m_s3 && task_s3.refill && !retry; // :613 assign tag_wway_oh = WAYS'(1) << task_s3.way; // ---- s3: MSHR allocation (MainPipe.scala:309, 1021-1059; @@ -303,12 +422,12 @@ module l2_dir_hit #( // state its hit decided ---- logic [5:0] alloc_state; assign alloc_state = { - !need_probe, // s_rprobe - !need_acquire, // s_acquire - hit, // w_replResp - !(need_acquire && req_acquire), // s_rcompack - !need_probe, // w_rprobeackfirst - !(hit && req_acquire_block) // s_refill + !need_pprobe_b, // s_pprobe + !need_acquire_a, // s_acquire + cmo || hit, // w_replResp (:1022) + !(need_acquire_a && !cmo), // s_rcompack (:1026) + !need_probe_a, // s_rprobe + !cmo // s_cmoresp }; l2_mshr_ctl #( .MSHRS (MSHRS), @@ -328,8 +447,11 @@ module l2_dir_hit #( // ---- s3: the L1 hint, "chnTask Hit will fire@s3" (CustomL1Hint.scala:91-126) ---- logic sink_resp, enq_s3, enq_has_data; logic [2:0] d_opcode; - assign sink_resp = a_s3 && !need_mshr; - assign d_opcode = req_get ? ACCESS_ACK_DATA : req_acquire_block ? GRANT_DATA : GRANT; + assign sink_resp = a_s3 && !need_mshr_a; // sink_resp_s3.valid (:432) + // odOpGen (MainPipe.scala:434): Get, AcquireBlock and AcquirePerm answer + // AccessAckData, GrantData and Grant; anything else no hint (HINT_ACK) + assign d_opcode = req_get ? ACCESS_ACK_DATA : req_acquire_block ? GRANT_DATA : + req_acquire ? GRANT : HINT_ACK; assign enq_s3 = v_s3 && sink_resp && (d_opcode == GRANT || d_opcode == GRANT_DATA || d_opcode == ACCESS_ACK_DATA); assign enq_has_data = d_opcode != GRANT; @@ -360,8 +482,12 @@ module l2_dir_hit #( // writes the MSHR's. logic ren, wen, ds_en, ds_req_valid, ds_cg_en; logic [WAYBITS-1:0] ds_way; - assign ren = a_s3 && hit && sink_resp && (req_get || req_acquire_block); - assign wen = c_s3 && hit || m_s3; + assign ren = hit && (req_get || req_acquire_block) || // need_data_a (:484-488) + b_s3 && (snp_resp_data || do_fwd || hit && state == TRUNK) || + task_s3.refill && need_repl && !retry || + cmo && hit && dirty; + assign wen = c_s3 && task_s3.c_from_t && hit || // wen_c (:490) + m_s3 && task_s3.ds_wen && (!task_s3.refill || !need_repl && !retry); assign ds_en = v_s3 && (ren || wen); assign ds_req_valid = v_s3_hold2 && (ren || wen); assign ds_way = m_s3 ? task_s3.way : dir_way; diff --git a/flow/designs/src/l2_dir_hit/l2_directory.sv b/flow/designs/src/l2_dir_hit/l2_directory.sv index 8076c6c34e..3cc5643f44 100644 --- a/flow/designs/src/l2_dir_hit/l2_directory.sv +++ b/flow/designs/src/l2_dir_hit/l2_directory.sv @@ -5,8 +5,9 @@ // // What is left out, because it is not on the path: the replacer's own // SRAM (its choice arrives as repl_way_oh_s3, as io.replResp does), the -// refill-retry free-way mask (:274-300), CMO by-way requests and the -// tag ECC check, which feeds only error_s3 (:259-268, :321), never the hit. +// refill-retry free-way mask (:274-300) and CMO by-way requests. The tag +// ECC check is kept: its error on the hit way goes to MainPipe's snoop +// decision the same cycle (errorOnSNP_s3, :259-268, :315-319). module l2_directory #( parameter int SETS = 1024, parameter int WAYS = 8, @@ -34,7 +35,8 @@ module l2_directory #( output logic hit_s3, output logic [$clog2(WAYS)-1:0] way_s3, output logic [METABITS-1:0] meta_s3, - output logic [METABITS-1:0] meta_on_hit_s3 + output logic [METABITS-1:0] meta_on_hit_s3, + output logic err_on_snp_s3 ); localparam int ENCTAG = TAGBITS + 7; // SECDED(32): 39 bits localparam int HALF = WAYS / 2; // waySplit = 2 (:173) @@ -60,6 +62,14 @@ module l2_directory #( return {^{c, d}, c, d}; endfunction + // the code's error flag, as dataCode.decode(tag).error: the check bits + // recomputed from the stored tag, against the stored ones and the parity + function automatic logic secded_err(input logic [ENCTAG-1:0] e); + logic [ENCTAG-1:0] r; + r = secded(e[TAGBITS-1:0]); + return r[ENCTAG-2:TAGBITS] != e[ENCTAG-2:TAGBITS] || ^e; + endfunction + // s1: one read of both arrays, writes when there is no read (single port) logic [ENCTAG*HALF-1:0] tag_rdata[2]; logic [ENCTAG*HALF-1:0] tag_wdata; @@ -109,13 +119,15 @@ module l2_directory #( // s3: hit and way, "in stage 3, Cuz SRAM latency is high under high // frequency" (:209-213, :270-272, :295-312) - logic [WAYS-1:0] hit_vec, inv_vec, inv_oh, chosen_oh, way_oh; + logic [WAYS-1:0] hit_vec, inv_vec, inv_oh, chosen_oh, way_oh, err_vec; for (genvar w = 0; w < WAYS; w++) begin : g_way logic [METABITS-1:0] m; assign m = meta_all_s3[w*METABITS+:METABITS]; assign inv_vec[w] = m[1:0] == INVALID; assign hit_vec[w] = tag_read_s3[w*ENCTAG+:TAGBITS] == tag_s3 && !inv_vec[w]; + assign err_vec[w] = secded_err(tag_read_s3[w*ENCTAG+:ENCTAG]); end + assign err_on_snp_s3 = |(hit_vec & err_vec); // Mux1H(hitOH, errorAll_s3) assign inv_oh = inv_vec & -inv_vec; // invalid_way_sel: the first invalid way assign chosen_oh = |inv_vec ? inv_oh : repl_way_oh_s3; assign hit_s3 = |hit_vec;